fsspec 2023.9.0__tar.gz → 2023.9.1__tar.gz

{fsspec-2023.9.0/fsspec.egg-info → fsspec-2023.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fsspec
-Version: 2023.9.0
+Version: 2023.9.1
 Summary: File-system specification
 Home-page: http://github.com/fsspec/filesystem_spec
 Maintainer: Martin Durant

{fsspec-2023.9.0 → fsspec-2023.9.1}/fsspec/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-09-02T07:50:40-0400",
+ "date": "2023-09-15T16:17:21-0400",
  "dirty": false,
  "error": null,
- "full-revisionid": "f5d24b80a0768bf07a113647d7b4e74a3a2999e0",
- "version": "2023.9.0"
+ "full-revisionid": "247b249a008990c584d2619f030bd42916a82e4a",
+ "version": "2023.9.1"
 }
 '''  # END VERSION_JSON
 

{fsspec-2023.9.0 → fsspec-2023.9.1}/fsspec/asyn.py

@@ -833,7 +833,7 @@ class AsyncFileSystem(AbstractFileSystem):
 
     async def _find(self, path, maxdepth=None, withdirs=False, **kwargs):
         path = self._strip_protocol(path)
-        out = dict()
+        out = {}
         detail = kwargs.pop("detail", False)
 
         # Add the root directory if withdirs is requested
@@ -891,7 +891,7 @@ class AsyncFileSystem(AbstractFileSystem):
                     out.add(p)
         if not out:
             raise FileNotFoundError(path)
-        return list(sorted(out))
+        return sorted(out)
 
     async def _mkdir(self, path, create_parents=True, **kwargs):
         pass  # not necessary to implement, may not have directories

{fsspec-2023.9.0 → fsspec-2023.9.1}/fsspec/caching.py

@@ -456,7 +456,7 @@ class KnownPartsOfAFile(BaseCache):
 
         # simple consolidation of contiguous blocks
         if data:
-            old_offsets = sorted(list(data.keys()))
+            old_offsets = sorted(data.keys())
             offsets = [old_offsets[0]]
             blocks = [data.pop(old_offsets[0])]
             for start, stop in old_offsets[1:]:

{fsspec-2023.9.0 → fsspec-2023.9.1}/fsspec/core.py

@@ -210,7 +210,6 @@ def open_files(
     num=1,
     protocol=None,
     newline=None,
-    auto_mkdir=True,
     expand=True,
     **kwargs,
 ):
@@ -249,9 +248,6 @@ def open_files(
     newline: bytes or None
         Used for line terminator in text mode. If None, uses system default;
         if blank, uses no translation.
-    auto_mkdir: bool (True)
-        If in write mode, this will ensure the target directory exists before
-        writing, by calling ``fs.mkdirs(exist_ok=True)``.
     expand: bool
     **kwargs: dict
         Extra options that make sense to a particular storage connection, e.g.
@@ -288,9 +284,6 @@ def open_files(
         protocol=protocol,
         expand=expand,
     )
-    if "r" not in mode and auto_mkdir:
-        parents = {fs._parent(path) for path in paths}
-        [fs.makedirs(parent, exist_ok=True) for parent in parents]
     return OpenFiles(
         [
             OpenFile(
@@ -363,7 +356,6 @@ def url_to_fs(url, **kwargs):
     # non-FS arguments that appear in fsspec.open()
     # inspect could keep this in sync with open()'s signature
     known_kwargs = {
-        "auto_mkdir",
         "compression",
         "encoding",
         "errors",
@@ -625,7 +617,7 @@ def get_fs_token_paths(
         pchains = [
             _un_chain(stringify_path(u), storage_options or {})[0] for u in urlpath
         ]
-        if len(set(pc[1] for pc in pchains)) > 1:
+        if len({pc[1] for pc in pchains}) > 1:
             raise ValueError("Protocol mismatch getting fs from %s", urlpath)
         paths = [pc[0] for pc in pchains]
     else:

{fsspec-2023.9.0 → fsspec-2023.9.1}/fsspec/gui.py

@@ -257,12 +257,14 @@ class FileSelector(SigSlot):
             width_policy="max",
         )
         self.protocol = pn.widgets.Select(
-            options=list(sorted(known_implementations)),
+            options=sorted(known_implementations),
             value=self.init_protocol,
             name="protocol",
             align="center",
         )
-        self.kwargs = pn.widgets.TextInput(name="kwargs", value="{}", align="center")
+        self.kwargs = pn.widgets.TextInput(
+            name="kwargs", value=self.init_kwargs, align="center"
+        )
         self.go = pn.widgets.Button(name="⇨", align="end", width=45)
         self.main = SingleSelect(size=10)
         self.home = pn.widgets.Button(name="🏠", width=40, height=30, align="end")

{fsspec-2023.9.0 → fsspec-2023.9.1}/fsspec/implementations/cache_metadata.py

@@ -7,6 +7,12 @@ from typing import TYPE_CHECKING
 
 from fsspec.utils import atomic_write
 
+try:
+    import ujson as json
+except ImportError:
+    if not TYPE_CHECKING:
+        import json
+
 if TYPE_CHECKING:
     from typing import Any, Dict, Iterator, Literal
 
@@ -23,7 +29,9 @@ class CacheMetadata:
     All reading and writing of cache metadata is performed by this class,
     accessing the cached files and blocks is not.
 
-    Metadata is stored in a single file per storage directory, pickled.
+    Metadata is stored in a single file per storage directory in JSON format.
+    For backward compatibility, also reads metadata stored in pickle format
+    which is converted to JSON when next saved.
     """
 
     def __init__(self, storage: list[str]):
@@ -41,6 +49,28 @@ class CacheMetadata:
         self._storage = storage
         self.cached_files: list[Detail] = [{}]
 
+        # Private attribute to force saving of metadata in pickle format rather than
+        # JSON for use in tests to confirm can read both pickle and JSON formats.
+        self._force_save_pickle = False
+
+    def _load(self, fn: str) -> Detail:
+        """Low-level function to load metadata from specific file"""
+        try:
+            with open(fn, "r") as f:
+                return json.load(f)
+        except ValueError:
+            with open(fn, "rb") as f:
+                return pickle.load(f)
+
+    def _save(self, metadata_to_save: Detail, fn: str) -> None:
+        """Low-level function to save metadata to specific file"""
+        if self._force_save_pickle:
+            with atomic_write(fn) as f:
+                pickle.dump(metadata_to_save, f)
+        else:
+            with atomic_write(fn, mode="w") as f:
+                json.dump(metadata_to_save, f)
+
     def _scan_locations(
         self, writable_only: bool = False
     ) -> Iterator[tuple[str, str, bool]]:
@@ -111,8 +141,7 @@ class CacheMetadata:
 
         if self.cached_files[-1]:
             cache_path = os.path.join(self._storage[-1], "cache")
-            with atomic_write(cache_path) as fc:
-                pickle.dump(self.cached_files[-1], fc)
+            self._save(self.cached_files[-1], cache_path)
 
         writable_cache_empty = not self.cached_files[-1]
         return expired_files, writable_cache_empty
@@ -122,13 +151,12 @@ class CacheMetadata:
         cached_files = []
         for fn, _, _ in self._scan_locations():
             if os.path.exists(fn):
-                with open(fn, "rb") as f:
-                    # TODO: consolidate blocks here
-                    loaded_cached_files = pickle.load(f)
-                    for c in loaded_cached_files.values():
-                        if isinstance(c["blocks"], list):
-                            c["blocks"] = set(c["blocks"])
-                    cached_files.append(loaded_cached_files)
+                # TODO: consolidate blocks here
+                loaded_cached_files = self._load(fn)
+                for c in loaded_cached_files.values():
+                    if isinstance(c["blocks"], list):
+                        c["blocks"] = set(c["blocks"])
+                cached_files.append(loaded_cached_files)
             else:
                 cached_files.append({})
         self.cached_files = cached_files or [{}]
@@ -170,8 +198,7 @@ class CacheMetadata:
                 continue
 
             if os.path.exists(fn):
-                with open(fn, "rb") as f:
-                    cached_files = pickle.load(f)
+                cached_files = self._load(fn)
                 for k, c in cached_files.items():
                     if k in cache:
                         if c["blocks"] is True or cache[k]["blocks"] is True:
@@ -197,8 +224,7 @@ class CacheMetadata:
             for c in cache.values():
                 if isinstance(c["blocks"], set):
                     c["blocks"] = list(c["blocks"])
-            with atomic_write(fn) as f:
-                pickle.dump(cache, f)
+            self._save(cache, fn)
             self.cached_files[-1] = cached_files
 
     def update_file(self, path: str, detail: Detail) -> None:

{fsspec-2023.9.0 → fsspec-2023.9.1}/fsspec/implementations/cached.py

@@ -5,6 +5,7 @@ import logging
 import os
 import tempfile
 import time
+import weakref
 from shutil import rmtree
 from typing import TYPE_CHECKING, Any, Callable, ClassVar
 
@@ -111,7 +112,9 @@ class CachingFileSystem(AbstractFileSystem):
                 "Both filesystems (fs) and target_protocol may not be both given."
             )
         if cache_storage == "TMP":
-            storage = [tempfile.mkdtemp()]
+            tempdir = tempfile.mkdtemp()
+            storage = [tempdir]
+            weakref.finalize(self, self._remove_tempdir, tempdir)
         else:
             if isinstance(cache_storage, str):
                 storage = [cache_storage]
@@ -152,6 +155,13 @@ class CachingFileSystem(AbstractFileSystem):
 
         self._strip_protocol: Callable = _strip_protocol
 
+    @staticmethod
+    def _remove_tempdir(tempdir):
+        try:
+            rmtree(tempdir)
+        except Exception:
+            pass
+
     def _mkcache(self):
         os.makedirs(self.storage[-1], exist_ok=True)
 

{fsspec-2023.9.0 → fsspec-2023.9.1}/fsspec/implementations/http.py

@@ -192,7 +192,7 @@ class HTTPFileSystem(AsyncFileSystem):
                 for u in out
             ]
         else:
-            return list(sorted(out))
+            return sorted(out)
 
     async def _ls(self, url, detail=True, **kwargs):
         if self.use_listings_cache and url in self.dircache:

{fsspec-2023.9.0 → fsspec-2023.9.1}/fsspec/implementations/reference.py

@@ -150,7 +150,7 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         """List top-level directories"""
         if self.dirs is None:
             dirs = [p.split("/", 1)[0] for p in self.zmetadata]
-            self.dirs = set(sorted(p for p in dirs if p and not p.startswith(".")))
+            self.dirs = {p for p in dirs if p and not p.startswith(".")}
         listing = self.dirs
         if basename:
             listing = [os.path.basename(path) for path in listing]
@@ -381,17 +381,17 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
                 raws[j] = kerchunk.df._proc_raw(data)
         # TODO: only save needed columns
         df = pd.DataFrame(
-            dict(
-                path=paths,
-                offset=offsets,
-                size=sizes,
-                raw=raws,
-            ),
+            {
+                "path": paths,
+                "offset": offsets,
+                "size": sizes,
+                "raw": raws,
+            },
             copy=False,
         )
         if df.path.count() / (df.path.nunique() or 1) > self.cat_thresh:
             df["path"] = df["path"].astype("category")
-        object_encoding = dict(raw="bytes", path="utf8")
+        object_encoding = {"raw": "bytes", "path": "utf8"}
         has_nulls = ["path", "raw"]
 
         self.fs.mkdirs(f"{base_url or self.out_root}/{field}", exist_ok=True)

{fsspec-2023.9.0 → fsspec-2023.9.1}/fsspec/implementations/smb.py

@@ -81,7 +81,7 @@ class SMBFileSystem(AbstractFileSystem):
         ----------
         host: str
             The remote server name/ip to connect to
-        port: int
+        port: int or None
             Port to connect with. Usually 445, sometimes 139.
         username: str or None
             Username to connect with. Required if Kerberos auth is not being used.
@@ -114,12 +114,16 @@ class SMBFileSystem(AbstractFileSystem):
         self.share_access = share_access
         self._connect()
 
+    @property
+    def _port(self):
+        return 445 if self.port is None else self.port
+
     def _connect(self):
         smbclient.register_session(
             self.host,
             username=self.username,
             password=self.password,
-            port=445 if self.port is None else self.port,
+            port=self._port,
             encrypt=self.encrypt,
             connection_timeout=self.timeout,
         )
@@ -139,23 +143,23 @@ class SMBFileSystem(AbstractFileSystem):
     def mkdir(self, path, create_parents=True, **kwargs):
         wpath = _as_unc_path(self.host, path)
         if create_parents:
-            smbclient.makedirs(wpath, exist_ok=False, port=self.port, **kwargs)
+            smbclient.makedirs(wpath, exist_ok=False, port=self._port, **kwargs)
         else:
-            smbclient.mkdir(wpath, port=self.port, **kwargs)
+            smbclient.mkdir(wpath, port=self._port, **kwargs)
 
     def makedirs(self, path, exist_ok=False):
         if _share_has_path(path):
             wpath = _as_unc_path(self.host, path)
-            smbclient.makedirs(wpath, exist_ok=exist_ok, port=self.port)
+            smbclient.makedirs(wpath, exist_ok=exist_ok, port=self._port)
 
     def rmdir(self, path):
         if _share_has_path(path):
             wpath = _as_unc_path(self.host, path)
-            smbclient.rmdir(wpath, port=self.port)
+            smbclient.rmdir(wpath, port=self._port)
 
     def info(self, path, **kwargs):
         wpath = _as_unc_path(self.host, path)
-        stats = smbclient.stat(wpath, port=self.port, **kwargs)
+        stats = smbclient.stat(wpath, port=self._port, **kwargs)
         if S_ISDIR(stats.st_mode):
             stype = "directory"
         elif S_ISLNK(stats.st_mode):
@@ -176,18 +180,18 @@ class SMBFileSystem(AbstractFileSystem):
     def created(self, path):
         """Return the created timestamp of a file as a datetime.datetime"""
         wpath = _as_unc_path(self.host, path)
-        stats = smbclient.stat(wpath, port=self.port)
+        stats = smbclient.stat(wpath, port=self._port)
         return datetime.datetime.fromtimestamp(stats.st_ctime, tz=datetime.timezone.utc)
 
     def modified(self, path):
         """Return the modified timestamp of a file as a datetime.datetime"""
         wpath = _as_unc_path(self.host, path)
-        stats = smbclient.stat(wpath, port=self.port)
+        stats = smbclient.stat(wpath, port=self._port)
         return datetime.datetime.fromtimestamp(stats.st_mtime, tz=datetime.timezone.utc)
 
     def ls(self, path, detail=True, **kwargs):
         unc = _as_unc_path(self.host, path)
-        listed = smbclient.listdir(unc, port=self.port, **kwargs)
+        listed = smbclient.listdir(unc, port=self._port, **kwargs)
         dirs = ["/".join([path.rstrip("/"), p]) for p in listed]
         if detail:
             dirs = [self.info(d) for d in dirs]
@@ -218,14 +222,14 @@ class SMBFileSystem(AbstractFileSystem):
         if "w" in mode and autocommit is False:
             temp = _as_temp_path(self.host, path, self.temppath)
             return SMBFileOpener(
-                wpath, temp, mode, port=self.port, block_size=bls, **kwargs
+                wpath, temp, mode, port=self._port, block_size=bls, **kwargs
             )
         return smbclient.open_file(
             wpath,
             mode,
             buffering=bls,
             share_access=share_access,
-            port=self.port,
+            port=self._port,
             **kwargs,
         )
 
@@ -233,21 +237,21 @@ class SMBFileSystem(AbstractFileSystem):
         """Copy within two locations in the same filesystem"""
         wpath1 = _as_unc_path(self.host, path1)
         wpath2 = _as_unc_path(self.host, path2)
-        smbclient.copyfile(wpath1, wpath2, port=self.port, **kwargs)
+        smbclient.copyfile(wpath1, wpath2, port=self._port, **kwargs)
 
     def _rm(self, path):
         if _share_has_path(path):
             wpath = _as_unc_path(self.host, path)
-            stats = smbclient.stat(wpath, port=self.port)
+            stats = smbclient.stat(wpath, port=self._port)
             if S_ISDIR(stats.st_mode):
-                smbclient.rmdir(wpath, port=self.port)
+                smbclient.rmdir(wpath, port=self._port)
             else:
-                smbclient.remove(wpath, port=self.port)
+                smbclient.remove(wpath, port=self._port)
 
     def mv(self, path1, path2, **kwargs):
         wpath1 = _as_unc_path(self.host, path1)
         wpath2 = _as_unc_path(self.host, path2)
-        smbclient.rename(wpath1, wpath2, port=self.port, **kwargs)
+        smbclient.rename(wpath1, wpath2, port=self._port, **kwargs)
 
 
 def _as_unc_path(host, path):

{fsspec-2023.9.0 → fsspec-2023.9.1}/fsspec/spec.py

@@ -485,7 +485,7 @@ class AbstractFileSystem(metaclass=_Cached):
         """
         # TODO: allow equivalent of -name parameter
         path = self._strip_protocol(path)
-        out = dict()
+        out = {}
 
         # Add the root directory if withdirs is requested
         # This is needed for posix glob compliance
@@ -1182,7 +1182,7 @@ class AbstractFileSystem(metaclass=_Cached):
                     out.add(p)
         if not out:
             raise FileNotFoundError(path)
-        return list(sorted(out))
+        return sorted(out)
 
     def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs):
         """Move file(s) from one location to another"""

{fsspec-2023.9.0 → fsspec-2023.9.1/fsspec.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fsspec
-Version: 2023.9.0
+Version: 2023.9.1
 Summary: File-system specification
 Home-page: http://github.com/fsspec/filesystem_spec
 Maintainer: Martin Durant