fsspec 2023.9.2__py3-none-any.whl → 2023.12.0__py3-none-any.whl

fsspec/implementations/jupyter.py

@@ -40,7 +40,7 @@ class JupyterFileSystem(fsspec.AbstractFileSystem):
 
     def ls(self, path, detail=True, **kwargs):
         path = self._strip_protocol(path)
-        r = self.session.get(self.url + "/" + path)
+        r = self.session.get(f"{self.url}/{path}")
         if r.status_code == 404:
             return FileNotFoundError(path)
         r.raise_for_status()
@@ -61,7 +61,7 @@ class JupyterFileSystem(fsspec.AbstractFileSystem):
 
     def cat_file(self, path, start=None, end=None, **kwargs):
         path = self._strip_protocol(path)
-        r = self.session.get(self.url + "/" + path)
+        r = self.session.get(f"{self.url}/{path}")
         if r.status_code == 404:
             return FileNotFoundError(path)
         r.raise_for_status()
@@ -83,7 +83,7 @@ class JupyterFileSystem(fsspec.AbstractFileSystem):
             "format": "base64",
             "type": "file",
         }
-        self.session.put(self.url + "/" + path, json=json)
+        self.session.put(f"{self.url}/{path}", json=json)
 
     def mkdir(self, path, create_parents=True, **kwargs):
         path = self._strip_protocol(path)
@@ -96,11 +96,11 @@ class JupyterFileSystem(fsspec.AbstractFileSystem):
             "content": None,
             "type": "directory",
         }
-        self.session.put(self.url + "/" + path, json=json)
+        self.session.put(f"{self.url}/{path}", json=json)
 
     def _rm(self, path):
         path = self._strip_protocol(path)
-        self.session.delete(self.url + "/" + path)
+        self.session.delete(f"{self.url}/{path}")
 
     def _open(self, path, mode="rb", **kwargs):
         path = self._strip_protocol(path)

fsspec/implementations/libarchive.py

@@ -122,8 +122,7 @@ class LibArchiveFileSystem(AbstractArchiveFileSystem):
             files = open_files(fo, protocol=target_protocol, **(target_options or {}))
             if len(files) != 1:
                 raise ValueError(
-                    'Path "{}" did not resolve to exactly'
-                    'one file: "{}"'.format(fo, files)
+                    f'Path "{fo}" did not resolve to exactly one file: "{files}"'
                 )
             fo = files[0]
         self.of = fo

fsspec/implementations/local.py

@@ -29,7 +29,7 @@ class LocalFileSystem(AbstractFileSystem):
     """
 
     root_marker = "/"
-    protocol = "file"
+    protocol = "file", "local"
     local_file = True
 
     def __init__(self, auto_mkdir=False, **kwargs):
@@ -98,7 +98,7 @@ class LocalFileSystem(AbstractFileSystem):
             "islink": link,
         }
         for field in ["mode", "uid", "gid", "mtime", "ino", "nlink"]:
-            result[field] = getattr(out, "st_" + field)
+            result[field] = getattr(out, f"st_{field}")
         if result["islink"]:
             result["destination"] = os.readlink(path)
             try:
@@ -215,6 +215,10 @@ class LocalFileSystem(AbstractFileSystem):
             path = path[7:]
         elif path.startswith("file:"):
             path = path[5:]
+        elif path.startswith("local://"):
+            path = path[8:]
+        elif path.startswith("local:"):
+            path = path[6:]
         return make_path_posix(path).rstrip("/") or cls.root_marker
 
     def _isfilestore(self):
@@ -240,7 +244,7 @@ def make_path_posix(path, sep=os.sep):
             return path
         if path.startswith("./"):
             path = path[2:]
-        return os.getcwd() + "/" + path
+        return f"{os.getcwd()}/{path}"
     if (
         (sep not in path and "/" not in path)
         or (sep == "/" and not path.startswith("/"))
@@ -251,7 +255,7 @@ def make_path_posix(path, sep=os.sep):
             # abspath made some more '\\' separators
             return make_path_posix(osp.abspath(path))
         else:
-            return os.getcwd() + "/" + path
+            return f"{os.getcwd()}/{path}"
     if path.startswith("file://"):
         path = path[7:]
     if re.match("/[A-Za-z]:", path):

fsspec/implementations/memory.py

@@ -175,7 +175,7 @@ class MemoryFileSystem(AbstractFileSystem):
             parent = self._parent(parent)
             if self.isfile(parent):
                 raise FileExistsError(parent)
-        if mode in ["rb", "ab", "rb+"]:
+        if mode in ["rb", "ab", "r+b"]:
             if path in self.store:
                 f = self.store[path]
                 if mode == "ab":

fsspec/implementations/reference.py

@@ -82,8 +82,12 @@ def ravel_multi_index(idx, sizes):
 
 
 class LazyReferenceMapper(collections.abc.MutableMapping):
-    """Interface to read parquet store as if it were a standard kerchunk
-    references dict."""
+    """This interface can be used to read/write references from Parquet stores.
+    It is not intended for other types of references.
+    It can be used with Kerchunk's MultiZarrToZarr method to combine
+    references into a parquet store.
+    Examples of this use-case can be found here:
+    https://fsspec.github.io/kerchunk/advanced.html?highlight=parquet#parquet-storage"""
 
     # import is class level to prevent numpy dep requirement for fsspec
     @property
@@ -108,17 +112,24 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
             Root of parquet store
         fs : fsspec.AbstractFileSystem
             fsspec filesystem object, default is local filesystem.
-        cache_size : int
+        cache_size : int, default=128
             Maximum size of LRU cache, where cache_size*record_size denotes
             the total number of references that can be loaded in memory at once.
+        categorical_threshold : int
+            Encode urls as pandas.Categorical to reduce memory footprint if the ratio
+            of the number of unique urls to total number of refs for each variable
+            is greater than or equal to this number. (default 10)
+
+
         """
         self.root = root
         self.chunk_sizes = {}
         self._items = {}
         self.dirs = None
         self.fs = fsspec.filesystem("file") if fs is None else fs
-        with self.fs.open("/".join([self.root, ".zmetadata"]), "rb") as f:
-            self._items[".zmetadata"] = f.read()
+        self._items[".zmetadata"] = self.fs.cat_file(
+            "/".join([self.root, ".zmetadata"])
+        )
         met = json.loads(self._items[".zmetadata"])
         self.record_size = met["record_size"]
         self.zmetadata = met["metadata"]
@@ -131,18 +142,37 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         def open_refs(field, record):
             """cached parquet file loader"""
             path = self.url.format(field=field, record=record)
-            with self.fs.open(path) as f:
-                # TODO: since all we do is iterate, is arrow without pandas
-                #  better here?
-                df = self.pd.read_parquet(f, engine="fastparquet")
+            data = io.BytesIO(self.fs.cat_file(path))
+            df = self.pd.read_parquet(data, engine="fastparquet")
             refs = {c: df[c].values for c in df.columns}
             return refs
 
         self.open_refs = open_refs
 
     @staticmethod
-    def create(record_size, root, fs, **kwargs):
+    def create(root, storage_options=None, fs=None, record_size=10000, **kwargs):
+        """Make empty parquet reference set
+
+        Parameters
+        ----------
+        root: str
+            Directory to contain the output; will be created
+        storage_options: dict | None
+            For making the filesystem to use for writing is fs is None
+        fs: FileSystem | None
+            Filesystem for writing
+        record_size: int
+            Number of references per parquet file
+        kwargs: passed to __init__
+
+        Returns
+        -------
+        LazyReferenceMapper instance
+        """
         met = {"metadata": {}, "record_size": record_size}
+        if fs is None:
+            fs, root = fsspec.core.url_to_fs(root, **(storage_options or {}))
+        fs.makedirs(root, exist_ok=True)
         fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode())
         return LazyReferenceMapper(root, fs, **kwargs)
 
@@ -283,7 +313,7 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
     def _generate_record(self, field, record):
         """The references for a given parquet file of a given field"""
         refs = self.open_refs(field, record)
-        it = iter(zip(refs.values()))
+        it = iter(zip(*refs.values()))
         if len(refs) == 3:
             # All urls
             return (list(t) for t in it)
@@ -594,7 +624,7 @@ class ReferenceFileSystem(AsyncFileSystem):
                 **(ref_storage_args or target_options or {}), protocol=target_protocol
             )
             ref_fs, fo2 = fsspec.core.url_to_fs(fo, **dic)
-            if ref_fs.isfile(fo):
+            if ref_fs.isfile(fo2):
                 # text JSON
                 with fsspec.open(fo, "rb", **dic) as f:
                     logger.info("Read reference from URL %s", fo)
@@ -641,6 +671,7 @@ class ReferenceFileSystem(AsyncFileSystem):
                     self.fss[protocol] = fs
         if remote_protocol is None:
             # get single protocol from references
+            # TODO: warning here, since this can be very expensive?
             for ref in self.references.values():
                 if callable(ref):
                     ref = ref()
@@ -763,24 +794,27 @@ class ReferenceFileSystem(AsyncFileSystem):
             raise NotImplementedError
         if isinstance(path, list) and (recursive or any("*" in p for p in path)):
             raise NotImplementedError
+        # TODO: if references is lazy, pre-fetch all paths in batch before access
         proto_dict = _protocol_groups(path, self.references)
         out = {}
         for proto, paths in proto_dict.items():
             fs = self.fss[proto]
-            urls, starts, ends = [], [], []
+            urls, starts, ends, valid_paths = [], [], [], []
             for p in paths:
                 # find references or label not-found. Early exit if any not
                 # found and on_error is "raise"
                 try:
                     u, s, e = self._cat_common(p)
-                    urls.append(u)
-                    starts.append(s)
-                    ends.append(e)
                 except FileNotFoundError as err:
                     if on_error == "raise":
                         raise
                     if on_error != "omit":
                         out[p] = err
+                else:
+                    urls.append(u)
+                    starts.append(s)
+                    ends.append(e)
+                    valid_paths.append(p)
 
             # process references into form for merging
             urls2 = []
@@ -788,7 +822,7 @@ class ReferenceFileSystem(AsyncFileSystem):
             ends2 = []
             paths2 = []
             whole_files = set()
-            for u, s, e, p in zip(urls, starts, ends, paths):
+            for u, s, e, p in zip(urls, starts, ends, valid_paths):
                 if isinstance(u, bytes):
                     # data
                     out[p] = u
@@ -800,7 +834,7 @@ class ReferenceFileSystem(AsyncFileSystem):
                     starts2.append(s)
                     ends2.append(e)
                     paths2.append(p)
-            for u, s, e, p in zip(urls, starts, ends, paths):
+            for u, s, e, p in zip(urls, starts, ends, valid_paths):
                 # second run to account for files that are to be loaded whole
                 if s is not None and u not in whole_files:
                     urls2.append(u)
@@ -820,7 +854,7 @@ class ReferenceFileSystem(AsyncFileSystem):
             bytes_out = fs.cat_ranges(new_paths, new_starts, new_ends)
 
             # unbundle from merged bytes - simple approach
-            for u, s, e, p in zip(urls, starts, ends, paths):
+            for u, s, e, p in zip(urls, starts, ends, valid_paths):
                 if p in out:
                     continue  # was bytes, already handled
                 for np, ns, ne, b in zip(new_paths, new_starts, new_ends, bytes_out):
@@ -954,16 +988,24 @@ class ReferenceFileSystem(AsyncFileSystem):
             elif len(part) == 1:
                 size = None
             else:
-                _, start, size = part
+                _, _, size = part
             par = path.rsplit("/", 1)[0] if "/" in path else ""
             par0 = par
+            subdirs = [par0]
             while par0 and par0 not in self.dircache:
-                # build parent directories
-                self.dircache[par0] = []
-                self.dircache.setdefault(
-                    par0.rsplit("/", 1)[0] if "/" in par0 else "", []
-                ).append({"name": par0, "type": "directory", "size": 0})
+                # collect parent directories
                 par0 = self._parent(par0)
+                subdirs.append(par0)
+
+            subdirs = subdirs[::-1]
+            for parent, child in zip(subdirs, subdirs[1:]):
+                # register newly discovered directories
+                assert child not in self.dircache
+                assert parent in self.dircache
+                self.dircache[parent].append(
+                    {"name": child, "type": "directory", "size": 0}
+                )
+                self.dircache[child] = []
 
             self.dircache[par].append({"name": path, "type": "file", "size": size})
 

fsspec/implementations/sftp.py

@@ -41,14 +41,14 @@ class SFTPFileSystem(AbstractFileSystem):
         """
         if self._cached:
             return
-        super(SFTPFileSystem, self).__init__(**ssh_kwargs)
+        super().__init__(**ssh_kwargs)
         self.temppath = ssh_kwargs.pop("temppath", "/tmp")  # remote temp directory
         self.host = host
         self.ssh_kwargs = ssh_kwargs
         self._connect()
 
     def _connect(self):
-        logger.debug("Connecting to SFTP server %s" % self.host)
+        logger.debug("Connecting to SFTP server %s", self.host)
         self.client = paramiko.SSHClient()
         self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
         self.client.connect(self.host, **self.ssh_kwargs)
@@ -66,9 +66,9 @@ class SFTPFileSystem(AbstractFileSystem):
         return out
 
     def mkdir(self, path, create_parents=False, mode=511):
-        logger.debug("Creating folder %s" % path)
+        logger.debug("Creating folder %s", path)
         if self.exists(path):
-            raise FileExistsError("File exists: {}".format(path))
+            raise FileExistsError(f"File exists: {path}")
 
         if create_parents:
             self.makedirs(path)
@@ -77,18 +77,18 @@ class SFTPFileSystem(AbstractFileSystem):
 
     def makedirs(self, path, exist_ok=False, mode=511):
         if self.exists(path) and not exist_ok:
-            raise FileExistsError("File exists: {}".format(path))
+            raise FileExistsError(f"File exists: {path}")
 
         parts = path.split("/")
         path = ""
 
         for part in parts:
-            path += "/" + part
+            path += f"/{part}"
             if not self.exists(path):
                 self.ftp.mkdir(path, mode)
 
     def rmdir(self, path):
-        logger.debug("Removing folder %s" % path)
+        logger.debug("Removing folder %s", path)
         self.ftp.rmdir(path)
 
     def info(self, path):
@@ -122,7 +122,7 @@ class SFTPFileSystem(AbstractFileSystem):
         return out
 
     def ls(self, path, detail=False):
-        logger.debug("Listing folder %s" % path)
+        logger.debug("Listing folder %s", path)
         stats = [self._decode_stat(stat, path) for stat in self.ftp.listdir_iter(path)]
         if detail:
             return stats
@@ -131,7 +131,7 @@ class SFTPFileSystem(AbstractFileSystem):
             return sorted(paths)
 
     def put(self, lpath, rpath, callback=None, **kwargs):
-        logger.debug("Put file %s into %s" % (lpath, rpath))
+        logger.debug("Put file %s into %s", lpath, rpath)
         self.ftp.put(lpath, rpath)
 
     def get_file(self, rpath, lpath, **kwargs):
@@ -146,7 +146,7 @@ class SFTPFileSystem(AbstractFileSystem):
             If 0, no buffering, if 1, line buffering, if >1, buffer that many
             bytes, if None use default from paramiko.
         """
-        logger.debug("Opening file %s" % path)
+        logger.debug("Opening file %s", path)
         if kwargs.get("autocommit", True) is False:
             # writes to temporary file, move on commit
             path2 = "/".join([self.temppath, str(uuid.uuid4())])
@@ -167,7 +167,7 @@ class SFTPFileSystem(AbstractFileSystem):
             self.ftp.remove(path)
 
     def mv(self, old, new):
-        logger.debug("Renaming %s into %s" % (old, new))
+        logger.debug("Renaming %s into %s", old, new)
         self.ftp.posix_rename(old, new)
 
 

fsspec/implementations/smb.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
 This module contains SMBFileSystem class responsible for handling access to
 Windows Samba network shares by using package smbprotocol
@@ -103,7 +102,7 @@ class SMBFileSystem(AbstractFileSystem):
             - 'w': Allow other handles to be opened with write access.
             - 'd': Allow other handles to be opened with delete access.
         """
-        super(SMBFileSystem, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.host = host
         self.port = port
         self.username = username
@@ -248,7 +247,7 @@ class SMBFileSystem(AbstractFileSystem):
             else:
                 smbclient.remove(wpath, port=self._port)
 
-    def mv(self, path1, path2, **kwargs):
+    def mv(self, path1, path2, recursive=None, maxdepth=None, **kwargs):
         wpath1 = _as_unc_path(self.host, path1)
         wpath2 = _as_unc_path(self.host, path2)
         smbclient.rename(wpath1, wpath2, port=self._port, **kwargs)
@@ -256,13 +255,13 @@ class SMBFileSystem(AbstractFileSystem):
 
 def _as_unc_path(host, path):
     rpath = path.replace("/", "\\")
-    unc = "\\\\{}{}".format(host, rpath)
+    unc = f"\\\\{host}{rpath}"
     return unc
 
 
 def _as_temp_path(host, path, temppath):
     share = path.split("/")[1]
-    temp_file = "/{}{}/{}".format(share, temppath, uuid.uuid4())
+    temp_file = f"/{share}{temppath}/{uuid.uuid4()}"
     unc = _as_unc_path(host, temp_file)
     return unc
 

fsspec/implementations/webhdfs.py

@@ -21,7 +21,7 @@ class WebHDFS(AbstractFileSystem):
     """
     Interface to HDFS over HTTP using the WebHDFS API. Supports also HttpFS gateways.
 
-    Three auth mechanisms are supported:
+    Four auth mechanisms are supported:
 
     insecure: no auth is done, and the user is assumed to be whoever they
         say they are (parameter ``user``), or a predefined value such as
@@ -34,6 +34,8 @@ class WebHDFS(AbstractFileSystem):
         service. Indeed, this client can also generate such tokens when
         not insecure. Note that tokens expire, but can be renewed (by a
         previously specified user) and may allow for proxying.
+    basic-auth: used when both parameter ``user`` and parameter ``password``
+        are provided.
 
     """
 
@@ -47,6 +49,7 @@ class WebHDFS(AbstractFileSystem):
         kerberos=False,
         token=None,
         user=None,
+        password=None,
         proxy_to=None,
         kerb_kwargs=None,
         data_proxy=None,
@@ -68,6 +71,9 @@ class WebHDFS(AbstractFileSystem):
             given
         user: str or None
             If given, assert the user name to connect with
+        password: str or None
+            If given, assert the password to use for basic auth. If password
+            is provided, user must be provided also
         proxy_to: str or None
             If given, the user has the authority to proxy, and this value is
             the user in who's name actions are taken
@@ -89,9 +95,7 @@ class WebHDFS(AbstractFileSystem):
         if self._cached:
             return
         super().__init__(**kwargs)
-        self.url = "{protocol}://{host}:{port}/webhdfs/v1".format(
-            protocol="https" if use_https else "http", host=host, port=port
-        )
+        self.url = f"{'https' if use_https else 'http'}://{host}:{port}/webhdfs/v1"
         self.kerb = kerberos
         self.kerb_kwargs = kerb_kwargs or {}
         self.pars = {}
@@ -104,8 +108,19 @@ class WebHDFS(AbstractFileSystem):
                     " token"
                 )
             self.pars["delegation"] = token
-        if user is not None:
-            self.pars["user.name"] = user
+        self.user = user
+        self.password = password
+
+        if password is not None:
+            if user is None:
+                raise ValueError(
+                    "If passing a password, the user must also be"
+                    "set in order to set up the basic-auth"
+                )
+        else:
+            if user is not None:
+                self.pars["user.name"] = user
+
         if proxy_to is not None:
             self.pars["doas"] = proxy_to
         if kerberos and user is not None:
@@ -115,7 +130,7 @@ class WebHDFS(AbstractFileSystem):
             )
         self._connect()
 
-        self._fsid = "webhdfs_" + tokenize(host, port)
+        self._fsid = f"webhdfs_{tokenize(host, port)}"
 
     @property
     def fsid(self):
@@ -128,8 +143,13 @@ class WebHDFS(AbstractFileSystem):
 
             self.session.auth = HTTPKerberosAuth(**self.kerb_kwargs)
 
+        if self.user is not None and self.password is not None:
+            from requests.auth import HTTPBasicAuth
+
+            self.session.auth = HTTPBasicAuth(self.user, self.password)
+
     def _call(self, op, method="get", path=None, data=None, redirect=True, **kwargs):
-        url = self.url + quote(path or "")
+        url = self._apply_proxy(self.url + quote(path or "", safe="/="))
         args = kwargs.copy()
         args.update(self.pars)
         args["op"] = op.upper()

fsspec/implementations/zip.py

@@ -113,10 +113,10 @@ class ZipFileSystem(AbstractArchiveFileSystem):
         path = self._strip_protocol(path)
         if "r" in mode and self.mode in set("wa"):
             if self.exists(path):
-                raise IOError("ZipFS can only be open for reading or writing, not both")
+                raise OSError("ZipFS can only be open for reading or writing, not both")
             raise FileNotFoundError(path)
         if "r" in self.mode and "w" in mode:
-            raise IOError("ZipFS can only be open for reading or writing, not both")
+            raise OSError("ZipFS can only be open for reading or writing, not both")
         out = self.zip.open(path, mode.strip("b"))
         if "r" in mode:
             info = self.info(path)

fsspec/mapping.py

@@ -54,8 +54,8 @@ class FSMap(MutableMapping):
         if check:
             if not self.fs.exists(root):
                 raise ValueError(
-                    "Path %s does not exist. Create "
-                    " with the ``create=True`` keyword" % root
+                    f"Path {root} does not exist. Create "
+                    f" with the ``create=True`` keyword"
                 )
             self.fs.touch(root + "/a")
             self.fs.rm(root + "/a")

fsspec/registry.py

@@ -38,29 +38,31 @@ def register_implementation(name, cls, clobber=False, errtxt=None):
         if name in known_implementations and clobber is False:
             if cls != known_implementations[name]["class"]:
                 raise ValueError(
-                    "Name (%s) already in the known_implementations and clobber "
-                    "is False" % name
+                    f"Name ({name}) already in the known_implementations and clobber "
+                    f"is False"
                 )
         else:
             known_implementations[name] = {
                 "class": cls,
-                "err": errtxt or "%s import failed for protocol %s" % (cls, name),
+                "err": errtxt or f"{cls} import failed for protocol {name}",
             }
 
     else:
         if name in registry and clobber is False:
             if _registry[name] is not cls:
                 raise ValueError(
-                    "Name (%s) already in the registry and clobber is False" % name
+                    f"Name ({name}) already in the registry and clobber is False"
                 )
         else:
             _registry[name] = cls
 
 
-# protocols mapped to the class which implements them. This dict can
+# protocols mapped to the class which implements them. This dict can be
 # updated with register_implementation
 known_implementations = {
+    "data": {"class": "fsspec.implementations.data.DataFileSystem"},
     "file": {"class": "fsspec.implementations.local.LocalFileSystem"},
+    "local": {"class": "fsspec.implementations.local.LocalFileSystem"},
     "memory": {"class": "fsspec.implementations.memory.MemoryFileSystem"},
     "dropbox": {
         "class": "dropboxdrivefs.DropboxDriveFileSystem",
@@ -228,7 +230,7 @@ def get_filesystem_class(protocol):
 
     if protocol not in registry:
         if protocol not in known_implementations:
-            raise ValueError("Protocol not known: %s" % protocol)
+            raise ValueError(f"Protocol not known: {protocol}")
         bit = known_implementations[protocol]
         try:
             register_implementation(protocol, _import_class(bit["class"]))