fsspec 2024.3.0__py3-none-any.whl → 2024.5.0__py3-none-any.whl

fsspec/implementations/tests/test_webhdfs.py

@@ -0,0 +1,197 @@
+import pickle
+import shlex
+import subprocess
+import time
+
+import pytest
+
+import fsspec
+
+requests = pytest.importorskip("requests")
+
+from fsspec.implementations.webhdfs import WebHDFS  # noqa: E402
+
+
+@pytest.fixture(scope="module")
+def hdfs_cluster():
+    cmd0 = shlex.split("htcluster shutdown")
+    try:
+        subprocess.check_output(cmd0, stderr=subprocess.STDOUT)
+    except FileNotFoundError:
+        pytest.skip("htcluster not found")
+    except subprocess.CalledProcessError as ex:
+        pytest.skip(f"htcluster failed: {ex.output.decode()}")
+    cmd1 = shlex.split("htcluster startup --image base")
+    subprocess.check_output(cmd1)
+    try:
+        while True:
+            t = 90
+            try:
+                requests.get("http://localhost:50070/webhdfs/v1/?op=LISTSTATUS")
+            except:  # noqa: E722
+                t -= 1
+                assert t > 0, "Timeout waiting for HDFS"
+                time.sleep(1)
+                continue
+            break
+        time.sleep(7)
+        yield "localhost"
+    finally:
+        subprocess.check_output(cmd0)
+
+
+def test_pickle(hdfs_cluster):
+    w = WebHDFS(hdfs_cluster, user="testuser")
+    w2 = pickle.loads(pickle.dumps(w))
+    assert w == w2
+
+
+def test_simple(hdfs_cluster):
+    w = WebHDFS(hdfs_cluster, user="testuser")
+    home = w.home_directory()
+    assert home == "/user/testuser"
+    with pytest.raises(PermissionError):
+        w.mkdir("/root")
+
+
+def test_url(hdfs_cluster):
+    url = "webhdfs://testuser@localhost:50070/user/testuser/myfile"
+    fo = fsspec.open(url, "wb", data_proxy={"worker.example.com": "localhost"})
+    with fo as f:
+        f.write(b"hello")
+    fo = fsspec.open(url, "rb", data_proxy={"worker.example.com": "localhost"})
+    with fo as f:
+        assert f.read() == b"hello"
+
+
+def test_workflow(hdfs_cluster):
+    w = WebHDFS(
+        hdfs_cluster, user="testuser", data_proxy={"worker.example.com": "localhost"}
+    )
+    fn = "/user/testuser/testrun/afile"
+    w.mkdir("/user/testuser/testrun")
+    with w.open(fn, "wb") as f:
+        f.write(b"hello")
+    assert w.exists(fn)
+    info = w.info(fn)
+    assert info["size"] == 5
+    assert w.isfile(fn)
+    assert w.cat(fn) == b"hello"
+    w.rm("/user/testuser/testrun", recursive=True)
+    assert not w.exists(fn)
+
+
+def test_with_gzip(hdfs_cluster):
+    from gzip import GzipFile
+
+    w = WebHDFS(
+        hdfs_cluster, user="testuser", data_proxy={"worker.example.com": "localhost"}
+    )
+    fn = "/user/testuser/gzfile"
+    with w.open(fn, "wb") as f:
+        gf = GzipFile(fileobj=f, mode="w")
+        gf.write(b"hello")
+        gf.close()
+    with w.open(fn, "rb") as f:
+        gf = GzipFile(fileobj=f, mode="r")
+        assert gf.read() == b"hello"
+
+
+def test_workflow_transaction(hdfs_cluster):
+    w = WebHDFS(
+        hdfs_cluster, user="testuser", data_proxy={"worker.example.com": "localhost"}
+    )
+    fn = "/user/testuser/testrun/afile"
+    w.mkdirs("/user/testuser/testrun")
+    with w.transaction:
+        with w.open(fn, "wb") as f:
+            f.write(b"hello")
+        assert not w.exists(fn)
+    assert w.exists(fn)
+    assert w.ukey(fn)
+    files = w.ls("/user/testuser/testrun", True)
+    summ = w.content_summary("/user/testuser/testrun")
+    assert summ["length"] == files[0]["size"]
+    assert summ["fileCount"] == 1
+
+    w.rm("/user/testuser/testrun", recursive=True)
+    assert not w.exists(fn)
+
+
+def test_webhdfs_cp_file(hdfs_cluster):
+    fs = WebHDFS(
+        hdfs_cluster, user="testuser", data_proxy={"worker.example.com": "localhost"}
+    )
+
+    src, dst = "/user/testuser/testrun/f1", "/user/testuser/testrun/f2"
+
+    fs.mkdir("/user/testuser/testrun")
+
+    with fs.open(src, "wb") as f:
+        f.write(b"hello")
+
+    fs.cp_file(src, dst)
+
+    assert fs.exists(src)
+    assert fs.exists(dst)
+    assert fs.cat(src) == fs.cat(dst)
+
+
+def test_path_with_equals(hdfs_cluster):
+    fs = WebHDFS(
+        hdfs_cluster, user="testuser", data_proxy={"worker.example.com": "localhost"}
+    )
+    path_with_equals = "/user/testuser/some_table/datestamp=2023-11-11"
+
+    fs.mkdir(path_with_equals)
+
+    result = fs.ls(path_with_equals)
+    assert result is not None
+    assert fs.exists(path_with_equals)
+
+
+def test_error_handling_with_equals_in_path(hdfs_cluster):
+    fs = WebHDFS(hdfs_cluster, user="testuser")
+    invalid_path_with_equals = (
+        "/user/testuser/some_table/invalid_path=datestamp=2023-11-11"
+    )
+
+    with pytest.raises(FileNotFoundError):
+        fs.ls(invalid_path_with_equals)
+
+
+def test_create_and_touch_file_with_equals(hdfs_cluster):
+    fs = WebHDFS(
+        hdfs_cluster,
+        user="testuser",
+        data_proxy={"worker.example.com": "localhost"},
+    )
+    base_path = "/user/testuser/some_table/datestamp=2023-11-11"
+    file_path = f"{base_path}/testfile.txt"
+
+    fs.mkdir(base_path)
+    fs.touch(file_path, "wb")
+    assert fs.exists(file_path)
+
+
+def test_write_read_verify_file_with_equals(hdfs_cluster):
+    fs = WebHDFS(
+        hdfs_cluster,
+        user="testuser",
+        data_proxy={"worker.example.com": "localhost"},
+    )
+    base_path = "/user/testuser/some_table/datestamp=2023-11-11"
+    file_path = f"{base_path}/testfile.txt"
+    content = b"This is some content!"
+
+    fs.mkdir(base_path)
+    with fs.open(file_path, "wb") as f:
+        f.write(content)
+
+    with fs.open(file_path, "rb") as f:
+        assert f.read() == content
+
+    file_info = fs.ls(base_path, detail=True)
+    assert len(file_info) == 1
+    assert file_info[0]["name"] == file_path
+    assert file_info[0]["size"] == len(content)

fsspec/implementations/tests/test_zip.py

@@ -0,0 +1,134 @@
+import collections.abc
+import os.path
+
+import pytest
+
+import fsspec
+from fsspec.implementations.tests.test_archive import archive_data, tempzip
+
+
+def test_info():
+    with tempzip(archive_data) as z:
+        fs = fsspec.filesystem("zip", fo=z)
+
+        # Iterate over all files.
+        for f in archive_data:
+            lhs = fs.info(f)
+
+            # Probe some specific fields of Zip archives.
+            assert "CRC" in lhs
+            assert "compress_size" in lhs
+
+
+def test_fsspec_get_mapper():
+    """Added for #788"""
+
+    with tempzip(archive_data) as z:
+        mapping = fsspec.get_mapper(f"zip::{z}")
+
+        assert isinstance(mapping, collections.abc.Mapping)
+        keys = sorted(mapping.keys())
+        assert keys == ["a", "b", "deeply/nested/path"]
+
+        # mapping.getitems() will call FSMap.fs.cat()
+        # which was not accurately implemented for zip.
+        assert isinstance(mapping, fsspec.mapping.FSMap)
+        items = dict(mapping.getitems(keys))
+        assert items == {"a": b"", "b": b"hello", "deeply/nested/path": b"stuff"}
+
+
+def test_not_cached():
+    with tempzip(archive_data) as z:
+        fs = fsspec.filesystem("zip", fo=z)
+        fs2 = fsspec.filesystem("zip", fo=z)
+        assert fs is not fs2
+
+
+def test_root_info():
+    with tempzip(archive_data) as z:
+        fs = fsspec.filesystem("zip", fo=z)
+        assert fs.info("/") == {"name": "", "type": "directory", "size": 0}
+        assert fs.info("") == {"name": "", "type": "directory", "size": 0}
+
+
+def test_write_seek(m):
+    with m.open("afile.zip", "wb") as f:
+        fs = fsspec.filesystem("zip", fo=f, mode="w")
+        fs.pipe("another", b"hi")
+        fs.zip.close()
+
+    with m.open("afile.zip", "rb") as f:
+        fs = fsspec.filesystem("zip", fo=f)
+        assert fs.cat("another") == b"hi"
+
+
+def test_rw(m):
+    # extra arg to zip means "create archive"
+    with fsspec.open(
+        "zip://afile::memory://out.zip", mode="wb", zip={"mode": "w"}
+    ) as f:
+        f.write(b"data")
+
+    with fsspec.open("zip://afile::memory://out.zip", mode="rb") as f:
+        assert f.read() == b"data"
+
+
+def test_mapper(m):
+    # extra arg to zip means "create archive"
+    mapper = fsspec.get_mapper("zip::memory://out.zip", zip={"mode": "w"})
+    with pytest.raises(KeyError):
+        mapper["a"]
+
+    mapper["a"] = b"data"
+    with pytest.raises(OSError):
+        # fails because this is write mode and we cannot also read
+        mapper["a"]
+    assert "a" in mapper  # but be can list
+
+
+def test_zip_glob_star(m):
+    with fsspec.open(
+        "zip://adir/afile::memory://out.zip", mode="wb", zip={"mode": "w"}
+    ) as f:
+        f.write(b"data")
+
+    fs, _ = fsspec.core.url_to_fs("zip::memory://out.zip")
+    outfiles = fs.glob("*")
+    assert len(outfiles) == 1
+
+    fs = fsspec.filesystem("zip", fo="memory://out.zip", mode="w")
+    fs.mkdir("adir")
+    fs.pipe("adir/afile", b"data")
+    outfiles = fs.glob("*")
+    assert len(outfiles) == 1
+
+    fn = f"{os.path.dirname(os.path.abspath((__file__)))}/out.zip"
+    fs = fsspec.filesystem("zip", fo=fn, mode="r")
+    outfiles = fs.glob("*")
+    assert len(outfiles) == 1
+
+
+def test_append(m, tmpdir):
+    fs = fsspec.filesystem("zip", fo="memory://out.zip", mode="w")
+    with fs.open("afile", "wb") as f:
+        f.write(b"data")
+    fs.close()
+
+    fs = fsspec.filesystem("zip", fo="memory://out.zip", mode="a")
+    with fs.open("bfile", "wb") as f:
+        f.write(b"data")
+    fs.close()
+
+    assert len(fsspec.open_files("zip://*::memory://out.zip")) == 2
+
+    fs = fsspec.filesystem("zip", fo=f"{tmpdir}/out.zip", mode="w")
+    with fs.open("afile", "wb") as f:
+        f.write(b"data")
+    fs.close()
+
+    fs = fsspec.filesystem("zip", fo=f"{tmpdir}/out.zip", mode="a")
+    with fs.open("bfile", "wb") as f:
+        f.write(b"data")
+    fs.close()
+
+    assert len(fsspec.open_files("zip://*::memory://out.zip")) == 2

fsspec/implementations/webhdfs.py

@@ -102,9 +102,7 @@ class WebHDFS(AbstractFileSystem):
         if self._cached:
             return
         super().__init__(**kwargs)
-        self.url = (
-            f"{'https' if use_https else 'http'}://{host}:{port}/webhdfs/v1"  # noqa
-        )
+        self.url = f"{'https' if use_https else 'http'}://{host}:{port}/webhdfs/v1"  # noqa
         self.kerb = kerberos
         self.kerb_kwargs = kerb_kwargs or {}
         self.pars = {}

fsspec/mapping.py

@@ -40,7 +40,7 @@ class FSMap(MutableMapping):
 
     def __init__(self, root, fs, check=False, create=False, missing_exceptions=None):
         self.fs = fs
-        self.root = fs._strip_protocol(root).rstrip("/")
+        self.root = fs._strip_protocol(root)
         self._root_key_to_str = fs._strip_protocol(posixpath.join(root, "x"))[:-1]
         if missing_exceptions is None:
             missing_exceptions = (
@@ -142,7 +142,7 @@ class FSMap(MutableMapping):
             if isinstance(key, list):
                 key = tuple(key)
             key = str(key)
-        return f"{self._root_key_to_str}{key}"
+        return f"{self._root_key_to_str}{key}".rstrip("/")
 
     def _str_to_key(self, s):
         """Strip path of to leave key name"""

fsspec/parquet.py

@@ -162,7 +162,6 @@ def _get_parquet_byte_ranges(
 
     # Pass to specialized function if metadata is defined
     if metadata is not None:
-
         # Use the provided parquet metadata object
         # to avoid transferring/parsing footer metadata
         return _get_parquet_byte_ranges_from_metadata(
@@ -236,7 +235,6 @@ def _get_parquet_byte_ranges(
 
         # Calculate required byte ranges for each path
         for i, path in enumerate(paths):
-
             # Deal with small-file case.
             # Just include all remaining bytes of the file
             # in a single range.
@@ -347,7 +345,6 @@ def _add_header_magic(data):
 
 
 def _set_engine(engine_str):
-
     # Define a list of parquet engines to try
     if engine_str == "auto":
         try_engines = ("fastparquet", "pyarrow")
@@ -383,7 +380,6 @@ def _set_engine(engine_str):
 
 
 class FastparquetEngine:
-
     # The purpose of the FastparquetEngine class is
     # to check if fastparquet can be imported (on initialization)
     # and to define a `_parquet_byte_ranges` method. In the
@@ -406,7 +402,6 @@ class FastparquetEngine:
         footer=None,
         footer_start=None,
     ):
-
         # Initialize offset ranges and define ParqetFile metadata
         pf = metadata
         data_paths, data_starts, data_ends = [], [], []
@@ -440,7 +435,6 @@ class FastparquetEngine:
             # Skip this row-group if we are targeting
             # specific row-groups
             if row_group_indices is None or r in row_group_indices:
-
                 # Find the target parquet-file path for `row_group`
                 fn = self._row_group_filename(row_group, pf)
 
@@ -471,7 +465,6 @@ class FastparquetEngine:
 
 
 class PyarrowEngine:
-
     # The purpose of the PyarrowEngine class is
     # to check if pyarrow can be imported (on initialization)
     # and to define a `_parquet_byte_ranges` method. In the
@@ -494,7 +487,6 @@ class PyarrowEngine:
         footer=None,
         footer_start=None,
     ):
-
         if metadata is not None:
             raise ValueError("metadata input not supported for PyarrowEngine")
 

fsspec/registry.py

@@ -214,6 +214,10 @@ known_implementations = {
     "zip": {"class": "fsspec.implementations.zip.ZipFileSystem"},
 }
 
+assert list(known_implementations) == sorted(
+    known_implementations
+), "Not in alphabetical order"
+
 
 def get_filesystem_class(protocol):
     """Fetch named protocol implementation from the registry

fsspec/spec.py

@@ -358,8 +358,10 @@ class AbstractFileSystem(metaclass=_Cached):
         but contains nothing), None if not in cache.
         """
         parent = self._parent(path)
-        if path.rstrip("/") in self.dircache:
+        try:
             return self.dircache[path.rstrip("/")]
+        except KeyError:
+            pass
         try:
             files = [
                 f
@@ -1175,7 +1177,10 @@ class AbstractFileSystem(metaclass=_Cached):
         if path1 == path2:
             logger.debug("%s mv: The paths are the same, so no files were moved.", self)
         else:
-            self.copy(path1, path2, recursive=recursive, maxdepth=maxdepth)
+            # explicitly raise exception to prevent data corruption
+            self.copy(
+                path1, path2, recursive=recursive, maxdepth=maxdepth, onerror="raise"
+            )
             self.rm(path1, recursive=recursive)
 
     def rm_file(self, path):
@@ -1693,7 +1698,12 @@ class AbstractBufferedFile(io.IOBase):
         """Files are equal if they have the same checksum, only in read mode"""
         if self is other:
             return True
-        return self.mode == "rb" and other.mode == "rb" and hash(self) == hash(other)
+        return (
+            isinstance(other, type(self))
+            and self.mode == "rb"
+            and other.mode == "rb"
+            and hash(self) == hash(other)
+        )
 
     def commit(self):
         """Move from temp to final destination"""
@@ -1839,11 +1849,18 @@ class AbstractBufferedFile(io.IOBase):
             length = self.size - self.loc
         if self.closed:
             raise ValueError("I/O operation on closed file.")
-        logger.debug("%s read: %i - %i", self, self.loc, self.loc + length)
         if length == 0:
             # don't even bother calling fetch
             return b""
         out = self.cache._fetch(self.loc, self.loc + length)
+
+        logger.debug(
+            "%s read: %i - %i %s",
+            self,
+            self.loc,
+            self.loc + length,
+            self.cache._log_stats(),
+        )
         self.loc += len(out)
         return out
 

fsspec/tests/abstract/mv.py

@@ -0,0 +1,57 @@
+import os
+
+import pytest
+
+import fsspec
+
+
+def test_move_raises_error_with_tmpdir(tmpdir):
+    # Create a file in the temporary directory
+    source = tmpdir.join("source_file.txt")
+    source.write("content")
+
+    # Define a destination that simulates a protected or invalid path
+    destination = tmpdir.join("non_existent_directory/destination_file.txt")
+
+    # Instantiate the filesystem (assuming the local file system interface)
+    fs = fsspec.filesystem("file")
+
+    # Use the actual file paths as string
+    with pytest.raises(FileNotFoundError):
+        fs.mv(str(source), str(destination))
+
+
+@pytest.mark.parametrize("recursive", (True, False))
+def test_move_raises_error_with_tmpdir_permission(recursive, tmpdir):
+    # Create a file in the temporary directory
+    source = tmpdir.join("source_file.txt")
+    source.write("content")
+
+    # Create a protected directory (non-writable)
+    protected_dir = tmpdir.mkdir("protected_directory")
+    protected_path = str(protected_dir)
+
+    # Set the directory to read-only
+    if os.name == "nt":
+        os.system(f'icacls "{protected_path}" /deny Everyone:(W)')
+    else:
+        os.chmod(protected_path, 0o555)  # Sets the directory to read-only
+
+    # Define a destination inside the protected directory
+    destination = protected_dir.join("destination_file.txt")
+
+    # Instantiate the filesystem (assuming the local file system interface)
+    fs = fsspec.filesystem("file")
+
+    # Try to move the file to the read-only directory, expecting a permission error
+    with pytest.raises(PermissionError):
+        fs.mv(str(source), str(destination), recursive=recursive)
+
+    # Assert the file was not created in the destination
+    assert not os.path.exists(destination)
+
+    # Cleanup: Restore permissions so the directory can be cleaned up
+    if os.name == "nt":
+        os.system(f'icacls "{protected_path}" /remove:d Everyone')
+    else:
+        os.chmod(protected_path, 0o755)  # Restore write permission for cleanup