fsspec 2024.5.0__py3-none-any.whl → 2024.6.0__py3-none-any.whl

fsspec/tests/test_fuse.py

@@ -1,147 +0,0 @@
-import os
-import subprocess
-import time
-from multiprocessing import Process
-
-import pytest
-
-try:
-    pytest.importorskip("fuse")  # noqa: E402
-except OSError:
-    # can succeed in importing fuse, but fail to load so
-    pytest.importorskip("nonexistent")  # noqa: E402
-
-from fsspec.fuse import main, run
-from fsspec.implementations.memory import MemoryFileSystem
-
-
-def host_fuse(mountdir):
-    fs = MemoryFileSystem()
-    fs.touch("/mounted/testfile")
-    run(fs, "/mounted/", mountdir)
-
-
-def test_basic(tmpdir, capfd):
-    mountdir = str(tmpdir.mkdir("mount"))
-
-    fuse_process = Process(target=host_fuse, args=(str(mountdir),))
-    fuse_process.start()
-
-    try:
-        timeout = 10
-        while True:
-            try:
-                # can fail with device not ready while waiting for fuse
-                if "testfile" in os.listdir(mountdir):
-                    break
-            except Exception:
-                pass
-            timeout -= 1
-            time.sleep(1)
-            if not timeout > 0:
-                import pdb
-
-                pdb.set_trace()
-                pytest.skip(msg="fuse didn't come live")
-
-        fn = os.path.join(mountdir, "test")
-        with open(fn, "wb") as f:
-            f.write(b"data")
-
-        with open(fn) as f:
-            assert f.read() == "data"
-
-        os.remove(fn)
-
-        os.mkdir(fn)
-        assert os.listdir(fn) == []
-
-        os.mkdir(fn + "/inner")
-
-        with pytest.raises(OSError):
-            os.rmdir(fn)
-
-        captured = capfd.readouterr()
-        assert "Traceback" not in captured.out
-        assert "Traceback" not in captured.err
-
-        os.rmdir(fn + "/inner")
-        os.rmdir(fn)
-    finally:
-        fuse_process.terminate()
-        fuse_process.join(timeout=10)
-        if fuse_process.is_alive():
-            fuse_process.kill()
-            fuse_process.join()
-
-
-def host_mount_local(source_dir, mount_dir, debug_log):
-    main(["local", source_dir, mount_dir, "-l", debug_log, "--ready-file"])
-
-
-@pytest.fixture()
-def mount_local(tmpdir):
-    source_dir = tmpdir.mkdir("source")
-    mount_dir = tmpdir.mkdir("local")
-    debug_log = tmpdir / "debug.log"
-    fuse_process = Process(
-        target=host_mount_local, args=(str(source_dir), str(mount_dir), str(debug_log))
-    )
-    fuse_process.start()
-    ready_file = mount_dir / ".fuse_ready"
-    for _ in range(20):
-        if ready_file.exists() and open(ready_file).read() == b"ready":
-            break
-        time.sleep(0.1)
-    try:
-        yield (source_dir, mount_dir)
-    finally:
-        fuse_process.terminate()
-        fuse_process.join(timeout=10)
-        if fuse_process.is_alive():
-            fuse_process.kill()
-            fuse_process.join()
-
-
-def test_mount(mount_local):
-    source_dir, mount_dir = mount_local
-    assert os.listdir(mount_dir) == []
-    assert os.listdir(source_dir) == []
-
-    mount_dir.mkdir("a")
-
-    assert os.listdir(mount_dir) == ["a"]
-    assert os.listdir(source_dir) == ["a"]
-
-
-def test_chmod(mount_local):
-    source_dir, mount_dir = mount_local
-    open(mount_dir / "text", "w").write("test")
-    assert os.listdir(source_dir) == ["text"]
-
-    cp = subprocess.run(
-        ["cp", str(mount_dir / "text"), str(mount_dir / "new")],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        check=False,
-    )
-
-    assert cp.stderr == b""
-    assert cp.stdout == b""
-    assert set(os.listdir(source_dir)) == {"text", "new"}
-    assert open(mount_dir / "new").read() == "test"
-
-
-def test_seek_rw(mount_local):
-    source_dir, mount_dir = mount_local
-    fh = open(mount_dir / "text", "w")
-    fh.write("teST")
-    fh.seek(2)
-    fh.write("st")
-    fh.close()
-
-    fh = open(mount_dir / "text", "r")
-    assert fh.read() == "test"
-    fh.seek(2)
-    assert fh.read() == "st"
-    fh.close()

fsspec/tests/test_generic.py

@@ -1,90 +0,0 @@
-import pytest
-
-import fsspec
-from fsspec.tests.conftest import data, server  # noqa: F401
-
-
-def test_remote_async_ops(server):
-    fsspec.filesystem("http", headers={"give_length": "true", "head_ok": "true"})
-    fs = fsspec.filesystem("generic", default_method="current")
-    out = fs.info(server + "/index/realfile")
-    assert out["size"] == len(data)
-    assert out["type"] == "file"
-    assert fs.isfile(server + "/index/realfile")  # this method from superclass
-
-
-def test_touch_rm(m):
-    m.touch("afile")
-    m.touch("dir/afile")
-
-    fs = fsspec.filesystem("generic", default_method="current")
-    fs.rm("memory://afile")
-    assert not m.exists("afile")
-
-    fs.rm("memory://dir", recursive=True)
-    assert not m.exists("dir/afile")
-    assert not m.exists("dir")
-
-
-def test_cp_async_to_sync(server, m):
-    fsspec.filesystem("http", headers={"give_length": "true", "head_ok": "true"})
-    fs = fsspec.filesystem("generic", default_method="current")
-    fs.cp([server + "/index/realfile"], ["memory://realfile"])
-    assert m.cat("realfile") == data
-
-    fs.rm("memory://realfile")
-    assert not m.exists("realfile")
-
-
-def test_pipe_cat_sync(m):
-    fs = fsspec.filesystem("generic", default_method="current")
-    fs.pipe("memory://afile", b"data")
-    assert fs.cat("memory://afile") == b"data"
-
-
-def test_cat_async(server):
-    fsspec.filesystem("http", headers={"give_length": "true", "head_ok": "true"})
-    fs = fsspec.filesystem("generic", default_method="current")
-    assert fs.cat(server + "/index/realfile") == data
-
-
-def test_rsync(tmpdir, m):
-    from fsspec.generic import GenericFileSystem, rsync
-
-    fs = GenericFileSystem()
-    fs.pipe("memory:///deep/path/afile", b"data1")
-    fs.pipe("memory:///deep/afile", b"data2")
-
-    with pytest.raises(ValueError):
-        rsync("memory:///deep/afile", f"file://{tmpdir}")
-    rsync("memory://", f"file://{tmpdir}")
-
-    allfiles = fs.find(f"file://{tmpdir}", withdirs=True, detail=True)
-    pos_tmpdir = fsspec.implementations.local.make_path_posix(str(tmpdir))  # for WIN
-    assert set(allfiles) == {
-        f"file://{pos_tmpdir}{_}"
-        for _ in [
-            "",
-            "/deep",
-            "/deep/path",
-            "/deep/path/afile",
-            "/deep/afile",
-        ]
-    }
-    fs.rm("memory:///deep/afile")
-    rsync("memory://", f"file://{tmpdir}", delete_missing=True)
-    allfiles2 = fs.find(f"file://{tmpdir}", withdirs=True, detail=True)
-    assert set(allfiles2) == {
-        f"file://{pos_tmpdir}{_}"
-        for _ in [
-            "",
-            "/deep",
-            "/deep/path",
-            "/deep/path/afile",
-        ]
-    }
-    # the file was not updated, since size was correct
-    assert (
-        allfiles[f"file://{pos_tmpdir}/deep/path/afile"]
-        == allfiles2[f"file://{pos_tmpdir}/deep/path/afile"]
-    )

fsspec/tests/test_gui.py

@@ -1,23 +0,0 @@
-import pytest
-
-panel = pytest.importorskip("panel")
-
-
-def test_basic():
-    import fsspec.gui
-
-    gui = fsspec.gui.FileSelector()
-    assert "url" in str(gui.panel)
-
-
-def test_kwargs(tmpdir):
-    """confirm kwargs are passed to the filesystem instance"""
-    import fsspec.gui
-
-    gui = fsspec.gui.FileSelector(f"file://{tmpdir}", kwargs="{'auto_mkdir': True}")
-
-    assert gui.fs.auto_mkdir
-
-    gui = fsspec.gui.FileSelector(f"file://{tmpdir}", kwargs={"auto_mkdir": True})
-
-    assert gui.fs.auto_mkdir

fsspec/tests/test_mapping.py

@@ -1,228 +0,0 @@
-import os
-import pickle
-import platform
-import sys
-import uuid
-
-import pytest
-
-import fsspec
-from fsspec.implementations.local import LocalFileSystem
-from fsspec.implementations.memory import MemoryFileSystem
-
-
-def test_mapping_prefix(tmpdir):
-    tmpdir = str(tmpdir)
-    os.makedirs(os.path.join(tmpdir, "afolder"))
-    open(os.path.join(tmpdir, "afile"), "w").write("test")
-    open(os.path.join(tmpdir, "afolder", "anotherfile"), "w").write("test2")
-
-    m = fsspec.get_mapper(f"file://{tmpdir}")
-    assert "afile" in m
-    assert m["afolder/anotherfile"] == b"test2"
-
-    fs = fsspec.filesystem("file")
-    m2 = fs.get_mapper(tmpdir)
-    m3 = fs.get_mapper(f"file://{tmpdir}")
-
-    assert m == m2 == m3
-
-
-def test_getitems_errors(tmpdir):
-    tmpdir = str(tmpdir)
-    os.makedirs(os.path.join(tmpdir, "afolder"))
-    open(os.path.join(tmpdir, "afile"), "w").write("test")
-    open(os.path.join(tmpdir, "afolder", "anotherfile"), "w").write("test2")
-    m = fsspec.get_mapper(f"file://{tmpdir}")
-    assert m.getitems(["afile", "bfile"], on_error="omit") == {"afile": b"test"}
-    with pytest.raises(KeyError):
-        m.getitems(["afile", "bfile"])
-    out = m.getitems(["afile", "bfile"], on_error="return")
-    assert isinstance(out["bfile"], KeyError)
-    m = fsspec.get_mapper(f"file://{tmpdir}", missing_exceptions=())
-    assert m.getitems(["afile", "bfile"], on_error="omit") == {"afile": b"test"}
-    with pytest.raises(FileNotFoundError):
-        m.getitems(["afile", "bfile"])
-
-
-def test_ops():
-    MemoryFileSystem.store.clear()
-    m = fsspec.get_mapper("memory://")
-    assert not m
-    assert list(m) == []
-
-    with pytest.raises(KeyError):
-        m["hi"]
-
-    assert m.pop("key", 0) == 0
-
-    m["key0"] = b"data"
-    assert list(m) == ["key0"]
-    assert m["key0"] == b"data"
-
-    m.clear()
-
-    assert list(m) == []
-
-
-def test_pickle():
-    m = fsspec.get_mapper("memory://")
-    assert isinstance(m.fs, MemoryFileSystem)
-    m["key"] = b"data"
-    m2 = pickle.loads(pickle.dumps(m))
-    assert list(m) == list(m2)
-    assert m.missing_exceptions == m2.missing_exceptions
-
-
-def test_keys_view():
-    # https://github.com/fsspec/filesystem_spec/issues/186
-    m = fsspec.get_mapper("memory://")
-    m["key"] = b"data"
-
-    keys = m.keys()
-    assert len(keys) == 1
-    # check that we don't consume the keys
-    assert len(keys) == 1
-    m.clear()
-
-
-def test_multi():
-    m = fsspec.get_mapper("memory:///")
-    data = {"a": b"data1", "b": b"data2"}
-    m.setitems(data)
-
-    assert m.getitems(list(data)) == data
-    m.delitems(list(data))
-    assert not list(m)
-
-
-def test_setitem_types():
-    import array
-
-    m = fsspec.get_mapper("memory://")
-    m["a"] = array.array("i", [1])
-    if sys.byteorder == "little":
-        assert m["a"] == b"\x01\x00\x00\x00"
-    else:
-        assert m["a"] == b"\x00\x00\x00\x01"
-    m["b"] = bytearray(b"123")
-    assert m["b"] == b"123"
-    m.setitems({"c": array.array("i", [1]), "d": bytearray(b"123")})
-    if sys.byteorder == "little":
-        assert m["c"] == b"\x01\x00\x00\x00"
-    else:
-        assert m["c"] == b"\x00\x00\x00\x01"
-    assert m["d"] == b"123"
-
-
-def test_setitem_numpy():
-    m = fsspec.get_mapper("memory://")
-    np = pytest.importorskip("numpy")
-    m["c"] = np.array(1, dtype="<i4")  # scalar
-    assert m["c"] == b"\x01\x00\x00\x00"
-    m["c"] = np.array([1, 2], dtype="<i4")  # array
-    assert m["c"] == b"\x01\x00\x00\x00\x02\x00\x00\x00"
-    m["c"] = np.array(
-        np.datetime64("2000-01-01T23:59:59.999999999"), dtype="<M8[ns]"
-    )  # datetime64 scalar
-    assert m["c"] == b"\xff\xff\x91\xe3c\x9b#\r"
-    m["c"] = np.array(
-        [
-            np.datetime64("1900-01-01T23:59:59.999999999"),
-            np.datetime64("2000-01-01T23:59:59.999999999"),
-        ],
-        dtype="<M8[ns]",
-    )  # datetime64 array
-    assert m["c"] == b"\xff\xff}p\xf8fX\xe1\xff\xff\x91\xe3c\x9b#\r"
-    m["c"] = np.array(
-        np.timedelta64(3155673612345678901, "ns"), dtype="<m8[ns]"
-    )  # timedelta64 scalar
-    assert m["c"] == b"5\x1c\xf0Rn4\xcb+"
-    m["c"] = np.array(
-        [
-            np.timedelta64(450810516049382700, "ns"),
-            np.timedelta64(3155673612345678901, "ns"),
-        ],
-        dtype="<m8[ns]",
-    )  # timedelta64 scalar
-    assert m["c"] == b',M"\x9e\xc6\x99A\x065\x1c\xf0Rn4\xcb+'
-
-
-def test_empty_url():
-    m = fsspec.get_mapper()
-    assert isinstance(m.fs, LocalFileSystem)
-
-
-def test_fsmap_access_with_root_prefix(tmp_path):
-    # "/a" and "a" are the same for LocalFileSystem
-    tmp_path.joinpath("a").write_bytes(b"data")
-    m = fsspec.get_mapper(f"file://{tmp_path}")
-    assert m["/a"] == m["a"] == b"data"
-
-    # "/a" and "a" differ for MemoryFileSystem
-    m = fsspec.get_mapper(f"memory://{uuid.uuid4()}")
-    m["/a"] = b"data"
-
-    assert m["/a"] == b"data"
-    with pytest.raises(KeyError):
-        _ = m["a"]
-
-
-@pytest.mark.parametrize(
-    "key",
-    [
-        pytest.param(b"k", id="bytes"),
-        pytest.param(1234, id="int"),
-        pytest.param((1,), id="tuple"),
-        pytest.param([""], id="list"),
-    ],
-)
-def test_fsmap_non_str_keys(key):
-    m = fsspec.get_mapper()
-
-    # Once the deprecation period passes
-    # FSMap.__getitem__ should raise TypeError for non-str keys
-    #   with pytest.raises(TypeError):
-    #       _ = m[key]
-
-    with pytest.warns(DeprecationWarning):
-        with pytest.raises(KeyError):
-            _ = m[key]
-
-
-def test_fsmap_error_on_protocol_keys():
-    root = uuid.uuid4()
-    m = fsspec.get_mapper(f"memory://{root}", create=True)
-    m["a"] = b"data"
-
-    assert m["a"] == b"data"
-    with pytest.raises(KeyError):
-        _ = m[f"memory://{root}/a"]
-
-
-def test_fsmap_access_with_suffix(tmp_path):
-    tmp_path.joinpath("b").mkdir()
-    tmp_path.joinpath("b", "a").write_bytes(b"data")
-    if platform.system() == "Windows":
-        # on Windows opening a directory will raise PermissionError
-        # see: https://bugs.python.org/issue43095
-        missing_exceptions = (
-            FileNotFoundError,
-            IsADirectoryError,
-            NotADirectoryError,
-            PermissionError,
-        )
-    else:
-        missing_exceptions = None
-    m = fsspec.get_mapper(f"file://{tmp_path}", missing_exceptions=missing_exceptions)
-    with pytest.raises(KeyError):
-        _ = m["b/"]
-    assert m["b/a/"] == b"data"
-
-
-def test_fsmap_dirfs():
-    m = fsspec.get_mapper("memory://")
-
-    fs = m.dirfs
-    assert isinstance(fs, fsspec.implementations.dirfs.DirFileSystem)
-    assert fs.path == m.root

fsspec/tests/test_parquet.py

@@ -1,140 +0,0 @@
-import os
-
-import pytest
-
-try:
-    import fastparquet
-except ImportError:
-    fastparquet = None
-try:
-    import pyarrow.parquet as pq
-except ImportError:
-    pq = None
-
-from fsspec.core import url_to_fs
-from fsspec.parquet import _get_parquet_byte_ranges, open_parquet_file
-
-# Define `engine` fixture
-FASTPARQUET_MARK = pytest.mark.skipif(not fastparquet, reason="fastparquet not found")
-PYARROW_MARK = pytest.mark.skipif(not pq, reason="pyarrow not found")
-ANY_ENGINE_MARK = pytest.mark.skipif(
-    not (fastparquet or pq),
-    reason="No parquet engine (fastparquet or pyarrow) found",
-)
-
-
-@pytest.fixture(
-    params=[
-        pytest.param("fastparquet", marks=FASTPARQUET_MARK),
-        pytest.param("pyarrow", marks=PYARROW_MARK),
-        pytest.param("auto", marks=ANY_ENGINE_MARK),
-    ]
-)
-def engine(request):
-    return request.param
-
-
-@pytest.mark.parametrize("columns", [None, ["x"], ["x", "y"], ["z"]])
-@pytest.mark.parametrize("max_gap", [0, 64])
-@pytest.mark.parametrize("max_block", [64, 256_000_000])
-@pytest.mark.parametrize("footer_sample_size", [8, 1_000])
-@pytest.mark.parametrize("range_index", [True, False])
-def test_open_parquet_file(
-    tmpdir, engine, columns, max_gap, max_block, footer_sample_size, range_index
-):
-    # Pandas required for this test
-    pd = pytest.importorskip("pandas")
-
-    # Write out a simple DataFrame
-    path = os.path.join(str(tmpdir), "test.parquet")
-    nrows = 40
-    df = pd.DataFrame(
-        {
-            "x": [i * 7 % 5 for i in range(nrows)],
-            "y": [[0, i] for i in range(nrows)],  # list
-            "z": [{"a": i, "b": "cat"} for i in range(nrows)],  # struct
-        },
-        index=pd.Index([10 * i for i in range(nrows)], name="myindex"),
-    )
-    if range_index:
-        df = df.reset_index(drop=True)
-        df.index.name = "myindex"
-    df.to_parquet(path)
-
-    # "Traditional read" (without `open_parquet_file`)
-    expect = pd.read_parquet(path, columns=columns)
-
-    # Use `_get_parquet_byte_ranges` to re-write a
-    # place-holder file with all bytes NOT required
-    # to read `columns` set to b"0". The purpose of
-    # this step is to make sure the read will fail
-    # if the correct bytes have not been accurately
-    # selected by `_get_parquet_byte_ranges`. If this
-    # test were reading from remote storage, we would
-    # not need this logic to capture errors.
-    fs = url_to_fs(path)[0]
-    data = _get_parquet_byte_ranges(
-        [path],
-        fs,
-        columns=columns,
-        engine=engine,
-        max_gap=max_gap,
-        max_block=max_block,
-        footer_sample_size=footer_sample_size,
-    )[path]
-    file_size = fs.size(path)
-    with open(path, "wb") as f:
-        f.write(b"0" * file_size)
-
-        if footer_sample_size == 8:
-            # We know 8 bytes is too small to include
-            # the footer metadata, so there should NOT
-            # be a key for the last 8 bytes of the file
-            bad_key = (file_size - 8, file_size)
-            assert bad_key not in data.keys()
-
-        for (start, stop), byte_data in data.items():
-            f.seek(start)
-            f.write(byte_data)
-
-    # Read back the modified file with `open_parquet_file`
-    with open_parquet_file(
-        path,
-        columns=columns,
-        engine=engine,
-        max_gap=max_gap,
-        max_block=max_block,
-        footer_sample_size=footer_sample_size,
-    ) as f:
-        result = pd.read_parquet(f, columns=columns)
-
-    # Check that `result` matches `expect`
-    pd.testing.assert_frame_equal(expect, result)
-
-    # Try passing metadata
-    if engine == "fastparquet":
-        # Should work fine for "fastparquet"
-        pf = fastparquet.ParquetFile(path)
-        with open_parquet_file(
-            path,
-            metadata=pf,
-            columns=columns,
-            engine=engine,
-            max_gap=max_gap,
-            max_block=max_block,
-            footer_sample_size=footer_sample_size,
-        ) as f:
-            result = pd.read_parquet(f, columns=columns)
-        pd.testing.assert_frame_equal(expect, result)
-    elif engine == "pyarrow":
-        # Should raise ValueError for "pyarrow"
-        with pytest.raises(ValueError):
-            open_parquet_file(
-                path,
-                metadata=["Not-None"],
-                columns=columns,
-                engine=engine,
-                max_gap=max_gap,
-                max_block=max_block,
-                footer_sample_size=footer_sample_size,
-            )