fsspec 2024.5.0__py3-none-any.whl → 2024.6.1__py3-none-any.whl

fsspec/implementations/tests/memory/memory_fixtures.py

@@ -1,27 +0,0 @@
-import pytest
-
-from fsspec import filesystem
-from fsspec.tests.abstract import AbstractFixtures
-
-
-class MemoryFixtures(AbstractFixtures):
-    @pytest.fixture(scope="class")
-    def fs(self):
-        m = filesystem("memory")
-        m.store.clear()
-        m.pseudo_dirs.clear()
-        m.pseudo_dirs.append("")
-        try:
-            yield m
-        finally:
-            m.store.clear()
-            m.pseudo_dirs.clear()
-            m.pseudo_dirs.append("")
-
-    @pytest.fixture
-    def fs_join(self):
-        return lambda *args: "/".join(args)
-
-    @pytest.fixture
-    def fs_path(self):
-        return ""

fsspec/implementations/tests/memory/memory_test.py

@@ -1,14 +0,0 @@
-import fsspec.tests.abstract as abstract
-from fsspec.implementations.tests.memory.memory_fixtures import MemoryFixtures
-
-
-class TestMemoryCopy(abstract.AbstractCopyTests, MemoryFixtures):
-    pass
-
-
-class TestMemoryGet(abstract.AbstractGetTests, MemoryFixtures):
-    pass
-
-
-class TestMemoryPut(abstract.AbstractPutTests, MemoryFixtures):
-    pass

fsspec/implementations/tests/test_archive.py

@@ -1,382 +0,0 @@
-import bz2
-import gzip
-import lzma
-import os
-import pickle
-import tarfile
-import tempfile
-import zipfile
-from contextlib import contextmanager
-from io import BytesIO
-
-import pytest
-
-import fsspec
-
-# The blueprint to create synthesized archive files from.
-archive_data = {"a": b"", "b": b"hello", "deeply/nested/path": b"stuff"}
-
-
-@contextmanager
-def tempzip(data=None):
-    """
-    Provide test cases with temporary synthesized Zip archives.
-    """
-    data = data or {}
-    f = tempfile.mkstemp(suffix=".zip")[1]
-    with zipfile.ZipFile(f, mode="w") as z:
-        for k, v in data.items():
-            z.writestr(k, v)
-    try:
-        yield f
-    finally:
-        try:
-            os.remove(f)
-        except OSError:
-            pass
-
-
-@contextmanager
-def temparchive(data=None):
-    """
-    Provide test cases with temporary synthesized 7-Zip archives.
-    """
-    data = data or {}
-    libarchive = pytest.importorskip("libarchive")
-    f = tempfile.mkstemp(suffix=".7z")[1]
-    with libarchive.file_writer(f, "7zip") as archive:
-        for k, v in data.items():
-            archive.add_file_from_memory(entry_path=k, entry_size=len(v), entry_data=v)
-    try:
-        yield f
-    finally:
-        try:
-            os.remove(f)
-        except OSError:
-            pass
-
-
-@contextmanager
-def temptar(data=None, mode="w", suffix=".tar"):
-    """
-    Provide test cases with temporary synthesized .tar archives.
-    """
-    data = data or {}
-    fn = tempfile.mkstemp(suffix=suffix)[1]
-    with tarfile.TarFile.open(fn, mode=mode) as t:
-        touched = {}
-        for name, data in data.items():
-            # Create directory hierarchy.
-            # https://bugs.python.org/issue22208#msg225558
-            if "/" in name and name not in touched:
-                parts = os.path.dirname(name).split("/")
-                for index in range(1, len(parts) + 1):
-                    info = tarfile.TarInfo("/".join(parts[:index]))
-                    info.type = tarfile.DIRTYPE
-                    t.addfile(info)
-                touched[name] = True
-
-            # Add file content.
-            info = tarfile.TarInfo(name=name)
-            info.size = len(data)
-            t.addfile(info, BytesIO(data))
-
-    try:
-        yield fn
-    finally:
-        try:
-            os.remove(fn)
-        except OSError:
-            pass
-
-
-@contextmanager
-def temptargz(data=None, mode="w", suffix=".tar.gz"):
-    """
-    Provide test cases with temporary synthesized .tar.gz archives.
-    """
-
-    with temptar(data=data, mode=mode) as tarname:
-        fn = tempfile.mkstemp(suffix=suffix)[1]
-        with open(tarname, "rb") as tar:
-            cf = gzip.GzipFile(filename=fn, mode=mode)
-            cf.write(tar.read())
-            cf.close()
-
-        try:
-            yield fn
-        finally:
-            try:
-                os.remove(fn)
-            except OSError:
-                pass
-
-
-@contextmanager
-def temptarbz2(data=None, mode="w", suffix=".tar.bz2"):
-    """
-    Provide test cases with temporary synthesized .tar.bz2 archives.
-    """
-
-    with temptar(data=data, mode=mode) as tarname:
-        fn = tempfile.mkstemp(suffix=suffix)[1]
-        with open(tarname, "rb") as tar:
-            cf = bz2.BZ2File(filename=fn, mode=mode)
-            cf.write(tar.read())
-            cf.close()
-
-        try:
-            yield fn
-        finally:
-            try:
-                os.remove(fn)
-            except OSError:
-                pass
-
-
-@contextmanager
-def temptarxz(data=None, mode="w", suffix=".tar.xz"):
-    """
-    Provide test cases with temporary synthesized .tar.xz archives.
-    """
-
-    with temptar(data=data, mode=mode) as tarname:
-        fn = tempfile.mkstemp(suffix=suffix)[1]
-        with open(tarname, "rb") as tar:
-            cf = lzma.open(filename=fn, mode=mode, format=lzma.FORMAT_XZ)
-            cf.write(tar.read())
-            cf.close()
-
-        try:
-            yield fn
-        finally:
-            try:
-                os.remove(fn)
-            except OSError:
-                pass
-
-
-class ArchiveTestScenario:
-    """
-    Describe a test scenario for any type of archive.
-    """
-
-    def __init__(self, protocol=None, provider=None, variant=None):
-        # The filesystem protocol identifier. Any of "zip", "tar" or "libarchive".
-        self.protocol = protocol
-        # A contextmanager function to provide temporary synthesized archives.
-        self.provider = provider
-        # The filesystem protocol variant identifier. Any of "gz", "bz2" or "xz".
-        self.variant = variant
-
-
-def pytest_generate_tests(metafunc):
-    """
-    Generate test scenario parametrization arguments with appropriate labels (idlist).
-
-    On the one hand, this yields an appropriate output like::
-
-        fsspec/implementations/tests/test_archive.py::TestArchive::test_empty[zip] PASSED  # noqa
-
-    On the other hand, it will support perfect test discovery, like::
-
-        pytest fsspec -vvv -k "zip or tar or libarchive"
-
-    https://docs.pytest.org/en/latest/example/parametrize.html#a-quick-port-of-testscenarios
-    """
-    idlist = []
-    argnames = ["scenario"]
-    argvalues = []
-    for scenario in metafunc.cls.scenarios:
-        scenario: ArchiveTestScenario = scenario
-        label = scenario.protocol
-        if scenario.variant:
-            label += "-" + scenario.variant
-        idlist.append(label)
-        argvalues.append([scenario])
-    metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class")
-
-
-# Define test scenarios.
-scenario_zip = ArchiveTestScenario(protocol="zip", provider=tempzip)
-scenario_tar = ArchiveTestScenario(protocol="tar", provider=temptar)
-scenario_targz = ArchiveTestScenario(protocol="tar", provider=temptargz, variant="gz")
-scenario_tarbz2 = ArchiveTestScenario(
-    protocol="tar", provider=temptarbz2, variant="bz2"
-)
-scenario_tarxz = ArchiveTestScenario(protocol="tar", provider=temptarxz, variant="xz")
-scenario_libarchive = ArchiveTestScenario(protocol="libarchive", provider=temparchive)
-
-
-class TestAnyArchive:
-    """
-    Validate that all filesystem adapter implementations for archive files
-    will adhere to the same specification.
-    """
-
-    scenarios = [
-        scenario_zip,
-        scenario_tar,
-        scenario_targz,
-        scenario_tarbz2,
-        scenario_tarxz,
-        scenario_libarchive,
-    ]
-
-    def test_repr(self, scenario: ArchiveTestScenario):
-        with scenario.provider() as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-            assert repr(fs).startswith("<Archive-like object")
-
-    def test_empty(self, scenario: ArchiveTestScenario):
-        with scenario.provider() as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-            assert fs.find("") == []
-            assert fs.find("", withdirs=True) == []
-            with pytest.raises(FileNotFoundError):
-                fs.info("")
-            assert fs.ls("") == []
-
-    def test_glob(self, scenario: ArchiveTestScenario):
-        with scenario.provider(archive_data) as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-            assert fs.glob("*/*/*th") == ["deeply/nested/path"]
-
-    def test_mapping(self, scenario: ArchiveTestScenario):
-        with scenario.provider(archive_data) as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-            m = fs.get_mapper()
-            assert list(m) == ["a", "b", "deeply/nested/path"]
-            assert m["b"] == archive_data["b"]
-
-    def test_pickle(self, scenario: ArchiveTestScenario):
-        with scenario.provider(archive_data) as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-            fs2 = pickle.loads(pickle.dumps(fs))
-            assert fs2.cat("b") == b"hello"
-
-    def test_all_dirnames(self, scenario: ArchiveTestScenario):
-        with scenario.provider(archive_data) as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-
-            # fx are files, dx are a directories
-            assert fs._all_dirnames([]) == set()
-            assert fs._all_dirnames(["f1"]) == set()
-            assert fs._all_dirnames(["f1", "f2"]) == set()
-            assert fs._all_dirnames(["f1", "f2", "d1/f1"]) == {"d1"}
-            assert fs._all_dirnames(["f1", "d1/f1", "d1/f2"]) == {"d1"}
-            assert fs._all_dirnames(["f1", "d1/f1", "d2/f1"]) == {"d1", "d2"}
-            assert fs._all_dirnames(["d1/d1/d1/f1"]) == {"d1", "d1/d1", "d1/d1/d1"}
-
-    def test_ls(self, scenario: ArchiveTestScenario):
-        with scenario.provider(archive_data) as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-
-            assert fs.ls("", detail=False) == ["a", "b", "deeply"]
-            assert fs.ls("/") == fs.ls("")
-
-            assert fs.ls("deeply", detail=False) == ["deeply/nested"]
-            assert fs.ls("deeply/") == fs.ls("deeply")
-
-            assert fs.ls("deeply/nested", detail=False) == ["deeply/nested/path"]
-            assert fs.ls("deeply/nested/") == fs.ls("deeply/nested")
-
-    def test_find(self, scenario: ArchiveTestScenario):
-        with scenario.provider(archive_data) as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-
-            assert fs.find("") == ["a", "b", "deeply/nested/path"]
-            assert fs.find("", withdirs=True) == [
-                "a",
-                "b",
-                "deeply",
-                "deeply/nested",
-                "deeply/nested/path",
-            ]
-
-            assert fs.find("deeply") == ["deeply/nested/path"]
-            assert fs.find("deeply/") == fs.find("deeply")
-
-    @pytest.mark.parametrize("topdown", [True, False])
-    @pytest.mark.parametrize("prune_nested", [True, False])
-    def test_walk(self, scenario: ArchiveTestScenario, topdown, prune_nested):
-        with scenario.provider(archive_data) as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-            expected = [
-                # (dirname, list of subdirs, list of files)
-                ("", ["deeply"], ["a", "b"]),
-                ("deeply", ["nested"], []),
-            ]
-            if not topdown or not prune_nested:
-                expected.append(("deeply/nested", [], ["path"]))
-            if not topdown:
-                expected.reverse()
-
-            result = []
-            for path, dirs, files in fs.walk("", topdown=topdown):
-                result.append((path, dirs.copy(), files))
-                # Bypass the "nested" dir
-                if prune_nested and "nested" in dirs:
-                    dirs.remove("nested")
-
-            # prior py3.10 zip() does not support strict=True, we need
-            # a manual len check here
-            assert len(result) == len(expected)
-            for lhs, rhs in zip(result, expected):
-                assert lhs[0] == rhs[0]
-                assert sorted(lhs[1]) == sorted(rhs[1])
-                assert sorted(lhs[2]) == sorted(rhs[2])
-
-    def test_info(self, scenario: ArchiveTestScenario):
-        # https://github.com/Suor/funcy/blob/1.15/funcy/colls.py#L243-L245
-        def project(mapping, keys):
-            """Leaves only given keys in mapping."""
-            return {k: mapping[k] for k in keys if k in mapping}
-
-        with scenario.provider(archive_data) as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-
-            with pytest.raises(FileNotFoundError):
-                fs.info("i-do-not-exist")
-
-            # Iterate over all directories.
-            for d in fs._all_dirnames(archive_data.keys()):
-                lhs = project(fs.info(d), ["name", "size", "type"])
-                expected = {"name": f"{d}", "size": 0, "type": "directory"}
-                assert lhs == expected
-
-            # Iterate over all files.
-            for f, v in archive_data.items():
-                lhs = fs.info(f)
-                assert lhs["name"] == f
-                assert lhs["size"] == len(v)
-                assert lhs["type"] == "file"
-
-    @pytest.mark.parametrize("scale", [128, 512, 4096])
-    def test_isdir_isfile(self, scenario: ArchiveTestScenario, scale: int):
-        def make_nested_dir(i):
-            x = f"{i}"
-            table = x.maketrans("0123456789", "ABCDEFGHIJ")
-            return "/".join(x.translate(table))
-
-        scaled_data = {f"{make_nested_dir(i)}/{i}": b"" for i in range(1, scale + 1)}
-        with scenario.provider(scaled_data) as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-
-            lhs_dirs, lhs_files = (
-                fs._all_dirnames(scaled_data.keys()),
-                scaled_data.keys(),
-            )
-
-            # Warm-up the Cache, this is done in both cases anyways...
-            fs._get_dirs()
-
-            entries = lhs_files | lhs_dirs
-
-            assert lhs_dirs == {e for e in entries if fs.isdir(e)}
-            assert lhs_files == {e for e in entries if fs.isfile(e)}
-
-    def test_read_empty_file(self, scenario: ArchiveTestScenario):
-        with scenario.provider(archive_data) as archive:
-            fs = fsspec.filesystem(scenario.protocol, fo=archive)
-            assert fs.open("a").read() == b""

fsspec/implementations/tests/test_arrow.py

@@ -1,259 +0,0 @@
-import secrets
-
-import pytest
-
-pyarrow_fs = pytest.importorskip("pyarrow.fs")
-FileSystem = pyarrow_fs.FileSystem
-
-from fsspec.implementations.arrow import ArrowFSWrapper, HadoopFileSystem  # noqa
-
-
-@pytest.fixture(scope="function")
-def fs():
-    fs, _ = FileSystem.from_uri("mock://")
-    return ArrowFSWrapper(fs)
-
-
-@pytest.fixture(scope="function", params=[False, True])
-def remote_dir(fs, request):
-    directory = secrets.token_hex(16)
-    fs.makedirs(directory)
-    yield ("hdfs://" if request.param else "/") + directory
-    fs.rm(directory, recursive=True)
-
-
-def test_protocol():
-    fs, _ = FileSystem.from_uri("mock://")
-    fss = ArrowFSWrapper(fs)
-    assert fss.protocol == "mock"
-
-
-def strip_keys(original_entry):
-    entry = original_entry.copy()
-    entry.pop("mtime")
-    return entry
-
-
-def test_strip(fs):
-    assert fs._strip_protocol("/a/file") == "/a/file"
-    assert fs._strip_protocol("hdfs:///a/file") == "/a/file"
-    assert fs._strip_protocol("hdfs://1.1.1.1/a/file") == "/a/file"
-    assert fs._strip_protocol("hdfs://1.1.1.1:8888/a/file") == "/a/file"
-
-
-def test_info(fs, remote_dir):
-    fs.touch(remote_dir + "/a.txt")
-    remote_dir_strip_protocol = fs._strip_protocol(remote_dir)
-    details = fs.info(remote_dir + "/a.txt")
-    assert details["type"] == "file"
-    assert details["name"] == remote_dir_strip_protocol + "/a.txt"
-    assert details["size"] == 0
-
-    fs.mkdir(remote_dir + "/dir")
-    details = fs.info(remote_dir + "/dir")
-    assert details["type"] == "directory"
-    assert details["name"] == remote_dir_strip_protocol + "/dir"
-
-    details = fs.info(remote_dir + "/dir/")
-    assert details["name"] == remote_dir_strip_protocol + "/dir/"
-
-
-def test_move(fs, remote_dir):
-    fs.touch(remote_dir + "/a.txt")
-    initial_info = fs.info(remote_dir + "/a.txt")
-
-    fs.move(remote_dir + "/a.txt", remote_dir + "/b.txt")
-    secondary_info = fs.info(remote_dir + "/b.txt")
-
-    assert not fs.exists(remote_dir + "/a.txt")
-    assert fs.exists(remote_dir + "/b.txt")
-
-    initial_info.pop("name")
-    secondary_info.pop("name")
-    assert initial_info == secondary_info
-
-
-def test_move_recursive(fs, remote_dir):
-    src = remote_dir + "/src"
-    dest = remote_dir + "/dest"
-
-    assert fs.isdir(src) is False
-    fs.mkdir(src)
-    assert fs.isdir(src)
-
-    fs.touch(src + "/a.txt")
-    fs.mkdir(src + "/b")
-    fs.touch(src + "/b/c.txt")
-    fs.move(src, dest, recursive=True)
-
-    assert fs.isdir(src) is False
-    assert not fs.exists(src)
-
-    assert fs.isdir(dest)
-    assert fs.exists(dest)
-    assert fs.cat(dest + "/b/c.txt") == fs.cat(dest + "/a.txt") == b""
-
-
-def test_copy(fs, remote_dir):
-    fs.touch(remote_dir + "/a.txt")
-    initial_info = fs.info(remote_dir + "/a.txt")
-
-    fs.copy(remote_dir + "/a.txt", remote_dir + "/b.txt")
-    secondary_info = fs.info(remote_dir + "/b.txt")
-
-    assert fs.exists(remote_dir + "/a.txt")
-    assert fs.exists(remote_dir + "/b.txt")
-
-    initial_info.pop("name")
-    secondary_info.pop("name")
-    assert strip_keys(initial_info) == strip_keys(secondary_info)
-
-
-def test_rm(fs, remote_dir):
-    fs.touch(remote_dir + "/a.txt")
-    fs.rm(remote_dir + "/a.txt", recursive=True)
-    assert not fs.exists(remote_dir + "/a.txt")
-
-    fs.mkdir(remote_dir + "/dir")
-    fs.rm(remote_dir + "/dir", recursive=True)
-    assert not fs.exists(remote_dir + "/dir")
-
-    fs.mkdir(remote_dir + "/dir")
-    fs.touch(remote_dir + "/dir/a")
-    fs.touch(remote_dir + "/dir/b")
-    fs.mkdir(remote_dir + "/dir/c/")
-    fs.touch(remote_dir + "/dir/c/a")
-    fs.rm(remote_dir + "/dir", recursive=True)
-    assert not fs.exists(remote_dir + "/dir")
-
-
-def test_ls(fs, remote_dir):
-    if remote_dir != "/":
-        remote_dir = remote_dir + "/"
-    remote_dir_strip_protocol = fs._strip_protocol(remote_dir)
-    fs.mkdir(remote_dir + "dir/")
-    files = set()
-    for no in range(8):
-        file = remote_dir + f"dir/test_{no}"
-        # we also want to make sure `fs.touch` works with protocol
-        fs.touch(file)
-        files.add(remote_dir_strip_protocol + f"dir/test_{no}")
-
-    assert set(fs.ls(remote_dir + "dir/")) == files
-
-    dirs = fs.ls(remote_dir + "dir/", detail=True)
-    expected = [fs.info(file) for file in files]
-
-    by_name = lambda details: details["name"]
-    dirs.sort(key=by_name)
-    expected.sort(key=by_name)
-
-    assert dirs == expected
-
-
-def test_mkdir(fs, remote_dir):
-    if remote_dir != "/":
-        remote_dir = remote_dir + "/"
-    fs.mkdir(remote_dir + "dir/")
-    assert fs.isdir(remote_dir + "dir/")
-    assert len(fs.ls(remote_dir + "dir/")) == 0
-
-    fs.mkdir(remote_dir + "dir/sub", create_parents=False)
-    assert fs.isdir(remote_dir + "dir/sub")
-
-
-def test_makedirs(fs, remote_dir):
-    fs.makedirs(remote_dir + "dir/a/b/c/")
-    assert fs.isdir(remote_dir + "dir/a/b/c/")
-    assert fs.isdir(remote_dir + "dir/a/b/")
-    assert fs.isdir(remote_dir + "dir/a/")
-
-    fs.makedirs(remote_dir + "dir/a/b/c/", exist_ok=True)
-
-
-def test_exceptions(fs, remote_dir):
-    with pytest.raises(FileNotFoundError):
-        with fs.open(remote_dir + "/a.txt"):
-            ...
-
-    with pytest.raises(FileNotFoundError):
-        fs.copy(remote_dir + "/u.txt", remote_dir + "/y.txt")
-
-
-def test_open_rw(fs, remote_dir):
-    data = b"dvc.org"
-
-    with fs.open(remote_dir + "/a.txt", "wb") as stream:
-        stream.write(data)
-
-    with fs.open(remote_dir + "/a.txt") as stream:
-        assert stream.read() == data
-
-
-def test_open_rw_flush(fs, remote_dir):
-    data = b"dvc.org"
-
-    with fs.open(remote_dir + "/b.txt", "wb") as stream:
-        for _ in range(200):
-            stream.write(data)
-            stream.write(data)
-            stream.flush()
-
-    with fs.open(remote_dir + "/b.txt", "rb") as stream:
-        assert stream.read() == data * 400
-
-
-def test_open_append(fs, remote_dir):
-    data = b"dvc.org"
-
-    with fs.open(remote_dir + "/a.txt", "wb") as stream:
-        stream.write(data)
-
-    with fs.open(remote_dir + "/a.txt", "ab") as stream:
-        stream.write(data)
-
-    with fs.open(remote_dir + "/a.txt") as stream:
-        assert stream.read() == 2 * data
-
-
-def test_open_seekable(fs, remote_dir):
-    data = b"dvc.org"
-
-    with fs.open(remote_dir + "/a.txt", "wb") as stream:
-        stream.write(data)
-
-    with fs.open(remote_dir + "/a.txt", "rb", seekable=True) as file:
-        file.seek(2)
-        assert file.read() == data[2:]
-
-
-def test_seekable(fs, remote_dir):
-    data = b"dvc.org"
-
-    with fs.open(remote_dir + "/a.txt", "wb") as stream:
-        stream.write(data)
-
-    for seekable in [True, False]:
-        with fs.open(remote_dir + "/a.txt", "rb", seekable=seekable) as file:
-            assert file.seekable() == seekable
-            assert file.read() == data
-
-    with fs.open(remote_dir + "/a.txt", "rb", seekable=False) as file:
-        with pytest.raises(OSError):
-            file.seek(5)
-
-
-def test_get_kwargs_from_urls_hadoop_fs():
-    kwargs = HadoopFileSystem._get_kwargs_from_urls(
-        "hdfs://user@localhost:8020/?replication=2"
-    )
-    assert kwargs["user"] == "user"
-    assert kwargs["host"] == "localhost"
-    assert kwargs["port"] == 8020
-    assert kwargs["replication"] == 2
-
-    kwargs = HadoopFileSystem._get_kwargs_from_urls("hdfs://user@localhost:8020/")
-    assert kwargs["user"] == "user"
-    assert kwargs["host"] == "localhost"
-    assert kwargs["port"] == 8020
-    assert "replication" not in kwargs