fsspec 2024.5.0__py3-none-any.whl → 2024.6.0__py3-none-any.whl

fsspec/implementations/tests/test_sftp.py

@@ -1,233 +0,0 @@
-import os
-import shlex
-import subprocess
-import time
-from tarfile import TarFile
-
-import pytest
-
-import fsspec
-
-pytest.importorskip("paramiko")
-
-
-def stop_docker(name):
-    cmd = shlex.split(f'docker ps -a -q --filter "name={name}"')
-    cid = subprocess.check_output(cmd).strip().decode()
-    if cid:
-        subprocess.call(["docker", "rm", "-f", cid])
-
-
-@pytest.fixture(scope="module")
-def ssh():
-    try:
-        pchk = ["docker", "run", "--name", "fsspec_test_sftp", "hello-world"]
-        subprocess.check_call(pchk)
-        stop_docker("fsspec_test_sftp")
-    except (subprocess.CalledProcessError, FileNotFoundError):
-        pytest.skip("docker run not available")
-        return
-
-    # requires docker
-    cmds = [
-        r"apt-get update",
-        r"apt-get install -y openssh-server",
-        r"mkdir /var/run/sshd",
-        "bash -c \"echo 'root:pass' | chpasswd\"",
-        (
-            r"sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' "
-            r"/etc/ssh/sshd_config"
-        ),
-        (
-            r"sed 's@session\s*required\s*pam_loginuid.so@session optional "
-            r"pam_loginuid.so@g' -i /etc/pam.d/sshd"
-        ),
-        r'bash -c "echo \"export VISIBLE=now\" >> /etc/profile"',
-        r"/usr/sbin/sshd",
-    ]
-    name = "fsspec_sftp"
-    stop_docker(name)
-    cmd = f"docker run -d -p 9200:22 --name {name} ubuntu:16.04 sleep 9000"
-    try:
-        cid = subprocess.check_output(shlex.split(cmd)).strip().decode()
-        for cmd in cmds:
-            subprocess.call(["docker", "exec", cid] + shlex.split(cmd))
-        time.sleep(1)
-        yield {
-            "host": "localhost",
-            "port": 9200,
-            "username": "root",
-            "password": "pass",
-        }
-    finally:
-        stop_docker(name)
-
-
-@pytest.fixture(scope="module")
-def root_path():
-    return "/home/someuser/"
-
-
-def test_simple(ssh, root_path):
-    f = fsspec.get_filesystem_class("sftp")(**ssh)
-    f.mkdirs(root_path + "deeper")
-    try:
-        f.touch(root_path + "deeper/afile")
-        assert f.find(root_path) == [root_path + "deeper/afile"]
-        assert f.ls(root_path + "deeper/") == [root_path + "deeper/afile"]
-        assert f.info(root_path + "deeper/afile")["type"] == "file"
-        assert f.info(root_path + "deeper/afile")["size"] == 0
-        assert f.exists(root_path)
-    finally:
-        f.rm(root_path, recursive=True)
-        assert not f.exists(root_path)
-
-
-@pytest.mark.parametrize("protocol", ["sftp", "ssh"])
-def test_with_url(protocol, ssh):
-    fo = fsspec.open(
-        protocol
-        + "://{username}:{password}@{host}:{port}/home/someuserout".format(**ssh),
-        "wb",
-    )
-    with fo as f:
-        f.write(b"hello")
-    fo = fsspec.open(
-        protocol
-        + "://{username}:{password}@{host}:{port}/home/someuserout".format(**ssh),
-        "rb",
-    )
-    with fo as f:
-        assert f.read() == b"hello"
-
-
-@pytest.mark.parametrize("protocol", ["sftp", "ssh"])
-def test_get_dir(protocol, ssh, root_path, tmpdir):
-    path = str(tmpdir)
-    f = fsspec.filesystem(protocol, **ssh)
-    f.mkdirs(root_path + "deeper", exist_ok=True)
-    f.touch(root_path + "deeper/afile")
-    f.get(root_path, path, recursive=True)
-
-    assert os.path.isdir(f"{path}/deeper")
-    assert os.path.isfile(f"{path}/deeper/afile")
-
-    f.get(
-        protocol
-        + "://{username}:{password}@{host}:{port}{root_path}".format(
-            root_path=root_path, **ssh
-        ),
-        f"{path}/test2",
-        recursive=True,
-    )
-
-    assert os.path.isdir(f"{path}/test2/deeper")
-    assert os.path.isfile(f"{path}/test2/deeper/afile")
-
-
-@pytest.fixture(scope="module")
-def netloc(ssh):
-    username = ssh.get("username")
-    password = ssh.get("password")
-    host = ssh.get("host")
-    port = ssh.get("port")
-    userpass = (
-        f"{username}:{password if password is not None else ''}@"
-        if username is not None
-        else ""
-    )
-    netloc = f"{host}:{port if port is not None else ''}"
-    return userpass + netloc
-
-
-def test_put_file(ssh, tmp_path, root_path):
-    tmp_file = tmp_path / "a.txt"
-    with open(tmp_file, mode="w") as fd:
-        fd.write("blabla")
-
-    f = fsspec.get_filesystem_class("sftp")(**ssh)
-    f.put_file(lpath=tmp_file, rpath=root_path + "a.txt")
-
-
-def test_simple_with_tar(ssh, netloc, tmp_path, root_path):
-    files_to_pack = ["a.txt", "b.txt"]
-
-    tar_filename = make_tarfile(files_to_pack, tmp_path)
-
-    f = fsspec.get_filesystem_class("sftp")(**ssh)
-    f.mkdirs(f"{root_path}deeper", exist_ok=True)
-    try:
-        remote_tar_filename = f"{root_path}deeper/somefile.tar"
-        with f.open(remote_tar_filename, mode="wb") as wfd:
-            with open(tar_filename, mode="rb") as rfd:
-                wfd.write(rfd.read())
-        fs = fsspec.open(f"tar::ssh://{netloc}{remote_tar_filename}").fs
-        files = fs.find("/")
-        assert files == files_to_pack
-    finally:
-        f.rm(root_path, recursive=True)
-
-
-def make_tarfile(files_to_pack, tmp_path):
-    """Create a tarfile with some files."""
-    tar_filename = tmp_path / "sometarfile.tar"
-    for filename in files_to_pack:
-        with open(tmp_path / filename, mode="w") as fd:
-            fd.write("")
-    with TarFile(tar_filename, mode="w") as tf:
-        for filename in files_to_pack:
-            tf.add(tmp_path / filename, arcname=filename)
-    return tar_filename
-
-
-def test_transaction(ssh, root_path):
-    f = fsspec.get_filesystem_class("sftp")(**ssh)
-    f.mkdirs(root_path + "deeper", exist_ok=True)
-    try:
-        f.start_transaction()
-        f.touch(root_path + "deeper/afile")
-        assert f.find(root_path) == []
-        f.end_transaction()
-        assert f.find(root_path) == [root_path + "deeper/afile"]
-
-        with f.transaction:
-            assert f._intrans
-            f.touch(root_path + "deeper/afile2")
-            assert f.find(root_path) == [root_path + "deeper/afile"]
-        assert f.find(root_path) == [
-            root_path + "deeper/afile",
-            root_path + "deeper/afile2",
-        ]
-    finally:
-        f.rm(root_path, recursive=True)
-
-
-@pytest.mark.parametrize("path", ["/a/b/c", "a/b/c"])
-def test_mkdir_create_parent(ssh, path):
-    f = fsspec.get_filesystem_class("sftp")(**ssh)
-
-    with pytest.raises(FileNotFoundError):
-        f.mkdir(path, create_parents=False)
-
-    f.mkdir(path)
-    assert f.exists(path)
-
-    with pytest.raises(FileExistsError, match=path):
-        f.mkdir(path)
-
-    f.rm(path, recursive=True)
-    assert not f.exists(path)
-
-
-@pytest.mark.parametrize("path", ["/a/b/c", "a/b/c"])
-def test_makedirs_exist_ok(ssh, path):
-    f = fsspec.get_filesystem_class("sftp")(**ssh)
-
-    f.makedirs(path, exist_ok=False)
-
-    with pytest.raises(FileExistsError, match=path):
-        f.makedirs(path, exist_ok=False)
-
-    f.makedirs(path, exist_ok=True)
-    f.rm(path, recursive=True)
-    assert not f.exists(path)

fsspec/implementations/tests/test_smb.py

@@ -1,139 +0,0 @@
-"""
-Test SMBFileSystem class using a docker container
-"""
-
-import logging
-import os
-import shlex
-import subprocess
-import time
-
-import pytest
-
-import fsspec
-
-pytest.importorskip("smbprotocol")
-
-# ruff: noqa: F821
-
-if os.environ.get("WSL_INTEROP"):
-    # Running on WSL (Windows)
-    port_test = [9999]
-
-else:
-    # ! pylint: disable=redefined-outer-name,missing-function-docstring
-
-    # Test standard and non-standard ports
-    default_port = 445
-    port_test = [None, default_port, 9999]
-
-
-def stop_docker(container):
-    cmd = shlex.split('docker ps -a -q --filter "name=%s"' % container)
-    cid = subprocess.check_output(cmd).strip().decode()
-    if cid:
-        subprocess.call(["docker", "rm", "-f", "-v", cid])
-
-
-@pytest.fixture(scope="module", params=port_test)
-def smb_params(request):
-    try:
-        pchk = ["docker", "run", "--name", "fsspec_test_smb", "hello-world"]
-        subprocess.check_call(pchk)
-        stop_docker("fsspec_test_smb")
-    except (subprocess.CalledProcessError, FileNotFoundError):
-        pytest.skip("docker run not available")
-
-    # requires docker
-    container = "fsspec_smb"
-    stop_docker(container)
-    cfg = "-p -u 'testuser;testpass' -s 'home;/share;no;no;no;testuser'"
-    port = request.param if request.param is not None else default_port
-    img = (
-        f"docker run --name {container} --detach -p 139:139 -p {port}:445 dperson/samba"  # noqa: E231 E501
-    )
-    cmd = f"{img} {cfg}"
-    try:
-        cid = subprocess.check_output(shlex.split(cmd)).strip().decode()
-        logger = logging.getLogger("fsspec")
-        logger.debug("Container: %s", cid)
-        time.sleep(1)
-        yield {
-            "host": "localhost",
-            "port": request.param,
-            "username": "testuser",
-            "password": "testpass",
-            "register_session_retries": 100,  # max ~= 10 seconds
-        }
-    finally:
-        import smbclient  # pylint: disable=import-outside-toplevel
-
-        smbclient.reset_connection_cache()
-        stop_docker(container)
-
-
-@pytest.mark.flaky(reruns=2, reruns_delay=2)
-def test_simple(smb_params):
-    adir = "/home/adir"
-    adir2 = "/home/adir/otherdir/"
-    afile = "/home/adir/otherdir/afile"
-    fsmb = fsspec.get_filesystem_class("smb")(**smb_params)
-    fsmb.mkdirs(adir2)
-    fsmb.touch(afile)
-    assert fsmb.find(adir) == [afile]
-    assert fsmb.ls(adir2, detail=False) == [afile]
-    assert fsmb.info(afile)["type"] == "file"
-    assert fsmb.info(afile)["size"] == 0
-    assert fsmb.exists(adir)
-    fsmb.rm(adir, recursive=True)
-    assert not fsmb.exists(adir)
-
-
-@pytest.mark.flaky(reruns=2, reruns_delay=2)
-def test_with_url(smb_params):
-    if smb_params["port"] is None:
-        smb_url = "smb://{username}:{password}@{host}/home/someuser.txt"
-    else:
-        smb_url = "smb://{username}:{password}@{host}:{port}/home/someuser.txt"
-    fwo = fsspec.open(smb_url.format(**smb_params), "wb")
-    with fwo as fwr:
-        fwr.write(b"hello")
-    fro = fsspec.open(smb_url.format(**smb_params), "rb")
-    with fro as frd:
-        read_result = frd.read()
-        assert read_result == b"hello"
-
-
-@pytest.mark.flaky(reruns=2, reruns_delay=2)
-def test_transaction(smb_params):
-    afile = "/home/afolder/otherdir/afile"
-    afile2 = "/home/afolder/otherdir/afile2"
-    adir = "/home/afolder"
-    adir2 = "/home/afolder/otherdir"
-    fsmb = fsspec.get_filesystem_class("smb")(**smb_params)
-    fsmb.mkdirs(adir2)
-    fsmb.start_transaction()
-    fsmb.touch(afile)
-    assert fsmb.find(adir) == []
-    fsmb.end_transaction()
-    assert fsmb.find(adir) == [afile]
-
-    with fsmb.transaction:
-        assert fsmb._intrans
-        fsmb.touch(afile2)
-        assert fsmb.find(adir) == [afile]
-    assert fsmb.find(adir) == [afile, afile2]
-
-
-@pytest.mark.flaky(reruns=2, reruns_delay=2)
-def test_makedirs_exist_ok(smb_params):
-    fsmb = fsspec.get_filesystem_class("smb")(**smb_params)
-    fsmb.makedirs("/home/a/b/c")
-    fsmb.makedirs("/home/a/b/c", exist_ok=True)
-
-
-@pytest.mark.flaky(reruns=2, reruns_delay=2)
-def test_rename_from_upath(smb_params):
-    fsmb = fsspec.get_filesystem_class("smb")(**smb_params)
-    fsmb.makedirs("/home/a/b/c", exist_ok=True)
-    fsmb.mv("/home/a/b/c", "/home/a/b/d", recursive=False, maxdepth=None)

fsspec/implementations/tests/test_tar.py

@@ -1,243 +0,0 @@
-from __future__ import annotations
-
-import os
-import shutil
-import tarfile
-import tempfile
-from io import BytesIO
-from pathlib import Path
-
-import pytest
-
-import fsspec
-from fsspec.core import OpenFile
-from fsspec.implementations.cached import WholeFileCacheFileSystem
-from fsspec.implementations.tar import TarFileSystem
-from fsspec.implementations.tests.test_archive import archive_data, temptar
-
-
-def test_info():
-    with temptar(archive_data) as t:
-        fs = fsspec.filesystem("tar", fo=t)
-
-        # Iterate over all directories.
-        # Probe specific fields of Tar archives.
-        for d in fs._all_dirnames(archive_data.keys()):
-            lhs = fs.info(d)
-            del lhs["chksum"]
-            expected = {
-                "name": f"{d}",
-                "size": 0,
-                "type": "directory",
-                "devmajor": 0,
-                "devminor": 0,
-                "gname": "",
-                "linkname": "",
-                "uid": 0,
-                "gid": 0,
-                "mode": 420,
-                "mtime": 0,
-                "uname": "",
-            }
-            assert lhs == expected
-
-        # Iterate over all files.
-        for f in archive_data:
-            lhs = fs.info(f)
-
-            # Probe some specific fields of Tar archives.
-            assert "mode" in lhs
-            assert "uid" in lhs
-            assert "gid" in lhs
-            assert "mtime" in lhs
-            assert "chksum" in lhs
-
-
-@pytest.mark.parametrize(
-    "recipe",
-    [
-        {"mode": "w", "suffix": ".tar", "magic": b"a\x00\x00\x00\x00"},
-        {"mode": "w:gz", "suffix": ".tar.gz", "magic": b"\x1f\x8b\x08\x08"},
-        {"mode": "w:bz2", "suffix": ".tar.bz2", "magic": b"BZh91AY"},
-        {"mode": "w:xz", "suffix": ".tar.xz", "magic": b"\xfd7zXZ\x00\x00"},
-    ],
-    ids=["tar", "tar-gz", "tar-bz2", "tar-xz"],
-)
-def test_compressions(recipe):
-    """
-    Run tests on all available tar file compression variants.
-    """
-    with temptar(archive_data, mode=recipe["mode"], suffix=recipe["suffix"]) as t:
-        fs = fsspec.filesystem("tar", fo=t)
-
-        # Verify that the tar archive has the correct compression.
-        with open(t, "rb") as raw:
-            assert raw.read()[:10].startswith(recipe["magic"])
-
-        # Verify content of a sample file.
-        assert fs.cat("b") == b"hello"
-
-
-@pytest.mark.parametrize(
-    "recipe",
-    [
-        {"mode": "w", "suffix": ".tar", "magic": b"a\x00\x00\x00\x00"},
-        {"mode": "w:gz", "suffix": ".tar.gz", "magic": b"\x1f\x8b\x08\x08"},
-        {"mode": "w:bz2", "suffix": ".tar.bz2", "magic": b"BZh91AY"},
-        {"mode": "w:xz", "suffix": ".tar.xz", "magic": b"\xfd7zXZ\x00\x00"},
-    ],
-    ids=["tar", "tar-gz", "tar-bz2", "tar-xz"],
-)
-def test_filesystem_direct(recipe, tmpdir):
-    """
-    Run tests through a real fsspec filesystem implementation.
-    Here: `LocalFileSystem`.
-    """
-
-    filename = os.path.join(tmpdir, f'temp{recipe["suffix"]}')
-
-    fs = fsspec.filesystem("file")
-    f = OpenFile(fs, filename, mode="wb")
-
-    with temptar(archive_data, mode=recipe["mode"], suffix=recipe["suffix"]) as tf:
-        with f as fo:
-            fo.write(open(tf, "rb").read())
-
-    # Verify that the tar archive has the correct compression.
-    with open(filename, "rb") as raw:
-        assert raw.read()[:10].startswith(recipe["magic"])
-
-    # Verify content of a sample file.
-    with fs.open(filename) as resource:
-        tarfs = fsspec.filesystem("tar", fo=resource)
-        assert tarfs.cat("b") == b"hello"
-
-
-@pytest.mark.parametrize(
-    "recipe",
-    [
-        {"mode": "w", "suffix": ".tar", "magic": b"a\x00\x00\x00\x00"},
-        {"mode": "w:gz", "suffix": ".tar.gz", "magic": b"\x1f\x8b\x08\x08"},
-        {"mode": "w:bz2", "suffix": ".tar.bz2", "magic": b"BZh91AY"},
-        {"mode": "w:xz", "suffix": ".tar.xz", "magic": b"\xfd7zXZ\x00\x00"},
-    ],
-    ids=["tar", "tar-gz", "tar-bz2", "tar-xz"],
-)
-def test_filesystem_cached(recipe, tmpdir):
-    """
-    Run tests through a real, cached, fsspec filesystem implementation.
-    Here: `TarFileSystem` over `WholeFileCacheFileSystem` over `LocalFileSystem`.
-    """
-
-    filename = os.path.join(tmpdir, f'temp{recipe["suffix"]}')
-
-    # Create a filesystem from test fixture.
-    fs = fsspec.filesystem("file")
-    f = OpenFile(fs, filename, mode="wb")
-
-    with temptar(archive_data, mode=recipe["mode"], suffix=recipe["suffix"]) as tf:
-        with f as fo:
-            fo.write(open(tf, "rb").read())
-
-    # Verify that the tar archive has the correct compression.
-    with open(filename, "rb") as raw:
-        assert raw.read()[:10].startswith(recipe["magic"])
-
-    # Access cached filesystem.
-    cachedir = tempfile.mkdtemp()
-    filesystem = WholeFileCacheFileSystem(fs=fs, cache_storage=cachedir)
-
-    # Verify the cache is empty beforehand.
-    assert os.listdir(cachedir) == []
-
-    # Verify content of a sample file.
-    with filesystem.open(filename) as resource:
-        tarfs = fsspec.filesystem("tar", fo=resource)
-        assert tarfs.cat("b") == b"hello"
-
-    # Verify the cache is populated afterwards.
-    assert len(os.listdir(cachedir)) == 2
-
-    # Verify that the cache is empty after clearing it.
-    filesystem.clear_cache()
-    assert os.listdir(cachedir) == []
-
-    filesystem.clear_cache()
-    shutil.rmtree(cachedir)
-
-
-@pytest.mark.parametrize(
-    "recipe",
-    [
-        {"mode": "w", "suffix": ".tar", "magic": b"a\x00\x00\x00\x00"},
-        {"mode": "w:gz", "suffix": ".tar.gz", "magic": b"\x1f\x8b\x08\x08"},
-        {"mode": "w:bz2", "suffix": ".tar.bz2", "magic": b"BZh91AY"},
-        {"mode": "w:xz", "suffix": ".tar.xz", "magic": b"\xfd7zXZ\x00\x00"},
-    ],
-    ids=["tar", "tar-gz", "tar-bz2", "tar-xz"],
-)
-def test_url_to_fs_direct(recipe, tmpdir):
-    with temptar(archive_data, mode=recipe["mode"], suffix=recipe["suffix"]) as tf:
-        url = f"tar://inner::file://{tf}"
-        fs, url = fsspec.core.url_to_fs(url=url)
-        assert fs.cat("b") == b"hello"
-
-
-@pytest.mark.parametrize(
-    "recipe",
-    [
-        {"mode": "w", "suffix": ".tar"},
-        {"mode": "w:gz", "suffix": ".tar.gz"},
-        {"mode": "w:bz2", "suffix": ".tar.bz2"},
-        {"mode": "w:xz", "suffix": ".tar.xz"},
-    ],
-    ids=["tar", "tar-gz", "tar-bz2", "tar-xz"],
-)
-def test_url_to_fs_cached(recipe, tmpdir):
-    with temptar(archive_data, mode=recipe["mode"], suffix=recipe["suffix"]) as tf:
-        url = f"tar://inner::simplecache::file://{tf}"
-        # requires same_names in order to be able to guess compression from
-        # filename
-        fs, url = fsspec.core.url_to_fs(url=url, simplecache={"same_names": True})
-        assert fs.cat("b") == b"hello"
-
-
-@pytest.mark.parametrize(
-    "compression", ["", "gz", "bz2", "xz"], ids=["tar", "tar-gz", "tar-bz2", "tar-xz"]
-)
-def test_ls_with_folders(compression: str, tmp_path: Path):
-    """
-    Create a tar file that doesn't include the intermediate folder structure,
-    but make sure that the reading filesystem is still able to resolve the
-    intermediate folders, like the ZipFileSystem.
-    """
-    tar_data: dict[str, bytes] = {
-        "a.pdf": b"Hello A!",
-        "b/c.pdf": b"Hello C!",
-        "d/e/f.pdf": b"Hello F!",
-        "d/g.pdf": b"Hello G!",
-    }
-    if compression:
-        temp_archive_file = tmp_path / f"test_tar_file.tar.{compression}"
-    else:
-        temp_archive_file = tmp_path / "test_tar_file.tar"
-    with open(temp_archive_file, "wb") as fd:
-        # We need to manually write the tarfile here, because temptar
-        # creates intermediate directories which is not how tars are always created
-        with tarfile.open(fileobj=fd, mode=f"w:{compression}") as tf:
-            for tar_file_path, data in tar_data.items():
-                content = data
-                info = tarfile.TarInfo(name=tar_file_path)
-                info.size = len(content)
-                tf.addfile(info, BytesIO(content))
-    with open(temp_archive_file, "rb") as fd:
-        fs = TarFileSystem(fd)
-        assert fs.find("/", withdirs=True) == [
-            "a.pdf",
-            "b",
-            "b/c.pdf",
-            "d",
-            "d/e",
-            "d/e/f.pdf",
-            "d/g.pdf",
-        ]