fsspec 2024.5.0__py3-none-any.whl → 2024.6.1__py3-none-any.whl

fsspec/implementations/tests/test_webhdfs.py

@@ -1,197 +0,0 @@
-import pickle
-import shlex
-import subprocess
-import time
-
-import pytest
-
-import fsspec
-
-requests = pytest.importorskip("requests")
-
-from fsspec.implementations.webhdfs import WebHDFS  # noqa: E402
-
-
-@pytest.fixture(scope="module")
-def hdfs_cluster():
-    cmd0 = shlex.split("htcluster shutdown")
-    try:
-        subprocess.check_output(cmd0, stderr=subprocess.STDOUT)
-    except FileNotFoundError:
-        pytest.skip("htcluster not found")
-    except subprocess.CalledProcessError as ex:
-        pytest.skip(f"htcluster failed: {ex.output.decode()}")
-    cmd1 = shlex.split("htcluster startup --image base")
-    subprocess.check_output(cmd1)
-    try:
-        while True:
-            t = 90
-            try:
-                requests.get("http://localhost:50070/webhdfs/v1/?op=LISTSTATUS")
-            except:  # noqa: E722
-                t -= 1
-                assert t > 0, "Timeout waiting for HDFS"
-                time.sleep(1)
-                continue
-            break
-        time.sleep(7)
-        yield "localhost"
-    finally:
-        subprocess.check_output(cmd0)
-
-
-def test_pickle(hdfs_cluster):
-    w = WebHDFS(hdfs_cluster, user="testuser")
-    w2 = pickle.loads(pickle.dumps(w))
-    assert w == w2
-
-
-def test_simple(hdfs_cluster):
-    w = WebHDFS(hdfs_cluster, user="testuser")
-    home = w.home_directory()
-    assert home == "/user/testuser"
-    with pytest.raises(PermissionError):
-        w.mkdir("/root")
-
-
-def test_url(hdfs_cluster):
-    url = "webhdfs://testuser@localhost:50070/user/testuser/myfile"
-    fo = fsspec.open(url, "wb", data_proxy={"worker.example.com": "localhost"})
-    with fo as f:
-        f.write(b"hello")
-    fo = fsspec.open(url, "rb", data_proxy={"worker.example.com": "localhost"})
-    with fo as f:
-        assert f.read() == b"hello"
-
-
-def test_workflow(hdfs_cluster):
-    w = WebHDFS(
-        hdfs_cluster, user="testuser", data_proxy={"worker.example.com": "localhost"}
-    )
-    fn = "/user/testuser/testrun/afile"
-    w.mkdir("/user/testuser/testrun")
-    with w.open(fn, "wb") as f:
-        f.write(b"hello")
-    assert w.exists(fn)
-    info = w.info(fn)
-    assert info["size"] == 5
-    assert w.isfile(fn)
-    assert w.cat(fn) == b"hello"
-    w.rm("/user/testuser/testrun", recursive=True)
-    assert not w.exists(fn)
-
-
-def test_with_gzip(hdfs_cluster):
-    from gzip import GzipFile
-
-    w = WebHDFS(
-        hdfs_cluster, user="testuser", data_proxy={"worker.example.com": "localhost"}
-    )
-    fn = "/user/testuser/gzfile"
-    with w.open(fn, "wb") as f:
-        gf = GzipFile(fileobj=f, mode="w")
-        gf.write(b"hello")
-        gf.close()
-    with w.open(fn, "rb") as f:
-        gf = GzipFile(fileobj=f, mode="r")
-        assert gf.read() == b"hello"
-
-
-def test_workflow_transaction(hdfs_cluster):
-    w = WebHDFS(
-        hdfs_cluster, user="testuser", data_proxy={"worker.example.com": "localhost"}
-    )
-    fn = "/user/testuser/testrun/afile"
-    w.mkdirs("/user/testuser/testrun")
-    with w.transaction:
-        with w.open(fn, "wb") as f:
-            f.write(b"hello")
-        assert not w.exists(fn)
-    assert w.exists(fn)
-    assert w.ukey(fn)
-    files = w.ls("/user/testuser/testrun", True)
-    summ = w.content_summary("/user/testuser/testrun")
-    assert summ["length"] == files[0]["size"]
-    assert summ["fileCount"] == 1
-
-    w.rm("/user/testuser/testrun", recursive=True)
-    assert not w.exists(fn)
-
-
-def test_webhdfs_cp_file(hdfs_cluster):
-    fs = WebHDFS(
-        hdfs_cluster, user="testuser", data_proxy={"worker.example.com": "localhost"}
-    )
-
-    src, dst = "/user/testuser/testrun/f1", "/user/testuser/testrun/f2"
-
-    fs.mkdir("/user/testuser/testrun")
-
-    with fs.open(src, "wb") as f:
-        f.write(b"hello")
-
-    fs.cp_file(src, dst)
-
-    assert fs.exists(src)
-    assert fs.exists(dst)
-    assert fs.cat(src) == fs.cat(dst)
-
-
-def test_path_with_equals(hdfs_cluster):
-    fs = WebHDFS(
-        hdfs_cluster, user="testuser", data_proxy={"worker.example.com": "localhost"}
-    )
-    path_with_equals = "/user/testuser/some_table/datestamp=2023-11-11"
-
-    fs.mkdir(path_with_equals)
-
-    result = fs.ls(path_with_equals)
-    assert result is not None
-    assert fs.exists(path_with_equals)
-
-
-def test_error_handling_with_equals_in_path(hdfs_cluster):
-    fs = WebHDFS(hdfs_cluster, user="testuser")
-    invalid_path_with_equals = (
-        "/user/testuser/some_table/invalid_path=datestamp=2023-11-11"
-    )
-
-    with pytest.raises(FileNotFoundError):
-        fs.ls(invalid_path_with_equals)
-
-
-def test_create_and_touch_file_with_equals(hdfs_cluster):
-    fs = WebHDFS(
-        hdfs_cluster,
-        user="testuser",
-        data_proxy={"worker.example.com": "localhost"},
-    )
-    base_path = "/user/testuser/some_table/datestamp=2023-11-11"
-    file_path = f"{base_path}/testfile.txt"
-
-    fs.mkdir(base_path)
-    fs.touch(file_path, "wb")
-    assert fs.exists(file_path)
-
-
-def test_write_read_verify_file_with_equals(hdfs_cluster):
-    fs = WebHDFS(
-        hdfs_cluster,
-        user="testuser",
-        data_proxy={"worker.example.com": "localhost"},
-    )
-    base_path = "/user/testuser/some_table/datestamp=2023-11-11"
-    file_path = f"{base_path}/testfile.txt"
-    content = b"This is some content!"
-
-    fs.mkdir(base_path)
-    with fs.open(file_path, "wb") as f:
-        f.write(content)
-
-    with fs.open(file_path, "rb") as f:
-        assert f.read() == content
-
-    file_info = fs.ls(base_path, detail=True)
-    assert len(file_info) == 1
-    assert file_info[0]["name"] == file_path
-    assert file_info[0]["size"] == len(content)

fsspec/implementations/tests/test_zip.py

@@ -1,134 +0,0 @@
-import collections.abc
-import os.path
-
-import pytest
-
-import fsspec
-from fsspec.implementations.tests.test_archive import archive_data, tempzip
-
-
-def test_info():
-    with tempzip(archive_data) as z:
-        fs = fsspec.filesystem("zip", fo=z)
-
-        # Iterate over all files.
-        for f in archive_data:
-            lhs = fs.info(f)
-
-            # Probe some specific fields of Zip archives.
-            assert "CRC" in lhs
-            assert "compress_size" in lhs
-
-
-def test_fsspec_get_mapper():
-    """Added for #788"""
-
-    with tempzip(archive_data) as z:
-        mapping = fsspec.get_mapper(f"zip::{z}")
-
-        assert isinstance(mapping, collections.abc.Mapping)
-        keys = sorted(mapping.keys())
-        assert keys == ["a", "b", "deeply/nested/path"]
-
-        # mapping.getitems() will call FSMap.fs.cat()
-        # which was not accurately implemented for zip.
-        assert isinstance(mapping, fsspec.mapping.FSMap)
-        items = dict(mapping.getitems(keys))
-        assert items == {"a": b"", "b": b"hello", "deeply/nested/path": b"stuff"}
-
-
-def test_not_cached():
-    with tempzip(archive_data) as z:
-        fs = fsspec.filesystem("zip", fo=z)
-        fs2 = fsspec.filesystem("zip", fo=z)
-        assert fs is not fs2
-
-
-def test_root_info():
-    with tempzip(archive_data) as z:
-        fs = fsspec.filesystem("zip", fo=z)
-        assert fs.info("/") == {"name": "", "type": "directory", "size": 0}
-        assert fs.info("") == {"name": "", "type": "directory", "size": 0}
-
-
-def test_write_seek(m):
-    with m.open("afile.zip", "wb") as f:
-        fs = fsspec.filesystem("zip", fo=f, mode="w")
-        fs.pipe("another", b"hi")
-        fs.zip.close()
-
-    with m.open("afile.zip", "rb") as f:
-        fs = fsspec.filesystem("zip", fo=f)
-        assert fs.cat("another") == b"hi"
-
-
-def test_rw(m):
-    # extra arg to zip means "create archive"
-    with fsspec.open(
-        "zip://afile::memory://out.zip", mode="wb", zip={"mode": "w"}
-    ) as f:
-        f.write(b"data")
-
-    with fsspec.open("zip://afile::memory://out.zip", mode="rb") as f:
-        assert f.read() == b"data"
-
-
-def test_mapper(m):
-    # extra arg to zip means "create archive"
-    mapper = fsspec.get_mapper("zip::memory://out.zip", zip={"mode": "w"})
-    with pytest.raises(KeyError):
-        mapper["a"]
-
-    mapper["a"] = b"data"
-    with pytest.raises(OSError):
-        # fails because this is write mode and we cannot also read
-        mapper["a"]
-    assert "a" in mapper  # but be can list
-
-
-def test_zip_glob_star(m):
-    with fsspec.open(
-        "zip://adir/afile::memory://out.zip", mode="wb", zip={"mode": "w"}
-    ) as f:
-        f.write(b"data")
-
-    fs, _ = fsspec.core.url_to_fs("zip::memory://out.zip")
-    outfiles = fs.glob("*")
-    assert len(outfiles) == 1
-
-    fs = fsspec.filesystem("zip", fo="memory://out.zip", mode="w")
-    fs.mkdir("adir")
-    fs.pipe("adir/afile", b"data")
-    outfiles = fs.glob("*")
-    assert len(outfiles) == 1
-
-    fn = f"{os.path.dirname(os.path.abspath((__file__)))}/out.zip"
-    fs = fsspec.filesystem("zip", fo=fn, mode="r")
-    outfiles = fs.glob("*")
-    assert len(outfiles) == 1
-
-
-def test_append(m, tmpdir):
-    fs = fsspec.filesystem("zip", fo="memory://out.zip", mode="w")
-    with fs.open("afile", "wb") as f:
-        f.write(b"data")
-    fs.close()
-
-    fs = fsspec.filesystem("zip", fo="memory://out.zip", mode="a")
-    with fs.open("bfile", "wb") as f:
-        f.write(b"data")
-    fs.close()
-
-    assert len(fsspec.open_files("zip://*::memory://out.zip")) == 2
-
-    fs = fsspec.filesystem("zip", fo=f"{tmpdir}/out.zip", mode="w")
-    with fs.open("afile", "wb") as f:
-        f.write(b"data")
-    fs.close()
-
-    fs = fsspec.filesystem("zip", fo=f"{tmpdir}/out.zip", mode="a")
-    with fs.open("bfile", "wb") as f:
-        f.write(b"data")
-    fs.close()
-
-    assert len(fsspec.open_files("zip://*::memory://out.zip")) == 2

fsspec/tests/conftest.py

@@ -1,188 +0,0 @@
-import contextlib
-import gzip
-import json
-import os
-import threading
-from collections import ChainMap
-from http.server import BaseHTTPRequestHandler, HTTPServer
-
-import pytest
-
-requests = pytest.importorskip("requests")
-port = 9898
-data = b"\n".join([b"some test data"] * 1000)
-realfile = f"http://127.0.0.1:{port}/index/realfile"
-index = b'<a href="%s">Link</a>' % realfile.encode()
-listing = open(
-    os.path.join(os.path.dirname(__file__), "data", "listing.html"), "rb"
-).read()
-win = os.name == "nt"
-
-
-def _make_listing(*paths):
-    return "\n".join(
-        f'<a href="http://127.0.0.1:{port}{f}">Link_{i}</a>'
-        for i, f in enumerate(paths)
-    ).encode()
-
-
-@pytest.fixture
-def reset_files():
-    yield
-
-    # Reset the newly added files after the
-    # test is completed.
-    HTTPTestHandler.dynamic_files.clear()
-
-
-class HTTPTestHandler(BaseHTTPRequestHandler):
-    static_files = {
-        "/index/realfile": data,
-        "/index/otherfile": data,
-        "/index": index,
-        "/data/20020401": listing,
-        "/simple/": _make_listing("/simple/file", "/simple/dir/"),
-        "/simple/file": data,
-        "/simple/dir/": _make_listing("/simple/dir/file"),
-        "/simple/dir/file": data,
-    }
-    dynamic_files = {}
-
-    files = ChainMap(dynamic_files, static_files)
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def _respond(self, code=200, headers=None, data=b""):
-        headers = headers or {}
-        headers.update({"User-Agent": "test"})
-        self.send_response(code)
-        for k, v in headers.items():
-            self.send_header(k, str(v))
-        self.end_headers()
-        if data:
-            self.wfile.write(data)
-
-    def do_GET(self):
-        file_path = self.path
-        if file_path.endswith("/") and file_path.rstrip("/") in self.files:
-            file_path = file_path.rstrip("/")
-        file_data = self.files.get(file_path)
-        if "give_path" in self.headers:
-            return self._respond(200, data=json.dumps({"path": self.path}).encode())
-        if "redirect" in self.headers and file_path != "/index/realfile":
-            new_url = f"http://127.0.0.1:{port}/index/realfile"
-            return self._respond(301, {"Location": new_url})
-        if file_data is None:
-            return self._respond(404)
-
-        status = 200
-        content_range = f"bytes 0-{len(file_data) - 1}/{len(file_data)}"
-        if ("Range" in self.headers) and ("ignore_range" not in self.headers):
-            ran = self.headers["Range"]
-            b, ran = ran.split("=")
-            start, end = ran.split("-")
-            if start:
-                content_range = f"bytes {start}-{end}/{len(file_data)}"
-                file_data = file_data[int(start) : (int(end) + 1) if end else None]
-            else:
-                # suffix only
-                l = len(file_data)
-                content_range = f"bytes {l - int(end)}-{l - 1}/{l}"
-                file_data = file_data[-int(end) :]
-            if "use_206" in self.headers:
-                status = 206
-        if "give_length" in self.headers:
-            if "gzip_encoding" in self.headers:
-                file_data = gzip.compress(file_data)
-                response_headers = {
-                    "Content-Length": len(file_data),
-                    "Content-Encoding": "gzip",
-                }
-            else:
-                response_headers = {"Content-Length": len(file_data)}
-            self._respond(status, response_headers, file_data)
-        elif "give_range" in self.headers:
-            self._respond(status, {"Content-Range": content_range}, file_data)
-        elif "give_mimetype" in self.headers:
-            self._respond(
-                status, {"Content-Type": "text/html; charset=utf-8"}, file_data
-            )
-        else:
-            self._respond(status, data=file_data)
-
-    def do_POST(self):
-        length = self.headers.get("Content-Length")
-        file_path = self.path.rstrip("/")
-        if length is None:
-            assert self.headers.get("Transfer-Encoding") == "chunked"
-            self.files[file_path] = b"".join(self.read_chunks())
-        else:
-            self.files[file_path] = self.rfile.read(length)
-        self._respond(200)
-
-    do_PUT = do_POST
-
-    def read_chunks(self):
-        length = -1
-        while length != 0:
-            line = self.rfile.readline().strip()
-            if len(line) == 0:
-                length = 0
-            else:
-                length = int(line, 16)
-            yield self.rfile.read(length)
-            self.rfile.readline()
-
-    def do_HEAD(self):
-        if "head_not_auth" in self.headers:
-            return self._respond(
-                403, {"Content-Length": 123}, b"not authorized for HEAD request"
-            )
-        elif "head_ok" not in self.headers:
-            return self._respond(405)
-
-        file_path = self.path.rstrip("/")
-        file_data = self.files.get(file_path)
-        if file_data is None:
-            return self._respond(404)
-
-        if ("give_length" in self.headers) or ("head_give_length" in self.headers):
-            response_headers = {"Content-Length": len(file_data)}
-            if "zero_length" in self.headers:
-                response_headers["Content-Length"] = 0
-            elif "gzip_encoding" in self.headers:
-                file_data = gzip.compress(file_data)
-                response_headers["Content-Encoding"] = "gzip"
-                response_headers["Content-Length"] = len(file_data)
-
-            self._respond(200, response_headers)
-        elif "give_range" in self.headers:
-            self._respond(
-                200, {"Content-Range": f"0-{len(file_data) - 1}/{len(file_data)}"}
-            )
-        elif "give_etag" in self.headers:
-            self._respond(200, {"ETag": "xxx"})
-        else:
-            self._respond(200)  # OK response, but no useful info
-
-
-@contextlib.contextmanager
-def serve():
-    server_address = ("", port)
-    httpd = HTTPServer(server_address, HTTPTestHandler)
-    th = threading.Thread(target=httpd.serve_forever)
-    th.daemon = True
-    th.start()
-    try:
-        yield f"http://127.0.0.1:{port}"
-    finally:
-        httpd.socket.close()
-        httpd.shutdown()
-        th.join()
-
-
-@pytest.fixture(scope="module")
-def server():
-    with serve() as s:
-        yield s

fsspec/tests/data/listing.html

@@ -1 +0,0 @@
-\n<html><head><title>nasagrace.unl.edu - /data/20020401/</title></head><body><H1>nasagrace.unl.edu - /data/20020401/</H1><hr>\n\n<pre><A HREF="/data/">[To Parent Directory]</A><br><br> 1/27/2020  9:54 AM      1194073 <A HREF="/data/20020401/GRACE_GWS_20020401.pdf">GRACE_GWS_20020401.pdf</A><br> 1/27/2020  9:54 AM       380043 <A HREF="/data/20020401/GRACE_GWS_20020401.png">GRACE_GWS_20020401.png</A><br> 1/27/2020  9:54 AM      1192987 <A HREF="/data/20020401/GRACE_RTZSM_20020401.pdf">GRACE_RTZSM_20020401.pdf</A><br> 1/27/2020  9:54 AM       384342 <A HREF="/data/20020401/GRACE_RTZSM_20020401.png">GRACE_RTZSM_20020401.png</A><br> 1/27/2020  9:55 AM      1202046 <A HREF="/data/20020401/GRACE_SFSM_20020401.pdf">GRACE_SFSM_20020401.pdf</A><br> 1/27/2020  9:55 AM       387932 <A HREF="/data/20020401/GRACE_SFSM_20020401.png">GRACE_SFSM_20020401.png</A><br> 1/27/2020  9:54 AM      4975980 <A HREF="/data/20020401/GRACEDADM_CLSM0125US_7D.A20020401.030.nc4">GRACEDADM_CLSM0125US_7D.A20020401.030.nc4</A><br> 1/27/2020  9:54 AM       345640 <A HREF="/data/20020401/gws_perc_0125deg_US_20020401.img">gws_perc_0125deg_US_20020401.img</A><br> 1/27/2020  9:54 AM         2272 <A HREF="/data/20020401/gws_perc_0125deg_US_20020401.img.aux.xml">gws_perc_0125deg_US_20020401.img.aux.xml</A><br> 1/27/2020  9:54 AM         5678 <A HREF="/data/20020401/gws_perc_0125deg_US_20020401.img.xml">gws_perc_0125deg_US_20020401.img.xml</A><br> 1/27/2020  9:54 AM       136081 <A HREF="/data/20020401/gws_perc_0125deg_US_20020401.rrd">gws_perc_0125deg_US_20020401.rrd</A><br> 1/27/2020  9:54 AM           83