fsspec 2024.3.1__py3-none-any.whl → 2024.5.0__py3-none-any.whl

fsspec/tests/conftest.py

@@ -0,0 +1,188 @@
+import contextlib
+import gzip
+import json
+import os
+import threading
+from collections import ChainMap
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+import pytest
+
+requests = pytest.importorskip("requests")
+port = 9898
+data = b"\n".join([b"some test data"] * 1000)
+realfile = f"http://127.0.0.1:{port}/index/realfile"
+index = b'<a href="%s">Link</a>' % realfile.encode()
+listing = open(
+    os.path.join(os.path.dirname(__file__), "data", "listing.html"), "rb"
+).read()
+win = os.name == "nt"
+
+
+def _make_listing(*paths):
+    return "\n".join(
+        f'<a href="http://127.0.0.1:{port}{f}">Link_{i}</a>'
+        for i, f in enumerate(paths)
+    ).encode()
+
+
+@pytest.fixture
+def reset_files():
+    yield
+
+    # Reset the newly added files after the
+    # test is completed.
+    HTTPTestHandler.dynamic_files.clear()
+
+
+class HTTPTestHandler(BaseHTTPRequestHandler):
+    static_files = {
+        "/index/realfile": data,
+        "/index/otherfile": data,
+        "/index": index,
+        "/data/20020401": listing,
+        "/simple/": _make_listing("/simple/file", "/simple/dir/"),
+        "/simple/file": data,
+        "/simple/dir/": _make_listing("/simple/dir/file"),
+        "/simple/dir/file": data,
+    }
+    dynamic_files = {}
+
+    files = ChainMap(dynamic_files, static_files)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def _respond(self, code=200, headers=None, data=b""):
+        headers = headers or {}
+        headers.update({"User-Agent": "test"})
+        self.send_response(code)
+        for k, v in headers.items():
+            self.send_header(k, str(v))
+        self.end_headers()
+        if data:
+            self.wfile.write(data)
+
+    def do_GET(self):
+        file_path = self.path
+        if file_path.endswith("/") and file_path.rstrip("/") in self.files:
+            file_path = file_path.rstrip("/")
+        file_data = self.files.get(file_path)
+        if "give_path" in self.headers:
+            return self._respond(200, data=json.dumps({"path": self.path}).encode())
+        if "redirect" in self.headers and file_path != "/index/realfile":
+            new_url = f"http://127.0.0.1:{port}/index/realfile"
+            return self._respond(301, {"Location": new_url})
+        if file_data is None:
+            return self._respond(404)
+
+        status = 200
+        content_range = f"bytes 0-{len(file_data) - 1}/{len(file_data)}"
+        if ("Range" in self.headers) and ("ignore_range" not in self.headers):
+            ran = self.headers["Range"]
+            b, ran = ran.split("=")
+            start, end = ran.split("-")
+            if start:
+                content_range = f"bytes {start}-{end}/{len(file_data)}"
+                file_data = file_data[int(start) : (int(end) + 1) if end else None]
+            else:
+                # suffix only
+                l = len(file_data)
+                content_range = f"bytes {l - int(end)}-{l - 1}/{l}"
+                file_data = file_data[-int(end) :]
+            if "use_206" in self.headers:
+                status = 206
+        if "give_length" in self.headers:
+            if "gzip_encoding" in self.headers:
+                file_data = gzip.compress(file_data)
+                response_headers = {
+                    "Content-Length": len(file_data),
+                    "Content-Encoding": "gzip",
+                }
+            else:
+                response_headers = {"Content-Length": len(file_data)}
+            self._respond(status, response_headers, file_data)
+        elif "give_range" in self.headers:
+            self._respond(status, {"Content-Range": content_range}, file_data)
+        elif "give_mimetype" in self.headers:
+            self._respond(
+                status, {"Content-Type": "text/html; charset=utf-8"}, file_data
+            )
+        else:
+            self._respond(status, data=file_data)
+
+    def do_POST(self):
+        length = self.headers.get("Content-Length")
+        file_path = self.path.rstrip("/")
+        if length is None:
+            assert self.headers.get("Transfer-Encoding") == "chunked"
+            self.files[file_path] = b"".join(self.read_chunks())
+        else:
+            self.files[file_path] = self.rfile.read(length)
+        self._respond(200)
+
+    do_PUT = do_POST
+
+    def read_chunks(self):
+        length = -1
+        while length != 0:
+            line = self.rfile.readline().strip()
+            if len(line) == 0:
+                length = 0
+            else:
+                length = int(line, 16)
+            yield self.rfile.read(length)
+            self.rfile.readline()
+
+    def do_HEAD(self):
+        if "head_not_auth" in self.headers:
+            return self._respond(
+                403, {"Content-Length": 123}, b"not authorized for HEAD request"
+            )
+        elif "head_ok" not in self.headers:
+            return self._respond(405)
+
+        file_path = self.path.rstrip("/")
+        file_data = self.files.get(file_path)
+        if file_data is None:
+            return self._respond(404)
+
+        if ("give_length" in self.headers) or ("head_give_length" in self.headers):
+            response_headers = {"Content-Length": len(file_data)}
+            if "zero_length" in self.headers:
+                response_headers["Content-Length"] = 0
+            elif "gzip_encoding" in self.headers:
+                file_data = gzip.compress(file_data)
+                response_headers["Content-Encoding"] = "gzip"
+                response_headers["Content-Length"] = len(file_data)
+
+            self._respond(200, response_headers)
+        elif "give_range" in self.headers:
+            self._respond(
+                200, {"Content-Range": f"0-{len(file_data) - 1}/{len(file_data)}"}
+            )
+        elif "give_etag" in self.headers:
+            self._respond(200, {"ETag": "xxx"})
+        else:
+            self._respond(200)  # OK response, but no useful info
+
+
+@contextlib.contextmanager
+def serve():
+    server_address = ("", port)
+    httpd = HTTPServer(server_address, HTTPTestHandler)
+    th = threading.Thread(target=httpd.serve_forever)
+    th.daemon = True
+    th.start()
+    try:
+        yield f"http://127.0.0.1:{port}"
+    finally:
+        httpd.socket.close()
+        httpd.shutdown()
+        th.join()
+
+
+@pytest.fixture(scope="module")
+def server():
+    with serve() as s:
+        yield s

fsspec/tests/data/listing.html

@@ -0,0 +1 @@
+\n<html><head><title>nasagrace.unl.edu - /data/20020401/</title></head><body><H1>nasagrace.unl.edu - /data/20020401/</H1><hr>\n\n<pre><A HREF="/data/">[To Parent Directory]</A><br><br> 1/27/2020  9:54 AM      1194073 <A HREF="/data/20020401/GRACE_GWS_20020401.pdf">GRACE_GWS_20020401.pdf</A><br> 1/27/2020  9:54 AM       380043 <A HREF="/data/20020401/GRACE_GWS_20020401.png">GRACE_GWS_20020401.png</A><br> 1/27/2020  9:54 AM      1192987 <A HREF="/data/20020401/GRACE_RTZSM_20020401.pdf">GRACE_RTZSM_20020401.pdf</A><br> 1/27/2020  9:54 AM       384342 <A HREF="/data/20020401/GRACE_RTZSM_20020401.png">GRACE_RTZSM_20020401.png</A><br> 1/27/2020  9:55 AM      1202046 <A HREF="/data/20020401/GRACE_SFSM_20020401.pdf">GRACE_SFSM_20020401.pdf</A><br> 1/27/2020  9:55 AM       387932 <A HREF="/data/20020401/GRACE_SFSM_20020401.png">GRACE_SFSM_20020401.png</A><br> 1/27/2020  9:54 AM      4975980 <A HREF="/data/20020401/GRACEDADM_CLSM0125US_7D.A20020401.030.nc4">GRACEDADM_CLSM0125US_7D.A20020401.030.nc4</A><br> 1/27/2020  9:54 AM       345640 <A HREF="/data/20020401/gws_perc_0125deg_US_20020401.img">gws_perc_0125deg_US_20020401.img</A><br> 1/27/2020  9:54 AM         2272 <A HREF="/data/20020401/gws_perc_0125deg_US_20020401.img.aux.xml">gws_perc_0125deg_US_20020401.img.aux.xml</A><br> 1/27/2020  9:54 AM         5678 <A HREF="/data/20020401/gws_perc_0125deg_US_20020401.img.xml">gws_perc_0125deg_US_20020401.img.xml</A><br> 1/27/2020  9:54 AM       136081 <A HREF="/data/20020401/gws_perc_0125deg_US_20020401.rrd">gws_perc_0125deg_US_20020401.rrd</A><br> 1/27/2020  9:54 AM           83

fsspec/tests/test_api.py

@@ -0,0 +1,498 @@
+"""Tests the spec, using memoryfs"""
+
+import contextlib
+import os
+import pickle
+import tempfile
+from unittest.mock import Mock
+
+import pytest
+
+import fsspec
+from fsspec.implementations.memory import MemoryFile, MemoryFileSystem
+
+
+def test_idempotent():
+    MemoryFileSystem.clear_instance_cache()
+    fs = MemoryFileSystem()
+    fs2 = MemoryFileSystem()
+    assert fs is fs2
+    assert MemoryFileSystem.current() is fs2
+
+    MemoryFileSystem.clear_instance_cache()
+    assert not MemoryFileSystem._cache
+
+    fs2 = MemoryFileSystem().current()
+    assert fs == fs2
+
+
+def test_pickle():
+    fs = MemoryFileSystem()
+    fs2 = pickle.loads(pickle.dumps(fs))
+    assert fs == fs2
+
+
+def test_class_methods():
+    assert MemoryFileSystem._strip_protocol("memory://stuff") == "/stuff"
+    assert MemoryFileSystem._strip_protocol("stuff") == "/stuff"
+    assert MemoryFileSystem._strip_protocol("other://stuff") == "other://stuff"
+
+    assert MemoryFileSystem._get_kwargs_from_urls("memory://user@thing") == {}
+
+
+def test_multi(m):
+    m.pipe("/afile", b"data")
+    fs, token, paths = fsspec.core.get_fs_token_paths(["/afile", "/afile"])
+    assert len(paths) == 2
+
+
+def test_get_put(tmpdir, m):
+    tmpdir = str(tmpdir)
+    fn = os.path.join(tmpdir, "one")
+    open(fn, "wb").write(b"one")
+    os.mkdir(os.path.join(tmpdir, "dir"))
+    fn2 = os.path.join(tmpdir, "dir", "two")
+    open(fn2, "wb").write(b"two")
+
+    fs = MemoryFileSystem()
+    fs.put(fn, "/afile")
+    assert fs.cat("/afile") == b"one"
+
+    fs.store["/bfile"] = MemoryFile(fs, "/bfile", b"data")
+    fn3 = os.path.join(tmpdir, "three")
+    fs.get("/bfile", fn3)
+    assert open(fn3, "rb").read() == b"data"
+
+    fs.put(tmpdir, "/more", recursive=True)
+    assert fs.find("/more") == ["/more/dir/two", "/more/one", "/more/three"]
+
+    @contextlib.contextmanager
+    def tmp_chdir(path):
+        curdir = os.getcwd()
+        os.chdir(path)
+        try:
+            yield
+        finally:
+            os.chdir(curdir)
+
+    with tmp_chdir(os.path.join(tmpdir, os.path.pardir)):
+        fs.put(os.path.basename(tmpdir), "/moretwo", recursive=True)
+        assert fs.find("/moretwo") == [
+            "/moretwo/dir/two",
+            "/moretwo/one",
+            "/moretwo/three",
+        ]
+
+    with tmp_chdir(tmpdir):
+        fs.put(os.path.curdir, "/morethree", recursive=True)
+        assert fs.find("/morethree") == [
+            "/morethree/dir/two",
+            "/morethree/one",
+            "/morethree/three",
+        ]
+
+    for f in [fn, fn2, fn3]:
+        os.remove(f)
+    os.rmdir(os.path.join(tmpdir, "dir"))
+
+    fs.get("/more/", tmpdir + "/", recursive=True)
+    assert open(fn3, "rb").read() == b"data"
+    assert open(fn, "rb").read() == b"one"
+
+
+def test_du(m):
+    fs = MemoryFileSystem()
+    fs.store.update(
+        {
+            "/dir/afile": MemoryFile(fs, "/afile", b"a"),
+            "/dir/dirb/afile": MemoryFile(fs, "/afile", b"bb"),
+            "/dir/dirb/bfile": MemoryFile(fs, "/afile", b"ccc"),
+        }
+    )
+    assert fs.du("/dir") == 6
+    assert fs.du("/dir", total=False) == {
+        "/dir/afile": 1,
+        "/dir/dirb/afile": 2,
+        "/dir/dirb/bfile": 3,
+    }
+    assert fs.du("/dir", withdirs=True) == 6
+    assert fs.du("/dir", total=False, withdirs=True) == {
+        "/dir": 0,
+        "/dir/afile": 1,
+        "/dir/dirb": 0,
+        "/dir/dirb/afile": 2,
+        "/dir/dirb/bfile": 3,
+    }
+    with pytest.raises(ValueError):
+        assert fs.du("/dir", maxdepth=0) == 1
+    assert fs.du("/dir", total=False, withdirs=True, maxdepth=1) == {
+        "/dir": 0,
+        "/dir/afile": 1,
+        "/dir/dirb": 0,
+    }
+
+    # Size of file only.
+    assert fs.du("/dir/afile") == 1
+    assert fs.du("/dir/afile", withdirs=True) == 1
+
+
+def test_head_tail(m):
+    fs = MemoryFileSystem()
+    with fs.open("/myfile", "wb") as f:
+        f.write(b"I had a nice big cabbage")
+    assert fs.head("/myfile", 5) == b"I had"
+    assert fs.tail("/myfile", 7) == b"cabbage"
+
+
+def test_move(m):
+    fs = MemoryFileSystem()
+    with fs.open("/myfile", "wb") as f:
+        f.write(b"I had a nice big cabbage")
+    fs.move("/myfile", "/otherfile")
+    assert not fs.exists("/myfile")
+    assert fs.info("/otherfile")
+    assert isinstance(fs.ukey("/otherfile"), str)
+
+
+def test_recursive_get_put(tmpdir, m):
+    fs = MemoryFileSystem()
+    os.makedirs(f"{tmpdir}/nest")
+    for file in ["one", "two", "nest/other"]:
+        with open(f"{tmpdir}/{file}", "wb") as f:
+            f.write(b"data")
+
+    fs.put(str(tmpdir), "test", recursive=True)
+
+    # get to directory with slash
+    d = tempfile.mkdtemp()
+    fs.get("test/", d, recursive=True)
+    for file in ["one", "two", "nest/other"]:
+        with open(f"{d}/{file}", "rb") as f:
+            f.read() == b"data"
+
+    # get to directory without slash
+    d = tempfile.mkdtemp()
+    fs.get("test", d, recursive=True)
+    for file in ["test/one", "test/two", "test/nest/other"]:
+        with open(f"{d}/{file}", "rb") as f:
+            f.read() == b"data"
+
+
+def test_pipe_cat(m):
+    fs = MemoryFileSystem()
+    fs.pipe("afile", b"contents")
+    assert fs.cat("afile") == b"contents"
+
+    data = {"/bfile": b"more", "/cfile": b"stuff"}
+    fs.pipe(data)
+    assert fs.cat(list(data)) == data
+
+
+def test_read_block_delimiter(m):
+    fs = MemoryFileSystem()
+    with fs.open("/myfile", "wb") as f:
+        f.write(b"some\nlines\nof\ntext")
+    assert fs.read_block("/myfile", 0, 2, b"\n") == b"some\n"
+    assert fs.read_block("/myfile", 2, 6, b"\n") == b"lines\n"
+    assert fs.read_block("/myfile", 6, 2, b"\n") == b""
+    assert fs.read_block("/myfile", 2, 9, b"\n") == b"lines\nof\n"
+    assert fs.read_block("/myfile", 12, 6, b"\n") == b"text"
+    assert fs.read_block("/myfile", 0, None) == fs.cat("/myfile")
+
+
+def test_open_text(m):
+    fs = MemoryFileSystem()
+    with fs.open("/myfile", "wb") as f:
+        f.write(b"some\nlines\nof\ntext")
+    f = fs.open("/myfile", "r", encoding="latin1")
+    assert f.encoding == "latin1"
+
+
+def test_read_text(m):
+    with m.open("/myfile", "w", encoding="utf-8") as f:
+        f.write("some\nlines\nof\ntext")
+    assert m.read_text("/myfile", encoding="utf-8") == "some\nlines\nof\ntext"
+
+
+def test_write_text(m):
+    m.write_text("/myfile", "some\nlines\nof\ntext", encoding="utf-8")
+    assert m.read_text("/myfile", encoding="utf-8") == "some\nlines\nof\ntext"
+
+
+def test_chained_fs():
+    d1 = tempfile.mkdtemp()
+    d2 = tempfile.mkdtemp()
+    f1 = os.path.join(d1, "f1")
+    with open(f1, "wb") as f:
+        f.write(b"test")
+
+    of = fsspec.open(
+        f"simplecache::file://{f1}",
+        simplecache={"cache_storage": d2, "same_names": True},
+    )
+    with of as f:
+        assert f.read() == b"test"
+
+    assert os.listdir(d2) == ["f1"]
+
+
+@pytest.mark.xfail(reason="see issue #334", strict=True)
+def test_multilevel_chained_fs():
+    """This test reproduces fsspec/filesystem_spec#334"""
+    import zipfile
+
+    d1 = tempfile.mkdtemp()
+    f1 = os.path.join(d1, "f1.zip")
+    with zipfile.ZipFile(f1, mode="w") as z:
+        # filename, content
+        z.writestr("foo.txt", "foo.txt")
+        z.writestr("bar.txt", "bar.txt")
+
+    # We expected this to be the correct syntax
+    with pytest.raises(IsADirectoryError):
+        of = fsspec.open_files(f"zip://*.txt::simplecache::file://{f1}")
+        assert len(of) == 2
+
+    # But this is what is actually valid...
+    of = fsspec.open_files(f"zip://*.txt::simplecache://{f1}::file://")
+
+    assert len(of) == 2
+    for open_file in of:
+        with open_file as f:
+            assert f.read().decode("utf-8") == f.name
+
+
+def test_multilevel_chained_fs_zip_zip_file():
+    """This test reproduces fsspec/filesystem_spec#334"""
+    import zipfile
+
+    d1 = tempfile.mkdtemp()
+    f1 = os.path.join(d1, "f1.zip")
+    f2 = os.path.join(d1, "f2.zip")
+    with zipfile.ZipFile(f1, mode="w") as z:
+        # filename, content
+        z.writestr("foo.txt", "foo.txt")
+        z.writestr("bar.txt", "bar.txt")
+
+    with zipfile.ZipFile(f2, mode="w") as z:
+        with open(f1, "rb") as f:
+            z.writestr("f1.zip", f.read())
+
+    # We expected this to be the correct syntax
+    of = fsspec.open_files(f"zip://*.txt::zip://f1.zip::file://{f2}")
+
+    assert len(of) == 2
+    for open_file in of:
+        with open_file as f:
+            assert f.read().decode("utf-8") == f.name
+
+
+def test_chained_equivalent():
+    d1 = tempfile.mkdtemp()
+    d2 = tempfile.mkdtemp()
+    f1 = os.path.join(d1, "f1")
+    with open(f1, "wb") as f:
+        f.write(b"test1")
+
+    of = fsspec.open(
+        f"simplecache::file://{f1}",
+        simplecache={"cache_storage": d2, "same_names": True},
+    )
+    of2 = fsspec.open(
+        f"simplecache://{f1}",
+        cache_storage=d2,
+        same_names=True,
+        target_protocol="file",
+        target_options={},
+    )
+    # the following line passes by fluke - they are not quite the same instance,
+    #  since the parameters don't quite match. Also, the url understood by the two
+    #  of s are not the same (path gets munged a bit differently)
+    assert of.fs == of2.fs
+    assert hash(of.fs) == hash(of2.fs)
+    assert of.open().read() == of2.open().read()
+
+
+def test_chained_fs_multi():
+    d1 = tempfile.mkdtemp()
+    d2 = tempfile.mkdtemp()
+    f1 = os.path.join(d1, "f1")
+    f2 = os.path.join(d1, "f2")
+    with open(f1, "wb") as f:
+        f.write(b"test1")
+    with open(f2, "wb") as f:
+        f.write(b"test2")
+
+    of = fsspec.open_files(
+        f"simplecache::file://{d1}/*",
+        simplecache={"cache_storage": d2, "same_names": True},
+    )
+    with of[0] as f:
+        assert f.read() == b"test1"
+    with of[1] as f:
+        assert f.read() == b"test2"
+
+    assert sorted(os.listdir(d2)) == ["f1", "f2"]
+
+    d2 = tempfile.mkdtemp()
+
+    of = fsspec.open_files(
+        [f"simplecache::file://{f1}", f"simplecache::file://{f2}"],
+        simplecache={"cache_storage": d2, "same_names": True},
+    )
+    with of[0] as f:
+        assert f.read() == b"test1"
+    with of[1] as f:
+        assert f.read() == b"test2"
+
+    assert sorted(os.listdir(d2)) == ["f1", "f2"]
+
+
+def test_chained_fo():
+    import zipfile
+
+    d1 = tempfile.mkdtemp()
+    f1 = os.path.join(d1, "temp.zip")
+    d3 = tempfile.mkdtemp()
+    with zipfile.ZipFile(f1, mode="w") as z:
+        z.writestr("afile", b"test")
+
+    of = fsspec.open(f"zip://afile::file://{f1}")
+    with of as f:
+        assert f.read() == b"test"
+
+    of = fsspec.open_files(f"zip://*::file://{f1}")
+    with of[0] as f:
+        assert f.read() == b"test"
+
+    of = fsspec.open_files(
+        f"simplecache::zip://*::file://{f1}",
+        simplecache={"cache_storage": d3, "same_names": True},
+    )
+    with of[0] as f:
+        assert f.read() == b"test"
+    assert "afile" in os.listdir(d3)
+
+
+def test_url_to_fs():
+    url = "memory://a.txt"
+    fs, url2 = fsspec.core.url_to_fs(url)
+
+    assert isinstance(fs, MemoryFileSystem)
+    assert url2 == "/a.txt"
+
+
+def test_walk(m):
+    # depth = 0
+    dir1 = "/dir1"
+    # depth = 1 (2 dirs, 1 file)
+    dir11 = dir1 + "/dir11"
+    dir12 = dir1 + "/dir12"
+    file11 = dir1 + "/file11"
+    # depth = 2
+    dir111 = dir11 + "/dir111"
+    file111 = dir11 + "/file111"
+    file121 = dir12 + "/file121"
+    # depth = 3
+    file1111 = dir111 + "/file1111"
+
+    m.mkdir(dir111)  # Creates parents too
+    m.mkdir(dir12)  # Creates parents too
+    m.touch(file11)
+    m.touch(file111)
+    m.touch(file121)
+    m.touch(file1111)
+
+    # No maxdepth
+    assert list(m.walk(dir1, topdown=True)) == [
+        (dir1, ["dir11", "dir12"], ["file11"]),
+        (dir11, ["dir111"], ["file111"]),
+        (dir111, [], ["file1111"]),
+        (dir12, [], ["file121"]),
+    ]
+    assert list(m.walk(dir1, topdown=False)) == [
+        (dir111, [], ["file1111"]),
+        (dir11, ["dir111"], ["file111"]),
+        (dir12, [], ["file121"]),
+        (dir1, ["dir11", "dir12"], ["file11"]),
+    ]
+
+    # maxdepth=2
+    assert list(m.walk(dir1, maxdepth=2, topdown=True)) == [
+        (dir1, ["dir11", "dir12"], ["file11"]),
+        (dir11, ["dir111"], ["file111"]),
+        (dir12, [], ["file121"]),
+    ]
+    assert list(m.walk(dir1, maxdepth=2, topdown=False)) == [
+        (dir11, ["dir111"], ["file111"]),
+        (dir12, [], ["file121"]),
+        (dir1, ["dir11", "dir12"], ["file11"]),
+    ]
+
+    # maxdepth=1
+    assert list(m.walk(dir1, maxdepth=1, topdown=True)) == [
+        (dir1, ["dir11", "dir12"], ["file11"]),
+    ]
+    assert list(m.walk(dir1, maxdepth=1, topdown=False)) == [
+        (dir1, ["dir11", "dir12"], ["file11"]),
+    ]
+
+    # maxdepth=0
+    with pytest.raises(ValueError):
+        list(m.walk(dir1, maxdepth=0, topdown=True))
+    with pytest.raises(ValueError):
+        list(m.walk(dir1, maxdepth=0, topdown=False))
+
+    # prune dir111
+    def _walk(*args, **kwargs):
+        for path, dirs, files in m.walk(*args, **kwargs):
+            yield (path, dirs.copy(), files)
+            if "dir111" in dirs:
+                dirs.remove("dir111")
+
+    assert list(_walk(dir1, topdown=True)) == [
+        (dir1, ["dir11", "dir12"], ["file11"]),
+        (dir11, ["dir111"], ["file111"]),
+        (dir12, [], ["file121"]),
+    ]
+    assert list(_walk(dir1, topdown=False)) == [
+        (dir111, [], ["file1111"]),
+        (dir11, ["dir111"], ["file111"]),
+        (dir12, [], ["file121"]),
+        (dir1, ["dir11", "dir12"], ["file11"]),
+    ]
+
+    # reverse dirs order
+    def _walk(*args, **kwargs):
+        for path, dirs, files in m.walk(*args, **kwargs):
+            yield (path, dirs.copy(), files)
+            dirs.reverse()
+
+    assert list(_walk(dir1, topdown=True)) == [
+        (dir1, ["dir11", "dir12"], ["file11"]),
+        # Here dir12 comes before dir11
+        (dir12, [], ["file121"]),
+        (dir11, ["dir111"], ["file111"]),
+        (dir111, [], ["file1111"]),
+    ]
+    assert list(_walk(dir1, topdown=False)) == [
+        (dir111, [], ["file1111"]),
+        (dir11, ["dir111"], ["file111"]),
+        (dir12, [], ["file121"]),
+        (dir1, ["dir11", "dir12"], ["file11"]),
+    ]
+
+    # on_error omit by default
+    assert list(m.walk("do_not_exist")) == []
+    # on_error omit
+    assert list(m.walk("do_not_exist", on_error="omit")) == []
+    # on_error raise
+    with pytest.raises(FileNotFoundError):
+        list(m.walk("do_not_exist", on_error="raise"))
+    # on_error callable function
+    mock = Mock()
+    assert list(m.walk("do_not_exist", on_error=mock.onerror)) == []
+    mock.onerror.assert_called()
+    assert mock.onerror.call_args.kwargs == {}
+    assert len(mock.onerror.call_args.args) == 1
+    assert isinstance(mock.onerror.call_args.args[0], FileNotFoundError)