fsspec 2023.6.0__py3-none-any.whl → 2023.9.1__py3-none-any.whl

fsspec/tests/abstract/get.py

@@ -1,3 +1,12 @@
+from hashlib import md5
+from itertools import product
+
+import pytest
+
+from fsspec.implementations.local import make_path_posix
+from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS
+
+
 class AbstractGetTests:
     def test_get_file_to_existing_directory(
         self,
@@ -66,7 +75,6 @@ class AbstractGetTests:
         self,
         fs,
         fs_join,
-        fs_path,
         fs_bulk_operations_scenario_0,
         local_fs,
         local_join,
@@ -117,6 +125,7 @@ class AbstractGetTests:
 
         target = local_target
         local_fs.mkdir(target)
+        assert local_fs.isdir(target)
 
         for source_slash, target_slash in zip([False, True], [False, True]):
             s = fs_join(source, "subdir")
@@ -125,9 +134,8 @@ class AbstractGetTests:
             t = target + "/" if target_slash else target
 
             # Without recursive does nothing
-            # ERROR: erroneously creates new directory
-            # fs.get(s, t)
-            # assert fs.ls(target) == []
+            fs.get(s, t)
+            assert local_fs.ls(target) == []
 
             # With recursive
             fs.get(s, t, recursive=True)
@@ -136,6 +144,7 @@ class AbstractGetTests:
                 assert local_fs.isfile(local_join(target, "subfile2"))
                 assert local_fs.isdir(local_join(target, "nesteddir"))
                 assert local_fs.isfile(local_join(target, "nesteddir", "nestedfile"))
+                assert not local_fs.exists(local_join(target, "subdir"))
 
                 local_fs.rm(
                     [
@@ -157,8 +166,29 @@ class AbstractGetTests:
                 local_fs.rm(local_join(target, "subdir"), recursive=True)
             assert local_fs.ls(target) == []
 
-            # Limit by maxdepth
-            # ERROR: maxdepth ignored here
+            # Limit recursive by maxdepth
+            fs.get(s, t, recursive=True, maxdepth=1)
+            if source_slash:
+                assert local_fs.isfile(local_join(target, "subfile1"))
+                assert local_fs.isfile(local_join(target, "subfile2"))
+                assert not local_fs.exists(local_join(target, "nesteddir"))
+                assert not local_fs.exists(local_join(target, "subdir"))
+
+                local_fs.rm(
+                    [
+                        local_join(target, "subfile1"),
+                        local_join(target, "subfile2"),
+                    ],
+                    recursive=True,
+                )
+            else:
+                assert local_fs.isdir(local_join(target, "subdir"))
+                assert local_fs.isfile(local_join(target, "subdir", "subfile1"))
+                assert local_fs.isfile(local_join(target, "subdir", "subfile2"))
+                assert not local_fs.exists(local_join(target, "subdir", "nesteddir"))
+
+                local_fs.rm(local_join(target, "subdir"), recursive=True)
+            assert local_fs.ls(target) == []
 
     def test_get_directory_to_new_directory(
         self,
@@ -184,9 +214,8 @@ class AbstractGetTests:
                 t += "/"
 
             # Without recursive does nothing
-            # ERROR: erroneously creates new directory
-            # fs.get(s, t)
-            # assert fs.ls(target) == []
+            fs.get(s, t)
+            assert local_fs.ls(target) == []
 
             # With recursive
             fs.get(s, t, recursive=True)
@@ -197,12 +226,21 @@ class AbstractGetTests:
             assert local_fs.isfile(
                 local_join(target, "newdir", "nesteddir", "nestedfile")
             )
+            assert not local_fs.exists(local_join(target, "subdir"))
 
             local_fs.rm(local_join(target, "newdir"), recursive=True)
             assert local_fs.ls(target) == []
 
-            # Limit by maxdepth
-            # ERROR: maxdepth ignored here
+            # Limit recursive by maxdepth
+            fs.get(s, t, recursive=True, maxdepth=1)
+            assert local_fs.isdir(local_join(target, "newdir"))
+            assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
+            assert local_fs.isfile(local_join(target, "newdir", "subfile2"))
+            assert not local_fs.exists(local_join(target, "newdir", "nesteddir"))
+            assert not local_fs.exists(local_join(target, "subdir"))
+
+            local_fs.rm(local_join(target, "newdir"), recursive=True)
+            assert not local_fs.exists(local_join(target, "newdir"))
 
     def test_get_glob_to_existing_directory(
         self,
@@ -219,20 +257,62 @@ class AbstractGetTests:
         target = local_target
         local_fs.mkdir(target)
 
-        # for target_slash in [False, True]:
-        for target_slash in [False]:
+        for target_slash in [False, True]:
             t = target + "/" if target_slash else target
 
             # Without recursive
             fs.get(fs_join(source, "subdir", "*"), t)
             assert local_fs.isfile(local_join(target, "subfile1"))
             assert local_fs.isfile(local_join(target, "subfile2"))
-            # assert not local_fs.isdir(local_join(target, "nesteddir"))  # ERROR
-            assert not local_fs.isdir(local_join(target, "subdir"))
+            assert not local_fs.isdir(local_join(target, "nesteddir"))
+            assert not local_fs.exists(local_join(target, "nesteddir", "nestedfile"))
+            assert not local_fs.exists(local_join(target, "subdir"))
+
+            local_fs.rm(
+                [
+                    local_join(target, "subfile1"),
+                    local_join(target, "subfile2"),
+                ],
+                recursive=True,
+            )
+            assert local_fs.ls(target) == []
 
             # With recursive
+            for glob, recursive in zip(["*", "**"], [True, False]):
+                fs.get(fs_join(source, "subdir", glob), t, recursive=recursive)
+                assert local_fs.isfile(local_join(target, "subfile1"))
+                assert local_fs.isfile(local_join(target, "subfile2"))
+                assert local_fs.isdir(local_join(target, "nesteddir"))
+                assert local_fs.isfile(local_join(target, "nesteddir", "nestedfile"))
+                assert not local_fs.exists(local_join(target, "subdir"))
+
+                local_fs.rm(
+                    [
+                        local_join(target, "subfile1"),
+                        local_join(target, "subfile2"),
+                        local_join(target, "nesteddir"),
+                    ],
+                    recursive=True,
+                )
+                assert local_fs.ls(target) == []
 
-            # Limit by maxdepth
+                # Limit recursive by maxdepth
+                fs.get(
+                    fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1
+                )
+                assert local_fs.isfile(local_join(target, "subfile1"))
+                assert local_fs.isfile(local_join(target, "subfile2"))
+                assert not local_fs.exists(local_join(target, "nesteddir"))
+                assert not local_fs.exists(local_join(target, "subdir"))
+
+                local_fs.rm(
+                    [
+                        local_join(target, "subfile1"),
+                        local_join(target, "subfile2"),
+                    ],
+                    recursive=True,
+                )
+                assert local_fs.ls(target) == []
 
     def test_get_glob_to_new_directory(
         self,
@@ -259,27 +339,91 @@ class AbstractGetTests:
             assert local_fs.isdir(local_join(target, "newdir"))
             assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
             assert local_fs.isfile(local_join(target, "newdir", "subfile2"))
-            # ERROR - do not copy empty directory
-            # assert not local_fs.exists(local_join(target, "newdir", "nesteddir"))
+            assert not local_fs.exists(local_join(target, "newdir", "nesteddir"))
+            assert not local_fs.exists(
+                local_join(target, "newdir", "nesteddir", "nestedfile")
+            )
+            assert not local_fs.exists(local_join(target, "subdir"))
+            assert not local_fs.exists(local_join(target, "newdir", "subdir"))
 
             local_fs.rm(local_join(target, "newdir"), recursive=True)
             assert local_fs.ls(target) == []
 
             # With recursive
-            fs.get(fs_join(source, "subdir", "*"), t, recursive=True)
-            assert local_fs.isdir(local_join(target, "newdir"))
-            assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
-            assert local_fs.isfile(local_join(target, "newdir", "subfile2"))
-            assert local_fs.isdir(local_join(target, "newdir", "nesteddir"))
-            assert local_fs.isfile(
-                local_join(target, "newdir", "nesteddir", "nestedfile")
-            )
+            for glob, recursive in zip(["*", "**"], [True, False]):
+                fs.get(fs_join(source, "subdir", glob), t, recursive=recursive)
+                assert local_fs.isdir(local_join(target, "newdir"))
+                assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
+                assert local_fs.isfile(local_join(target, "newdir", "subfile2"))
+                assert local_fs.isdir(local_join(target, "newdir", "nesteddir"))
+                assert local_fs.isfile(
+                    local_join(target, "newdir", "nesteddir", "nestedfile")
+                )
+                assert not local_fs.exists(local_join(target, "subdir"))
+                assert not local_fs.exists(local_join(target, "newdir", "subdir"))
 
-            local_fs.rm(local_join(target, "newdir"), recursive=True)
-            assert local_fs.ls(target) == []
+                local_fs.rm(local_join(target, "newdir"), recursive=True)
+                assert not local_fs.exists(local_join(target, "newdir"))
+
+                # Limit recursive by maxdepth
+                fs.get(
+                    fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1
+                )
+                assert local_fs.isdir(local_join(target, "newdir"))
+                assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
+                assert local_fs.isfile(local_join(target, "newdir", "subfile2"))
+                assert not local_fs.exists(local_join(target, "newdir", "nesteddir"))
+                assert not local_fs.exists(local_join(target, "subdir"))
+                assert not local_fs.exists(local_join(target, "newdir", "subdir"))
+
+                local_fs.rm(local_fs.ls(target, detail=False), recursive=True)
+                assert not local_fs.exists(local_join(target, "newdir"))
+
+    @pytest.mark.parametrize(
+        GLOB_EDGE_CASES_TESTS["argnames"],
+        GLOB_EDGE_CASES_TESTS["argvalues"],
+    )
+    def test_get_glob_edge_cases(
+        self,
+        path,
+        recursive,
+        maxdepth,
+        expected,
+        fs,
+        fs_join,
+        fs_glob_edge_cases_files,
+        local_fs,
+        local_join,
+        local_target,
+    ):
+        # Copy scenario 1g
+        source = fs_glob_edge_cases_files
+
+        target = local_target
+
+        for new_dir, target_slash in product([True, False], [True, False]):
+            local_fs.mkdir(target)
 
-            # Limit by maxdepth
-            # ERROR: this is not correct
+            t = local_join(target, "newdir") if new_dir else target
+            t = t + "/" if target_slash else t
+
+            fs.get(fs_join(source, path), t, recursive=recursive, maxdepth=maxdepth)
+
+            output = local_fs.find(target)
+            if new_dir:
+                prefixed_expected = [
+                    make_path_posix(local_join(target, "newdir", p)) for p in expected
+                ]
+            else:
+                prefixed_expected = [
+                    make_path_posix(local_join(target, p)) for p in expected
+                ]
+            assert sorted(output) == sorted(prefixed_expected)
+
+            try:
+                local_fs.rm(target, recursive=True)
+            except FileNotFoundError:
+                pass
 
     def test_get_list_of_files_to_existing_directory(
         self,
@@ -310,7 +454,14 @@ class AbstractGetTests:
             assert local_fs.isfile(local_join(target, "file2"))
             assert local_fs.isfile(local_join(target, "subfile1"))
 
-            local_fs.rm(local_fs.find(target))
+            local_fs.rm(
+                [
+                    local_join(target, "file1"),
+                    local_join(target, "file2"),
+                    local_join(target, "subfile1"),
+                ],
+                recursive=True,
+            )
             assert local_fs.ls(target) == []
 
     def test_get_list_of_files_to_new_directory(
@@ -358,13 +509,13 @@ class AbstractGetTests:
             fs.get(src, target, recursive=True)
             assert local_fs.isdir(target)
 
-        if loop == 0:
-            assert local_fs.isfile(local_join(target, "file"))
-            assert not local_fs.exists(local_join(target, "src"))
-        else:
-            assert local_fs.isfile(local_join(target, "file"))
-            assert local_fs.isdir(local_join(target, "src"))
-            assert local_fs.isfile(local_join(target, "src", "file"))
+            if loop == 0:
+                assert local_fs.isfile(local_join(target, "file"))
+                assert not local_fs.exists(local_join(target, "src"))
+            else:
+                assert local_fs.isfile(local_join(target, "file"))
+                assert local_fs.isdir(local_join(target, "src"))
+                assert local_fs.isfile(local_join(target, "src", "file"))
 
         local_fs.rm(target, recursive=True)
 
@@ -375,3 +526,62 @@ class AbstractGetTests:
             assert local_fs.isdir(target)
             assert local_fs.isfile(local_join(target, "file"))
             assert not local_fs.exists(local_join(target, "src"))
+
+    def test_get_directory_without_files_with_same_name_prefix(
+        self,
+        fs,
+        fs_join,
+        local_fs,
+        local_join,
+        local_target,
+        fs_dir_and_file_with_same_name_prefix,
+    ):
+        # Create the test dirs
+        source = fs_dir_and_file_with_same_name_prefix
+        target = local_target
+
+        # Test without glob
+        fs.get(fs_join(source, "subdir"), target, recursive=True)
+
+        assert local_fs.isfile(local_join(target, "subfile.txt"))
+        assert not local_fs.isfile(local_join(target, "subdir.txt"))
+
+        local_fs.rm([local_join(target, "subfile.txt")])
+        assert local_fs.ls(target) == []
+
+        # Test with glob
+        fs.get(fs_join(source, "subdir*"), target, recursive=True)
+
+        assert local_fs.isdir(local_join(target, "subdir"))
+        assert local_fs.isfile(local_join(target, "subdir", "subfile.txt"))
+        assert local_fs.isfile(local_join(target, "subdir.txt"))
+
+    def test_get_with_source_and_destination_as_list(
+        self,
+        fs,
+        fs_join,
+        local_fs,
+        local_join,
+        local_target,
+        fs_10_files_with_hashed_names,
+    ):
+        # Create the test dir
+        source = fs_10_files_with_hashed_names
+        target = local_target
+
+        # Create list of files for source and destination
+        source_files = []
+        destination_files = []
+        for i in range(10):
+            hashed_i = md5(str(i).encode("utf-8")).hexdigest()
+            source_files.append(fs_join(source, f"{hashed_i}.txt"))
+            destination_files.append(
+                make_path_posix(local_join(target, f"{hashed_i}.txt"))
+            )
+
+        # Copy and assert order was kept
+        fs.get(rpath=source_files, lpath=destination_files)
+
+        for i in range(10):
+            file_content = local_fs.cat(destination_files[i]).decode("utf-8")
+            assert file_content == str(i)