fsspec 2023.6.0__py3-none-any.whl → 2023.9.1__py3-none-any.whl

fsspec/tests/abstract/common.py

@@ -0,0 +1,175 @@
+GLOB_EDGE_CASES_TESTS = {
+    "argnames": ("path", "recursive", "maxdepth", "expected"),
+    "argvalues": [
+        ("fil?1", False, None, ["file1"]),
+        ("fil?1", True, None, ["file1"]),
+        ("file[1-2]", False, None, ["file1", "file2"]),
+        ("file[1-2]", True, None, ["file1", "file2"]),
+        ("*", False, None, ["file1", "file2"]),
+        (
+            "*",
+            True,
+            None,
+            [
+                "file1",
+                "file2",
+                "subdir0/subfile1",
+                "subdir0/subfile2",
+                "subdir0/nesteddir/nestedfile",
+                "subdir1/subfile1",
+                "subdir1/subfile2",
+                "subdir1/nesteddir/nestedfile",
+            ],
+        ),
+        ("*", True, 1, ["file1", "file2"]),
+        (
+            "*",
+            True,
+            2,
+            [
+                "file1",
+                "file2",
+                "subdir0/subfile1",
+                "subdir0/subfile2",
+                "subdir1/subfile1",
+                "subdir1/subfile2",
+            ],
+        ),
+        ("*1", False, None, ["file1"]),
+        (
+            "*1",
+            True,
+            None,
+            [
+                "file1",
+                "subdir1/subfile1",
+                "subdir1/subfile2",
+                "subdir1/nesteddir/nestedfile",
+            ],
+        ),
+        ("*1", True, 2, ["file1", "subdir1/subfile1", "subdir1/subfile2"]),
+        (
+            "**",
+            False,
+            None,
+            [
+                "file1",
+                "file2",
+                "subdir0/subfile1",
+                "subdir0/subfile2",
+                "subdir0/nesteddir/nestedfile",
+                "subdir1/subfile1",
+                "subdir1/subfile2",
+                "subdir1/nesteddir/nestedfile",
+            ],
+        ),
+        (
+            "**",
+            True,
+            None,
+            [
+                "file1",
+                "file2",
+                "subdir0/subfile1",
+                "subdir0/subfile2",
+                "subdir0/nesteddir/nestedfile",
+                "subdir1/subfile1",
+                "subdir1/subfile2",
+                "subdir1/nesteddir/nestedfile",
+            ],
+        ),
+        ("**", True, 1, ["file1", "file2"]),
+        (
+            "**",
+            True,
+            2,
+            [
+                "file1",
+                "file2",
+                "subdir0/subfile1",
+                "subdir0/subfile2",
+                "subdir0/nesteddir/nestedfile",
+                "subdir1/subfile1",
+                "subdir1/subfile2",
+                "subdir1/nesteddir/nestedfile",
+            ],
+        ),
+        (
+            "**",
+            False,
+            2,
+            [
+                "file1",
+                "file2",
+                "subdir0/subfile1",
+                "subdir0/subfile2",
+                "subdir1/subfile1",
+                "subdir1/subfile2",
+            ],
+        ),
+        ("**1", False, None, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
+        (
+            "**1",
+            True,
+            None,
+            [
+                "file1",
+                "subdir0/subfile1",
+                "subdir1/subfile1",
+                "subdir1/subfile2",
+                "subdir1/nesteddir/nestedfile",
+            ],
+        ),
+        ("**1", True, 1, ["file1"]),
+        (
+            "**1",
+            True,
+            2,
+            ["file1", "subdir0/subfile1", "subdir1/subfile1", "subdir1/subfile2"],
+        ),
+        ("**1", False, 2, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
+        ("**/subdir0", False, None, []),
+        ("**/subdir0", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]),
+        ("**/subdir0/nested*", False, 2, []),
+        ("**/subdir0/nested*", True, 2, ["nestedfile"]),
+        ("subdir[1-2]", False, None, []),
+        ("subdir[1-2]", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]),
+        ("subdir[1-2]", True, 2, ["subfile1", "subfile2"]),
+        ("subdir[0-1]", False, None, []),
+        (
+            "subdir[0-1]",
+            True,
+            None,
+            [
+                "subdir0/subfile1",
+                "subdir0/subfile2",
+                "subdir0/nesteddir/nestedfile",
+                "subdir1/subfile1",
+                "subdir1/subfile2",
+                "subdir1/nesteddir/nestedfile",
+            ],
+        ),
+        (
+            "subdir[0-1]/*fil[e]*",
+            False,
+            None,
+            [
+                "subdir0/subfile1",
+                "subdir0/subfile2",
+                "subdir1/subfile1",
+                "subdir1/subfile2",
+            ],
+        ),
+        (
+            "subdir[0-1]/*fil[e]*",
+            True,
+            None,
+            [
+                "subdir0/subfile1",
+                "subdir0/subfile2",
+                "subdir1/subfile1",
+                "subdir1/subfile2",
+            ],
+        ),
+    ],
+}

fsspec/tests/abstract/copy.py

@@ -1,13 +1,26 @@
+from hashlib import md5
+from itertools import product
+
+import pytest
+
+from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS
+
+
 class AbstractCopyTests:
     def test_copy_file_to_existing_directory(
-        self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
+        self,
+        fs,
+        fs_join,
+        fs_bulk_operations_scenario_0,
+        fs_target,
+        supports_empty_directories,
     ):
         # Copy scenario 1a
         source = fs_bulk_operations_scenario_0
 
         target = fs_target
         fs.mkdir(target)
-        if not self.supports_empty_directories():
+        if not supports_empty_directories:
             # Force target directory to exist by adding a dummy file
             fs.touch(fs_join(target, "dummy"))
         assert fs.isdir(target)
@@ -53,13 +66,22 @@ class AbstractCopyTests:
         assert fs.isfile(fs_join(target, "newdir", "subfile1"))
 
     def test_copy_file_to_file_in_existing_directory(
-        self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
+        self,
+        fs,
+        fs_join,
+        fs_bulk_operations_scenario_0,
+        fs_target,
+        supports_empty_directories,
     ):
         # Copy scenario 1c
         source = fs_bulk_operations_scenario_0
 
         target = fs_target
         fs.mkdir(target)
+        if not supports_empty_directories:
+            # Force target directory to exist by adding a dummy file
+            fs.touch(fs_join(target, "dummy"))
+        assert fs.isdir(target)
 
         fs.cp(fs_join(source, "subdir", "subfile1"), fs_join(target, "newfile"))
         assert fs.isfile(fs_join(target, "newfile"))
@@ -80,14 +102,19 @@ class AbstractCopyTests:
         assert fs.isfile(fs_join(target, "newdir", "newfile"))
 
     def test_copy_directory_to_existing_directory(
-        self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
+        self,
+        fs,
+        fs_join,
+        fs_bulk_operations_scenario_0,
+        fs_target,
+        supports_empty_directories,
     ):
         # Copy scenario 1e
         source = fs_bulk_operations_scenario_0
 
         target = fs_target
         fs.mkdir(target)
-        if not self.supports_empty_directories():
+        if not supports_empty_directories:
             # Force target directory to exist by adding a dummy file
             dummy = fs_join(target, "dummy")
             fs.touch(dummy)
@@ -101,7 +128,7 @@ class AbstractCopyTests:
 
             # Without recursive does nothing
             fs.cp(s, t)
-            assert fs.ls(target) == [] if self.supports_empty_directories() else [dummy]
+            assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
 
             # With recursive
             fs.cp(s, t, recursive=True)
@@ -112,7 +139,14 @@ class AbstractCopyTests:
                 assert fs.isfile(fs_join(target, "nesteddir", "nestedfile"))
                 assert not fs.exists(fs_join(target, "subdir"))
 
-                fs.rm(fs.ls(target, detail=False), recursive=True)
+                fs.rm(
+                    [
+                        fs_join(target, "subfile1"),
+                        fs_join(target, "subfile2"),
+                        fs_join(target, "nesteddir"),
+                    ],
+                    recursive=True,
+                )
             else:
                 assert fs.isdir(fs_join(target, "subdir"))
                 assert fs.isfile(fs_join(target, "subdir", "subfile1"))
@@ -121,7 +155,7 @@ class AbstractCopyTests:
                 assert fs.isfile(fs_join(target, "subdir", "nesteddir", "nestedfile"))
 
                 fs.rm(fs_join(target, "subdir"), recursive=True)
-            assert fs.ls(target) == [] if self.supports_empty_directories() else [dummy]
+            assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
 
             # Limit recursive by maxdepth
             fs.cp(s, t, recursive=True, maxdepth=1)
@@ -131,7 +165,13 @@ class AbstractCopyTests:
                 assert not fs.exists(fs_join(target, "nesteddir"))
                 assert not fs.exists(fs_join(target, "subdir"))
 
-                fs.rm(fs.ls(target, detail=False), recursive=True)
+                fs.rm(
+                    [
+                        fs_join(target, "subfile1"),
+                        fs_join(target, "subfile2"),
+                    ],
+                    recursive=True,
+                )
             else:
                 assert fs.isdir(fs_join(target, "subdir"))
                 assert fs.isfile(fs_join(target, "subdir", "subfile1"))
@@ -139,10 +179,15 @@ class AbstractCopyTests:
                 assert not fs.exists(fs_join(target, "subdir", "nesteddir"))
 
                 fs.rm(fs_join(target, "subdir"), recursive=True)
-            assert fs.ls(target) == [] if self.supports_empty_directories() else [dummy]
+            assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
 
     def test_copy_directory_to_new_directory(
-        self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
+        self,
+        fs,
+        fs_join,
+        fs_bulk_operations_scenario_0,
+        fs_target,
+        supports_empty_directories,
     ):
         # Copy scenario 1f
         source = fs_bulk_operations_scenario_0
@@ -160,7 +205,11 @@ class AbstractCopyTests:
 
             # Without recursive does nothing
             fs.cp(s, t)
-            assert fs.ls(target) == []
+            if supports_empty_directories:
+                assert fs.ls(target) == []
+            else:
+                with pytest.raises(FileNotFoundError):
+                    fs.ls(target)
 
             # With recursive
             fs.cp(s, t, recursive=True)
@@ -186,13 +235,23 @@ class AbstractCopyTests:
             assert not fs.exists(fs_join(target, "newdir"))
 
     def test_copy_glob_to_existing_directory(
-        self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
+        self,
+        fs,
+        fs_join,
+        fs_bulk_operations_scenario_0,
+        fs_target,
+        supports_empty_directories,
     ):
         # Copy scenario 1g
         source = fs_bulk_operations_scenario_0
 
         target = fs_target
         fs.mkdir(target)
+        if not supports_empty_directories:
+            # Force target directory to exist by adding a dummy file
+            dummy = fs_join(target, "dummy")
+            fs.touch(dummy)
+        assert fs.isdir(target)
 
         for target_slash in [False, True]:
             t = target + "/" if target_slash else target
@@ -205,29 +264,51 @@ class AbstractCopyTests:
             assert not fs.exists(fs_join(target, "nesteddir", "nestedfile"))
             assert not fs.exists(fs_join(target, "subdir"))
 
-            fs.rm(fs.ls(target, detail=False), recursive=True)
-            assert fs.ls(target) == []
+            fs.rm(
+                [
+                    fs_join(target, "subfile1"),
+                    fs_join(target, "subfile2"),
+                ],
+                recursive=True,
+            )
+            assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
 
             # With recursive
-            fs.cp(fs_join(source, "subdir", "*"), t, recursive=True)
-            assert fs.isfile(fs_join(target, "subfile1"))
-            assert fs.isfile(fs_join(target, "subfile2"))
-            assert fs.isdir(fs_join(target, "nesteddir"))
-            assert fs.isfile(fs_join(target, "nesteddir", "nestedfile"))
-            assert not fs.exists(fs_join(target, "subdir"))
-
-            fs.rm(fs.ls(target, detail=False), recursive=True)
-            assert fs.ls(target) == []
+            for glob, recursive in zip(["*", "**"], [True, False]):
+                fs.cp(fs_join(source, "subdir", glob), t, recursive=recursive)
+                assert fs.isfile(fs_join(target, "subfile1"))
+                assert fs.isfile(fs_join(target, "subfile2"))
+                assert fs.isdir(fs_join(target, "nesteddir"))
+                assert fs.isfile(fs_join(target, "nesteddir", "nestedfile"))
+                assert not fs.exists(fs_join(target, "subdir"))
 
-            # Limit recursive by maxdepth
-            fs.cp(fs_join(source, "subdir", "*"), t, recursive=True, maxdepth=1)
-            assert fs.isfile(fs_join(target, "subfile1"))
-            assert fs.isfile(fs_join(target, "subfile2"))
-            assert not fs.exists(fs_join(target, "nesteddir"))
-            assert not fs.exists(fs_join(target, "subdir"))
+                fs.rm(
+                    [
+                        fs_join(target, "subfile1"),
+                        fs_join(target, "subfile2"),
+                        fs_join(target, "nesteddir"),
+                    ],
+                    recursive=True,
+                )
+                assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
+
+                # Limit recursive by maxdepth
+                fs.cp(
+                    fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1
+                )
+                assert fs.isfile(fs_join(target, "subfile1"))
+                assert fs.isfile(fs_join(target, "subfile2"))
+                assert not fs.exists(fs_join(target, "nesteddir"))
+                assert not fs.exists(fs_join(target, "subdir"))
 
-            fs.rm(fs.ls(target, detail=False), recursive=True)
-            assert fs.ls(target) == []
+                fs.rm(
+                    [
+                        fs_join(target, "subfile1"),
+                        fs_join(target, "subfile2"),
+                    ],
+                    recursive=True,
+                )
+                assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
 
     def test_copy_glob_to_new_directory(
         self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
@@ -257,39 +338,92 @@ class AbstractCopyTests:
             assert not fs.exists(fs_join(target, "newdir"))
 
             # With recursive
-            fs.cp(fs_join(source, "subdir", "*"), t, recursive=True)
-            assert fs.isdir(fs_join(target, "newdir"))
-            assert fs.isfile(fs_join(target, "newdir", "subfile1"))
-            assert fs.isfile(fs_join(target, "newdir", "subfile2"))
-            assert fs.isdir(fs_join(target, "newdir", "nesteddir"))
-            assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile"))
-            assert not fs.exists(fs_join(target, "subdir"))
-            assert not fs.exists(fs_join(target, "newdir", "subdir"))
+            for glob, recursive in zip(["*", "**"], [True, False]):
+                fs.cp(fs_join(source, "subdir", glob), t, recursive=recursive)
+                assert fs.isdir(fs_join(target, "newdir"))
+                assert fs.isfile(fs_join(target, "newdir", "subfile1"))
+                assert fs.isfile(fs_join(target, "newdir", "subfile2"))
+                assert fs.isdir(fs_join(target, "newdir", "nesteddir"))
+                assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile"))
+                assert not fs.exists(fs_join(target, "subdir"))
+                assert not fs.exists(fs_join(target, "newdir", "subdir"))
+
+                fs.rm(fs_join(target, "newdir"), recursive=True)
+                assert not fs.exists(fs_join(target, "newdir"))
+
+                # Limit recursive by maxdepth
+                fs.cp(
+                    fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1
+                )
+                assert fs.isdir(fs_join(target, "newdir"))
+                assert fs.isfile(fs_join(target, "newdir", "subfile1"))
+                assert fs.isfile(fs_join(target, "newdir", "subfile2"))
+                assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
+                assert not fs.exists(fs_join(target, "subdir"))
+                assert not fs.exists(fs_join(target, "newdir", "subdir"))
+
+                fs.rm(fs_join(target, "newdir"), recursive=True)
+                assert not fs.exists(fs_join(target, "newdir"))
+
+    @pytest.mark.parametrize(
+        GLOB_EDGE_CASES_TESTS["argnames"],
+        GLOB_EDGE_CASES_TESTS["argvalues"],
+    )
+    def test_copy_glob_edge_cases(
+        self,
+        path,
+        recursive,
+        maxdepth,
+        expected,
+        fs,
+        fs_join,
+        fs_glob_edge_cases_files,
+        fs_target,
+        fs_sanitize_path,
+    ):
+        # Copy scenario 1g
+        source = fs_glob_edge_cases_files
 
-            fs.rm(fs_join(target, "newdir"), recursive=True)
-            assert not fs.exists(fs_join(target, "newdir"))
+        target = fs_target
 
-            # Limit recursive by maxdepth
-            fs.cp(fs_join(source, "subdir", "*"), t, recursive=True, maxdepth=1)
-            assert fs.isdir(fs_join(target, "newdir"))
-            assert fs.isfile(fs_join(target, "newdir", "subfile1"))
-            assert fs.isfile(fs_join(target, "newdir", "subfile2"))
-            assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
-            assert not fs.exists(fs_join(target, "subdir"))
-            assert not fs.exists(fs_join(target, "newdir", "subdir"))
+        for new_dir, target_slash in product([True, False], [True, False]):
+            fs.mkdir(target)
 
-            fs.rm(fs.ls(target, detail=False), recursive=True)
-            assert not fs.exists(fs_join(target, "newdir"))
+            t = fs_join(target, "newdir") if new_dir else target
+            t = t + "/" if target_slash else t
+
+            fs.copy(fs_join(source, path), t, recursive=recursive, maxdepth=maxdepth)
+
+            output = fs.find(target)
+            if new_dir:
+                prefixed_expected = [
+                    fs_sanitize_path(fs_join(target, "newdir", p)) for p in expected
+                ]
+            else:
+                prefixed_expected = [
+                    fs_sanitize_path(fs_join(target, p)) for p in expected
+                ]
+            assert sorted(output) == sorted(prefixed_expected)
+
+            try:
+                fs.rm(target, recursive=True)
+            except FileNotFoundError:
+                pass
 
     def test_copy_list_of_files_to_existing_directory(
-        self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
+        self,
+        fs,
+        fs_join,
+        fs_bulk_operations_scenario_0,
+        fs_target,
+        supports_empty_directories,
     ):
         # Copy scenario 2a
         source = fs_bulk_operations_scenario_0
 
         target = fs_target
         fs.mkdir(target)
-        if not self.supports_empty_directories():
+        if not supports_empty_directories:
             # Force target directory to exist by adding a dummy file
             dummy = fs_join(target, "dummy")
             fs.touch(dummy)
@@ -309,8 +443,15 @@ class AbstractCopyTests:
             assert fs.isfile(fs_join(target, "file2"))
             assert fs.isfile(fs_join(target, "subfile1"))
 
-            fs.rm(fs.find(target))
-            assert fs.ls(target) == [] if self.supports_empty_directories() else [dummy]
+            fs.rm(
+                [
+                    fs_join(target, "file1"),
+                    fs_join(target, "file2"),
+                    fs_join(target, "subfile1"),
+                ],
+                recursive=True,
+            )
+            assert fs.ls(target) == ([] if supports_empty_directories else [dummy])
 
     def test_copy_list_of_files_to_new_directory(
         self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
@@ -347,3 +488,56 @@ class AbstractCopyTests:
         assert fs.isdir(target)
         assert fs.isfile(fs_join(target, "file1"))
         assert fs.isfile(fs_join(target, "file2"))
+
+    def test_copy_directory_without_files_with_same_name_prefix(
+        self,
+        fs,
+        fs_join,
+        fs_target,
+        fs_dir_and_file_with_same_name_prefix,
+        supports_empty_directories,
+    ):
+        # Create the test dirs
+        source = fs_dir_and_file_with_same_name_prefix
+        target = fs_target
+
+        # Test without glob
+        fs.cp(fs_join(source, "subdir"), target, recursive=True)
+
+        assert fs.isfile(fs_join(target, "subfile.txt"))
+        assert not fs.isfile(fs_join(target, "subdir.txt"))
+
+        fs.rm([fs_join(target, "subfile.txt")])
+        if supports_empty_directories:
+            assert fs.ls(target) == []
+        else:
+            assert not fs.exists(target)
+
+        # Test with glob
+        fs.cp(fs_join(source, "subdir*"), target, recursive=True)
+
+        assert fs.isdir(fs_join(target, "subdir"))
+        assert fs.isfile(fs_join(target, "subdir", "subfile.txt"))
+        assert fs.isfile(fs_join(target, "subdir.txt"))
+
+    def test_copy_with_source_and_destination_as_list(
+        self, fs, fs_target, fs_join, fs_10_files_with_hashed_names
+    ):
+        # Create the test dir
+        source = fs_10_files_with_hashed_names
+        target = fs_target
+
+        # Create list of files for source and destination
+        source_files = []
+        destination_files = []
+        for i in range(10):
+            hashed_i = md5(str(i).encode("utf-8")).hexdigest()
+            source_files.append(fs_join(source, f"{hashed_i}.txt"))
+            destination_files.append(fs_join(target, f"{hashed_i}.txt"))
+
+        # Copy and assert order was kept
+        fs.copy(path1=source_files, path2=destination_files)
+
+        for i in range(10):
+            file_content = fs.cat(destination_files[i]).decode("utf-8")
+            assert file_content == str(i)