dvc-objects 1.3.1__tar.gz → 1.4.0__tar.gz

{dvc-objects-1.3.1/src/dvc_objects.egg-info → dvc-objects-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dvc-objects
-Version: 1.3.1
+Version: 1.4.0
 Summary: dvc objects
 Author-email: Iterative <support@dvc.org>
 License: Apache-2.0
@@ -28,6 +28,8 @@ Requires-Dist: pytest-cov==4.0.0; extra == "tests"
 Requires-Dist: pytest-mock==3.8.2; extra == "tests"
 Requires-Dist: mypy==1.5.1; extra == "tests"
 Requires-Dist: pytest-servers[s3]==0.1.3; extra == "tests"
+Requires-Dist: pytest-benchmark; extra == "tests"
+Requires-Dist: reflink; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: dvc-objects[tests]; extra == "dev"
 Requires-Dist: types-tqdm; extra == "dev"

{dvc-objects-1.3.1 → dvc-objects-1.4.0}/noxfile.py

@@ -8,10 +8,12 @@ nox.options.reuse_existing_virtualenvs = True
 nox.options.sessions = "lint", "tests"
 locations = "src", "tests"
 
+pip_dev_flags = ["--use-pep517"]  # reflink package is still missing wheels
+
 
 @nox.session(python=["3.8", "3.9", "3.10", "3.11", "pypy3.8", "pypy3.9"])
 def tests(session: nox.Session) -> None:
-    session.install(".[tests]")
+    session.install(".[tests]", *pip_dev_flags)
     session.run(
         "pytest",
         "--cov",
@@ -24,7 +26,7 @@ def tests(session: nox.Session) -> None:
 @nox.session
 def lint(session: nox.Session) -> None:
     session.install("pre-commit")
-    session.install("-e", ".[dev]")
+    session.install("-e", ".[dev]", *pip_dev_flags)
 
     args = *(session.posargs or ("--show-diff-on-failure",)), "--all-files"
     session.run("pre-commit", "run", *args)
@@ -34,7 +36,7 @@ def lint(session: nox.Session) -> None:
 @nox.session
 def safety(session: nox.Session) -> None:
     """Scan dependencies for insecure packages."""
-    session.install(".[dev]")
+    session.install(".[dev]", *pip_dev_flags)
     session.install("safety")
     session.run("safety", "check", "--full-report")
 
@@ -58,4 +60,6 @@ def dev(session: nox.Session) -> None:
     session.run("virtualenv", venv_dir, silent=True)
 
     python = os.path.join(venv_dir, "bin/python")
-    session.run(python, "-m", "pip", "install", "-e", ".[dev]", external=True)
+    session.run(
+        python, "-m", "pip", "install", "-e", ".[dev]", *pip_dev_flags, external=True
+    )

{dvc-objects-1.3.1 → dvc-objects-1.4.0}/pyproject.toml

@@ -41,6 +41,8 @@ tests = [
     "pytest-mock==3.8.2",
     "mypy==1.5.1",
     "pytest-servers[s3]==0.1.3",
+    "pytest-benchmark",
+    "reflink",
 ]
 dev = [
     "dvc-objects[tests]",
@@ -98,6 +100,8 @@ ignore_missing_imports = true
 module = [
     "fsspec.*",
     "funcy",
+    "reflink.*",
+    "shortuuid",
 ]
 
 [tool.codespell]

{dvc-objects-1.3.1 → dvc-objects-1.4.0}/src/dvc_objects/executors.py

@@ -45,10 +45,15 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor):
         It does not create all the futures at once to reduce memory usage.
         """
 
+        it = zip(*iterables)
+        if self.max_workers == 1:
+            for args in it:
+                yield fn(*args)
+            return
+
         def create_taskset(n: int) -> Set[futures.Future]:
             return {self.submit(fn, *args) for args in islice(it, n)}
 
-        it = zip(*iterables)
         tasks = create_taskset(self.max_workers * 5)
         while tasks:
             done, tasks = futures.wait(tasks, return_when=futures.FIRST_COMPLETED)

{dvc-objects-1.3.1 → dvc-objects-1.4.0}/src/dvc_objects/fs/generic.py

@@ -86,7 +86,7 @@ def copy(
             to_fs,
             to_path,
             callback=callback,
-            batch_size=batch_size,
+            batch_size=1 if isinstance(to_fs, LocalFileSystem) else batch_size,
             on_error=on_error,
         )
     if isinstance(to_fs, LocalFileSystem):

{dvc-objects-1.3.1 → dvc-objects-1.4.0}/src/dvc_objects/fs/system.py

@@ -4,6 +4,7 @@ import os
 import platform
 import stat
 import sys
+import functools
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -36,7 +37,8 @@ def symlink(source: "AnyFSPath", link_name: "AnyFSPath") -> None:
     os.symlink(source, link_name)
 
 
-def _reflink_darwin(src: "AnyFSPath", dst: "AnyFSPath") -> None:
+@functools.lru_cache(maxsize=1)
+def _clonefile():
     import ctypes
 
     def _cdll(name):
@@ -48,22 +50,30 @@ def _reflink_darwin(src: "AnyFSPath", dst: "AnyFSPath") -> None:
         clib = _cdll(LIBC)
     except OSError as exc:
         logger.debug(
-            "unable to access '{}' (errno '{}'). " "Falling back to '{}'.".format(
-                LIBC, exc.errno, LIBC_FALLBACK
-            )
+            "unable to access '%s' (errno '%d'). Falling back to '%s'.",
+            LIBC,
+            exc.errno,
+            LIBC_FALLBACK,
         )
         if exc.errno != errno.ENOENT:
             raise
         # NOTE: trying to bypass System Integrity Protection (SIP)
         clib = _cdll(LIBC_FALLBACK)
 
-    if not hasattr(clib, "clonefile"):
+    clonefile = getattr(clib, "clonefile", None)
+    if clonefile is None:
         raise OSError(
             errno.ENOTSUP,
             "'clonefile' not supported by the standard library",
         )
 
-    clonefile = clib.clonefile
+    return clonefile
+
+
+def _reflink_darwin(src: "AnyFSPath", dst: "AnyFSPath") -> None:
+    import ctypes
+
+    clonefile = _clonefile()
     clonefile.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int]
     clonefile.restype = ctypes.c_int
 

{dvc-objects-1.3.1 → dvc-objects-1.4.0/src/dvc_objects.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dvc-objects
-Version: 1.3.1
+Version: 1.4.0
 Summary: dvc objects
 Author-email: Iterative <support@dvc.org>
 License: Apache-2.0
@@ -28,6 +28,8 @@ Requires-Dist: pytest-cov==4.0.0; extra == "tests"
 Requires-Dist: pytest-mock==3.8.2; extra == "tests"
 Requires-Dist: mypy==1.5.1; extra == "tests"
 Requires-Dist: pytest-servers[s3]==0.1.3; extra == "tests"
+Requires-Dist: pytest-benchmark; extra == "tests"
+Requires-Dist: reflink; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: dvc-objects[tests]; extra == "dev"
 Requires-Dist: types-tqdm; extra == "dev"

{dvc-objects-1.3.1 → dvc-objects-1.4.0}/src/dvc_objects.egg-info/SOURCES.txt

@@ -42,6 +42,7 @@ tests/conftest.py
 tests/test_odb.py
 tests/test_reflink.py
 tests/test_transfer.py
+tests/benchmarks/test_fs.py
 tests/fs/__init__.py
 tests/fs/test_callbacks.py
 tests/fs/test_localfs.py

{dvc-objects-1.3.1 → dvc-objects-1.4.0}/src/dvc_objects.egg-info/requires.txt

@@ -16,3 +16,5 @@ pytest-cov==4.0.0
 pytest-mock==3.8.2
 mypy==1.5.1
 pytest-servers[s3]==0.1.3
+pytest-benchmark
+reflink

dvc-objects-1.4.0/tests/benchmarks/test_fs.py

@@ -0,0 +1,42 @@
+import errno
+import pytest
+import shutil
+from reflink import reflink as pyreflink
+from reflink.error import ReflinkImpossibleError
+from dvc_objects.fs.system import reflink, hardlink, symlink
+
+NLINKS = 10000
+
+
+@pytest.mark.parametrize(
+    "link", [pytest.param(pyreflink, id="pyreflink"), reflink, hardlink, symlink]
+)
+def test_link(benchmark, tmp_path, link):
+    (tmp_path / "original").mkdir()
+
+    for idx in range(NLINKS):
+        (tmp_path / "original" / str(idx)).write_text(str(idx))
+
+    def _setup():
+        try:
+            shutil.rmtree(tmp_path / "links")
+        except FileNotFoundError:
+            pass
+
+        (tmp_path / "links").mkdir()
+
+    original = str(tmp_path / "original")
+    links = str(tmp_path / "links")
+
+    def _link():
+        for idx in range(NLINKS):
+            try:
+                link(f"{original}/{idx}", f"{links}/{idx}")
+            except Exception as exc:
+                if isinstance(exc, (ReflinkImpossibleError, NotImplementedError)) or (
+                    isinstance(exc, OSError) and exc.errno == errno.ENOTSUP
+                ):
+                    pytest.skip(str(exc))
+                raise
+
+    benchmark.pedantic(_link, setup=_setup)