scmrepo 1.4.1__tar.gz → 1.5.0__tar.gz

{scmrepo-1.4.1 → scmrepo-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scmrepo
-Version: 1.4.1
+Version: 1.5.0
 Summary: SCM wrapper and fsspec filesystem for Git for use in DVC
 Home-page: https://github.com/iterative/scmrepo
 Maintainer-email: support@dvc.org
@@ -17,13 +17,15 @@ Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: gitpython>3
 Requires-Dist: dulwich>=0.21.6
-Requires-Dist: pygit2>=1.13.0
+Requires-Dist: pygit2>=1.13.3
 Requires-Dist: pygtrie>=2.3.2
 Requires-Dist: fsspec>=2021.7.0
 Requires-Dist: pathspec>=0.9.0
 Requires-Dist: asyncssh<3,>=2.13.1
 Requires-Dist: funcy>=1.14
 Requires-Dist: shortuuid>=0.5.0
+Requires-Dist: dvc-objects<2,>=1.0.1
+Requires-Dist: dvc-http>=2.29.0
 Provides-Extra: tests
 Requires-Dist: pytest==7.2.0; extra == "tests"
 Requires-Dist: pytest-sugar==0.9.5; extra == "tests"

{scmrepo-1.4.1 → scmrepo-1.5.0}/pyproject.toml

@@ -70,6 +70,7 @@ files = ["src", "tests"]
 [[tool.mypy.overrides]]
 module = [
     "pygtrie",
+    "dvc_http.*",
     "funcy",
     "git",
     "gitdb.*",

{scmrepo-1.4.1 → scmrepo-1.5.0}/setup.cfg

@@ -26,13 +26,15 @@ packages = find:
 install_requires = 
 	gitpython>3
 	dulwich>=0.21.6
-	pygit2>=1.13.0
+	pygit2>=1.13.3
 	pygtrie>=2.3.2
 	fsspec>=2021.7.0
 	pathspec>=0.9.0
 	asyncssh>=2.13.1,<3
 	funcy>=1.14
 	shortuuid>=0.5.0
+	dvc-objects>=1.0.1,<2
+	dvc-http>=2.29.0
 
 [options.extras_require]
 tests = 

{scmrepo-1.4.1 → scmrepo-1.5.0}/src/scmrepo/base.py

@@ -1,7 +1,8 @@
 """Manages source control systems (e.g. Git) in DVC."""
+from contextlib import AbstractContextManager
 
 
-class Base:
+class Base(AbstractContextManager):
     """Base class for source control management driver implementations."""
 
     def __init__(self, root_dir=None):
@@ -18,6 +19,9 @@ class Base:
             class_name=type(self).__name__, directory=self.dir
         )
 
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
     @property
     def dir(self):
         """Path to a directory with SCM specific information."""

{scmrepo-1.4.1 → scmrepo-1.5.0}/src/scmrepo/fs.py

@@ -187,9 +187,10 @@ class GitFileSystem(AbstractFileSystem):
         self,
         path: str,
         mode: str = "rb",
-        block_size: int = None,
+        block_size: Optional[int] = None,
         autocommit: bool = True,
-        cache_options: Dict = None,
+        cache_options: Optional[Dict] = None,
+        raw: bool = False,
         **kwargs: Any,
     ) -> BinaryIO:
         if mode != "rb":
@@ -197,7 +198,7 @@ class GitFileSystem(AbstractFileSystem):
 
         key = self._get_key(path)
         try:
-            obj = self.trie.open(key, mode=mode)
+            obj = self.trie.open(key, mode=mode, raw=raw)
             obj.size = bytesio_len(obj)
             return obj
         except KeyError as exc:

{scmrepo-1.4.1 → scmrepo-1.5.0}/src/scmrepo/git/__init__.py

@@ -171,6 +171,13 @@ class Git(Base):
     def ignore_file(self):
         return self.GITIGNORE
 
+    @cached_property
+    def lfs_storage(self):
+        from .lfs import LFSStorage
+        from .lfs.storage import get_storage_path
+
+        return LFSStorage(get_storage_path(self))
+
     def _get_gitignore(self, path):
         ignore_file_dir = os.path.dirname(path)
 
@@ -267,6 +274,8 @@ class Git(Base):
 
     def close(self):
         self.backends.close_initialized()
+        if "lfs_storage" in self.__dict__:
+            self.lfs_storage.close()
 
     @property
     def no_commits(self):
@@ -358,6 +367,7 @@ class Git(Base):
     is_tracked = partialmethod(_backend_func, "is_tracked")
     is_dirty = partialmethod(_backend_func, "is_dirty")
     active_branch = partialmethod(_backend_func, "active_branch")
+    active_branch_remote = partialmethod(_backend_func, "active_branch_remote")
     list_branches = partialmethod(_backend_func, "list_branches")
     list_tags = partialmethod(_backend_func, "list_tags")
     list_all_commits = partialmethod(_backend_func, "list_all_commits")
@@ -383,8 +393,11 @@ class Git(Base):
     status = partialmethod(_backend_func, "status")
     merge = partialmethod(_backend_func, "merge")
     validate_git_remote = partialmethod(_backend_func, "validate_git_remote")
+    get_remote_url = partialmethod(_backend_func, "get_remote_url")
     check_ref_format = partialmethod(_backend_func, "check_ref_format")
     get_tag = partialmethod(_backend_func, "get_tag")
+    get_config = partialmethod(_backend_func, "get_config")
+    check_attr = partialmethod(_backend_func, "check_attr")
 
     get_tree_obj = partialmethod(_backend_func, "get_tree_obj")
 

{scmrepo-1.4.1 → scmrepo-1.5.0}/src/scmrepo/git/backend/base.py

@@ -10,6 +10,7 @@ from ..objects import GitObject
 if TYPE_CHECKING:
     from scmrepo.progress import GitProgressEvent
 
+    from ..config import Config
     from ..objects import GitCommit, GitTag
 
 
@@ -135,6 +136,10 @@ class BaseGitBackend(ABC):
     def active_branch(self) -> str:
         pass
 
+    @abstractmethod
+    def active_branch_remote(self) -> str:
+        """Return the fetch remote name for the current branch."""
+
     @abstractmethod
     def list_branches(self) -> Iterable[str]:
         pass
@@ -397,6 +402,10 @@ class BaseGitBackend(ABC):
     def validate_git_remote(self, url: str, **kwargs):
         """Verify that url is a valid git URL or remote name."""
 
+    @abstractmethod
+    def get_remote_url(self, remote: str) -> str:
+        """Return URL for the specified remote."""
+
     @abstractmethod
     def check_ref_format(self, refname: str) -> bool:
         """Check if a reference name is well formed."""
@@ -413,3 +422,34 @@ class BaseGitBackend(ABC):
             String SHA for the target object if the tag is a lightweight tag.
             GitTag object if the tag is an annotated tag.
         """
+
+    @abstractmethod
+    def get_config(self, path: Optional[str] = None) -> "Config":
+        """Return a Git config object.
+
+        Args:
+            path: If set, a config object for the specified config file will be
+                returned. By default, the standard Git system/global/repo config
+                stack object will be returned.
+        """
+
+    @abstractmethod
+    def check_attr(
+        self,
+        path: str,
+        attr: str,
+        source: Optional[str] = None,
+    ) -> Optional[Union[bool, str]]:
+        """Return the value of the specified attribute for a pathname.
+
+        Args:
+            path: Pathname to check.
+            attr: Attribute to check.
+            source: Optional tree-ish source to check.
+
+        Returns:
+            None when the attribute is not defined for the path (unspecified).
+            True when the attribute is defined as true (set).
+            False when the attribute is defined as false (unset).
+            The value of the attribute when a value has been assigned.
+        """

{scmrepo-1.4.1 → scmrepo-1.5.0}/src/scmrepo/git/backend/dulwich/__init__.py

@@ -27,11 +27,13 @@ from scmrepo.exceptions import AuthError, CloneError, InvalidRemote, RevError, S
 from scmrepo.progress import GitProgressReporter
 from scmrepo.utils import relpath
 
+from ...config import Config
 from ...objects import GitObject, GitTag
 from ..base import BaseGitBackend, SyncStatus
 
 if TYPE_CHECKING:
     from dulwich.client import SSHVendor
+    from dulwich.config import ConfigFile, StackedConfig
     from dulwich.repo import Repo
 
     from scmrepo.progress import GitProgressEvent
@@ -49,7 +51,16 @@ class DulwichObject(GitObject):
         self._mode = mode
         self._sha = sha
 
-    def open(self, mode: str = "r", encoding: str = None):
+    def open(  # pylint: disable=unused-argument
+        self,
+        mode: str = "r",
+        encoding: Optional[str] = None,
+        raw: bool = True,
+        rev: Optional[str] = None,
+        **kwargs,
+    ):
+        if not raw:
+            raise NotImplementedError
         if not encoding:
             encoding = locale.getpreferredencoding(False)
         # NOTE: we didn't load the object before as Dulwich will also try to
@@ -130,6 +141,35 @@ def _get_ssh_vendor() -> "SSHVendor":
     return AsyncSSHVendor()
 
 
+class DulwichConfig(Config):
+    def __init__(self, config: Union["ConfigFile", "StackedConfig"]):
+        self._config = config
+
+    @property
+    def encoding(self) -> str:
+        from dulwich.config import ConfigFile
+
+        if isinstance(self._config, ConfigFile):
+            return self._config.encoding
+        return self._config.backends[0].encoding
+
+    def get(self, section: Tuple[str, ...], name: str) -> str:
+        """Return the specified setting as a string."""
+        return self._config.get(section, name).decode(self.encoding)
+
+    def get_bool(self, section: Tuple[str, ...], name: str) -> bool:
+        """Return the specified setting as a boolean."""
+        value = self._config.get_boolean(section, name)
+        if value is None:
+            raise ValueError("setting is not a valid boolean")
+        return value
+
+    def get_multivar(self, section: Tuple[str, ...], name: str) -> Iterator[str]:
+        """Iterate over string values in the specified multivar setting."""
+        for value in self._config.get_multivar(section, name):
+            yield value.decode(self.encoding)
+
+
 class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
     """Dulwich Git backend."""
 
@@ -430,6 +470,9 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def active_branch(self) -> str:
         raise NotImplementedError
 
+    def active_branch_remote(self) -> str:
+        raise NotImplementedError
+
     def list_branches(self) -> Iterable[str]:
         base = "refs/heads/"
         return sorted(ref[len(base) :] for ref in self.iter_refs(base))
@@ -884,6 +927,14 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
         ):
             raise InvalidRemote(url)
 
+    def get_remote_url(self, remote: str) -> str:
+        from dulwich.porcelain import get_remote_repo
+
+        remote_name, location = get_remote_repo(self.repo, remote)
+        if not remote_name:
+            raise InvalidRemote(remote)
+        return location
+
     def check_ref_format(self, refname: str) -> bool:
         from dulwich.refs import check_ref_format
 
@@ -913,6 +964,21 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
             )
         return os.fsdecode(ref)
 
+    def get_config(self, path: Optional[str] = None) -> "Config":
+        from dulwich.config import ConfigFile
+
+        if path:
+            return DulwichConfig(ConfigFile.from_path(path))
+        return DulwichConfig(self.repo.get_config_stack())
+
+    def check_attr(
+        self,
+        path: str,
+        attr: str,
+        source: Optional[str] = None,
+    ) -> Optional[Union[bool, str]]:
+        raise NotImplementedError
+
 
 _IDENTITY_RE = re.compile(r"(?P<name>.+)\s+<(?P<email>.+)>")
 

{scmrepo-1.4.1 → scmrepo-1.5.0}/src/scmrepo/git/backend/gitpython.py

@@ -2,6 +2,7 @@ import io
 import locale
 import logging
 import os
+import re
 import sys
 from functools import partial, wraps
 from typing import (
@@ -21,6 +22,7 @@ from funcy import ignore
 
 from scmrepo.exceptions import (
     CloneError,
+    InvalidRemote,
     MergeConflictError,
     RevError,
     SCMError,
@@ -34,6 +36,8 @@ from .base import BaseGitBackend, SyncStatus
 if TYPE_CHECKING:
     from scmrepo.progress import GitProgressEvent
 
+    from ..config import Config
+
 
 logger = logging.getLogger(__name__)
 
@@ -80,7 +84,15 @@ class GitPythonObject(GitObject):
     def __init__(self, obj):
         self.obj = obj
 
-    def open(self, mode: str = "r", encoding: str = None):
+    def open(
+        self,
+        mode: str = "r",
+        encoding: str = None,
+        raw: bool = True,
+        **kwargs,
+    ):
+        if not raw:
+            raise NotImplementedError
         if not encoding:
             encoding = locale.getpreferredencoding(False)
         # GitPython's obj.data_stream is a fragile thing, it is better to
@@ -341,6 +353,12 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
         except TypeError as exc:
             raise SCMError("No active branch") from exc
 
+    def active_branch_remote(self) -> str:
+        try:
+            return self.repo.active_branch.tracking_branch()
+        except (TypeError, ValueError) as exc:
+            raise SCMError("No active branch tracking remote") from exc
+
     def list_branches(self):
         return [h.name for h in self.repo.heads]
 
@@ -714,6 +732,14 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def validate_git_remote(self, url: str, **kwargs):
         raise NotImplementedError
 
+    def get_remote_url(self, remote: str) -> str:
+        from git.exc import GitCommandError
+
+        try:
+            return self.repo.remotes[remote].url
+        except (KeyError, GitCommandError) as exc:
+            raise InvalidRemote(remote) from exc
+
     def check_ref_format(self, refname: str):
         raise NotImplementedError
 
@@ -736,3 +762,32 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
         except IndexError:
             pass
         return None
+
+    def get_config(self, path: Optional[str] = None) -> "Config":
+        raise NotImplementedError
+
+    def check_attr(
+        self,
+        path: str,
+        attr: str,
+        source: Optional[str] = None,
+    ) -> Optional[Union[bool, str]]:
+        from git.exc import GitCommandError
+
+        try:
+            result = self.git.check_attr(attr, "--", path, source=source)
+        except GitCommandError as exc:
+            raise SCMError("Failed to check attribute") from exc
+        escaped_path = re.escape(path)
+        escaped_attr = re.escape(attr)
+        m = re.match(f"{escaped_path}: {escaped_attr}: (?P<info>.*)", result)
+        if not m or not m.group("info"):
+            raise SCMError("Failed to check attribute")
+        info = m.group("info")
+        if info == "unspecified":
+            return None
+        if info == "set":
+            return True
+        if info == "unset":
+            return False
+        return info

{scmrepo-1.4.1 → scmrepo-1.5.0}/src/scmrepo/git/backend/pygit2/__init__.py

@@ -3,13 +3,14 @@ import logging
 import os
 import stat
 from contextlib import contextmanager
-from io import BytesIO, StringIO
+from io import BytesIO, StringIO, TextIOWrapper
 from typing import (
     TYPE_CHECKING,
     Callable,
     Dict,
     Generator,
     Iterable,
+    Iterator,
     List,
     Mapping,
     Optional,
@@ -20,8 +21,15 @@ from urllib.parse import urlparse
 
 from funcy import cached_property, reraise
 
-from scmrepo.exceptions import CloneError, MergeConflictError, RevError, SCMError
+from scmrepo.exceptions import (
+    CloneError,
+    InvalidRemote,
+    MergeConflictError,
+    RevError,
+    SCMError,
+)
 from scmrepo.git.backend.base import BaseGitBackend, SyncStatus
+from scmrepo.git.config import Config
 from scmrepo.git.objects import GitCommit, GitObject, GitTag
 from scmrepo.utils import relpath
 
@@ -29,7 +37,8 @@ logger = logging.getLogger(__name__)
 
 
 if TYPE_CHECKING:
-    from pygit2 import Oid, Signature
+    from pygit2 import Commit, Oid, Signature
+    from pygit2.config import Config as _Pygit2Config
     from pygit2.remote import Remote  # type: ignore
     from pygit2.repository import Repository
 
@@ -37,13 +46,47 @@ if TYPE_CHECKING:
 
 
 class Pygit2Object(GitObject):
-    def __init__(self, obj):
+    def __init__(self, obj, backend: Optional["Pygit2Backend"] = None):
         self.obj = obj
+        self.backend = backend
+
+    def open(
+        self,
+        mode: str = "r",
+        encoding: str = None,
+        key: Optional[Tuple[str, ...]] = None,
+        raw: bool = True,
+        rev: Optional[str] = None,
+        **kwargs,
+    ):
+        from pygit2 import BlobIO, GitError
 
-    def open(self, mode: str = "r", encoding: str = None):
         if not encoding:
             encoding = locale.getpreferredencoding(False)
-        data = self.obj.read_raw()
+        if self.backend is not None:
+            try:
+                if rev:
+                    # pylint: disable-next=protected-access
+                    commit, _ref = self.backend._resolve_refish(rev)
+                else:
+                    pass
+                if raw:
+                    blob_kwargs = {}
+                else:
+                    assert key is not None
+                    path = "/".join(key)
+                    blob_kwargs = {
+                        "as_path": path,
+                        "commit_id": commit.oid,
+                    }
+                blobio = BlobIO(self.obj, **blob_kwargs)
+                if mode == "rb":
+                    return blobio
+                return TextIOWrapper(blobio, encoding=encoding)
+            except GitError as exc:
+                raise SCMError("failed to read git blob") from exc
+        else:
+            data = self.obj.read_raw()
         if mode == "rb":
             return BytesIO(data)
         return StringIO(data.decode(encoding))
@@ -74,7 +117,34 @@ class Pygit2Object(GitObject):
 
     def scandir(self) -> Iterable["Pygit2Object"]:
         for entry in self.obj:  # noqa: B301
-            yield Pygit2Object(entry)
+            yield Pygit2Object(entry, backend=self.backend)
+
+
+class Pygit2Config(Config):
+    def __init__(self, config: "_Pygit2Config"):
+        self._config = config
+
+    def _key(self, section: Tuple[str, ...], name: str) -> str:
+        return ".".join(section + (name,))
+
+    def get(self, section: Tuple[str, ...], name: str) -> str:
+        return self._config[self._key(section, name)]
+
+    def get_bool(self, section: Tuple[str, ...], name: str) -> bool:
+        from pygit2 import GitError
+
+        try:
+            return self._config.get_bool(self._key(section, name))
+        except GitError as exc:
+            raise ValueError("invalid boolean config entry") from exc
+
+    def get_multivar(self, section: Tuple[str, ...], name: str) -> Iterator[str]:
+        from pygit2 import GitError
+
+        try:
+            yield from self._config.get_multivar(self._key(section, name))
+        except GitError as exc:
+            raise ValueError("invalid multivar config entry") from exc
 
 
 class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
@@ -83,6 +153,8 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
     ):
         import pygit2
 
+        from .filter import LFSFilter
+
         if search_parent_directories:
             ceiling_dirs = ""
         else:
@@ -99,6 +171,14 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
 
         self._stashes: dict = {}
 
+        try:
+            # NOTE: we want this init to be lazy so we do it on backend init.
+            # for subsequent backend instances, this call will error out since
+            # the filter is already registered
+            pygit2.filter_register("lfs", LFSFilter)
+        except ValueError:
+            pass
+
     def close(self):
         if hasattr(self, "_refdb"):
             del self._refdb
@@ -353,6 +433,15 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
             return self.repo.references["HEAD"].target[11:]
         return self.repo.head.shorthand
 
+    def active_branch_remote(self) -> str:
+        try:
+            upstream = self.repo.branches[self.active_branch()].upstream
+            if upstream:
+                return upstream.remote_name
+        except (KeyError, ValueError):
+            pass
+        raise SCMError("No active branch tracking remote")
+
     def list_branches(self) -> Iterable[str]:
         base = "refs/heads/"
         return sorted(ref[len(base) :] for ref in self.iter_refs(base))
@@ -366,7 +455,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
 
     def get_tree_obj(self, rev: str, **kwargs) -> Pygit2Object:
         tree = self.repo[rev].tree
-        return Pygit2Object(tree)
+        return Pygit2Object(tree, backend=self)
 
     def get_rev(self) -> str:
         raise NotImplementedError
@@ -968,6 +1057,12 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
     def validate_git_remote(self, url: str, **kwargs):
         raise NotImplementedError
 
+    def get_remote_url(self, remote: str) -> str:
+        try:
+            return self.repo.remotes[remote].url
+        except KeyError as exc:
+            raise InvalidRemote(remote) from exc
+
     def check_ref_format(self, refname: str):
         raise NotImplementedError
 
@@ -996,3 +1091,38 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         except KeyError:
             pass
         return str(ref.target)
+
+    def get_config(self, path: Optional[str] = None) -> "Config":
+        from pygit2.config import Config as _Pygit2Config
+
+        if path:
+            return Pygit2Config(_Pygit2Config(path))
+        return Pygit2Config(self.repo.config)
+
+    def check_attr(
+        self,
+        path: str,
+        attr: str,
+        source: Optional[str] = None,
+    ) -> Optional[Union[bool, str]]:
+        from pygit2 import (
+            GIT_ATTR_CHECK_FILE_THEN_INDEX,
+            GIT_ATTR_CHECK_INCLUDE_COMMIT,
+            GIT_ATTR_CHECK_INDEX_ONLY,
+            GitError,
+        )
+
+        commit: Optional["Commit"] = None
+        flags = GIT_ATTR_CHECK_FILE_THEN_INDEX
+        if source:
+            try:
+                commit, _ref = self._resolve_refish(source)
+                flags = GIT_ATTR_CHECK_INDEX_ONLY | GIT_ATTR_CHECK_INCLUDE_COMMIT
+            except (KeyError, GitError) as exc:
+                raise SCMError(f"Invalid commit '{source}'") from exc
+        try:
+            return self.repo.get_attr(
+                path, attr, flags=flags, commit=commit.id if commit else None
+            )
+        except GitError as exc:
+            raise SCMError("Failed to check attribute") from exc