scmrepo 3.0.0__tar.gz → 3.2.0__tar.gz

{scmrepo-3.0.0 → scmrepo-3.2.0}/.pre-commit-config.yaml

@@ -20,7 +20,7 @@ repos:
       - id: sort-simple-yaml
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.1.13'
+    rev: 'v0.2.2'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{scmrepo-3.0.0/src/scmrepo.egg-info → scmrepo-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scmrepo
-Version: 3.0.0
+Version: 3.2.0
 Summary: scmrepo
 Author-email: Iterative <support@dvc.org>
 License: Apache-2.0
@@ -19,13 +19,12 @@ Requires-Dist: gitpython>3
 Requires-Dist: dulwich>=0.21.6
 Requires-Dist: pygit2>=1.14.0
 Requires-Dist: pygtrie>=2.3.2
-Requires-Dist: fsspec>=2024.2.0
+Requires-Dist: fsspec[tqdm]>=2024.2.0
 Requires-Dist: pathspec>=0.9.0
 Requires-Dist: asyncssh<3,>=2.13.1
 Requires-Dist: funcy>=1.14
-Requires-Dist: shortuuid>=0.5.0
-Requires-Dist: dvc-objects<5,>=4
-Requires-Dist: dvc-http>=2.29.0
+Requires-Dist: aiohttp-retry>=2.5.0
+Requires-Dist: tqdm
 Provides-Extra: tests
 Requires-Dist: pytest==7.2.0; extra == "tests"
 Requires-Dist: pytest-sugar==0.9.5; extra == "tests"
@@ -40,6 +39,7 @@ Requires-Dist: paramiko==3.3.1; extra == "tests"
 Requires-Dist: types-certifi==2021.10.8.3; extra == "tests"
 Requires-Dist: types-mock==5.1.0.2; extra == "tests"
 Requires-Dist: types-paramiko==3.4.0.20240120; extra == "tests"
+Requires-Dist: types-tqdm; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: scmrepo[tests]; extra == "dev"
 

{scmrepo-3.0.0 → scmrepo-3.2.0}/pyproject.toml

@@ -25,13 +25,12 @@ dependencies = [
     "dulwich>=0.21.6",
     "pygit2>=1.14.0",
     "pygtrie>=2.3.2",
-    "fsspec>=2024.2.0",
+    "fsspec[tqdm]>=2024.2.0",
     "pathspec>=0.9.0",
     "asyncssh>=2.13.1,<3",
     "funcy>=1.14",
-    "shortuuid>=0.5.0",
-    "dvc-objects>=4,<5",
-    "dvc-http>=2.29.0",
+    "aiohttp-retry>=2.5.0",
+    "tqdm",
 ]
 
 [project.urls]
@@ -54,6 +53,7 @@ tests = [
     "types-certifi==2021.10.8.3",
     "types-mock==5.1.0.2",
     "types-paramiko==3.4.0.20240120",
+    "types-tqdm",
 ]
 dev = [
     "scmrepo[tests]",
@@ -109,7 +109,6 @@ files = ["src", "tests"]
 [[tool.mypy.overrides]]
 module = [
     "pygtrie",
-    "dvc_http.*",
     "funcy",
     "git",
     "gitdb.*",

{scmrepo-3.0.0 → scmrepo-3.2.0}/src/scmrepo/git/backend/dulwich/__init__.py

@@ -842,7 +842,7 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
             if revision and revision not in rev_mapping:
                 rev_mapping[revision] = ref
         for rev in revs:
-            results[rev] = rev_mapping.get(rev, None)
+            results[rev] = rev_mapping.get(rev)
         return results
 
     def diff(self, rev_a: str, rev_b: str, binary=False) -> str:
@@ -978,3 +978,16 @@ def _parse_identity(identity: str) -> tuple[str, str]:
     if not m:
         raise SCMError("Could not parse tagger identity '{identity}'")
     return m.group("name"), m.group("email")
+
+
+def ls_remote(url: str) -> dict[str, str]:
+    from dulwich import porcelain
+    from dulwich.client import HTTPUnauthorized
+
+    try:
+        refs = porcelain.ls_remote(url)
+        return {os.fsdecode(ref): sha.decode("ascii") for ref, sha in refs.items()}
+    except HTTPUnauthorized as exc:
+        raise AuthError(url) from exc
+    except Exception as exc:  # noqa: BLE001
+        raise InvalidRemote(url) from exc

scmrepo-3.2.0/src/scmrepo/git/lfs/client.py

@@ -0,0 +1,275 @@
+import json
+import logging
+import os
+import re
+import shutil
+from abc import abstractmethod
+from collections.abc import Iterable, Iterator
+from contextlib import AbstractContextManager, contextmanager, suppress
+from tempfile import NamedTemporaryFile
+from typing import TYPE_CHECKING, Any, Optional
+
+import aiohttp
+from aiohttp_retry import ExponentialRetry, RetryClient
+from fsspec.asyn import _run_coros_in_chunks, sync_wrapper
+from fsspec.callbacks import DEFAULT_CALLBACK
+from fsspec.implementations.http import HTTPFileSystem
+from funcy import cached_property
+
+from scmrepo.git.backend.dulwich import _get_ssh_vendor
+from scmrepo.git.credentials import Credential, CredentialNotFoundError
+
+from .exceptions import LFSError
+from .pointer import Pointer
+
+if TYPE_CHECKING:
+    from fsspec.callbacks import Callback
+
+    from .storage import LFSStorage
+
+logger = logging.getLogger(__name__)
+
+
+class LFSClient(AbstractContextManager):
+    """Naive read-only LFS HTTP client."""
+
+    JSON_CONTENT_TYPE = "application/vnd.git-lfs+json"
+
+    _REQUEST_TIMEOUT = 60
+    _SESSION_RETRIES = 5
+    _SESSION_BACKOFF_FACTOR = 0.1
+
+    def __init__(self, url: str):
+        """
+        Args:
+            url: LFS server URL.
+        """
+        self.url = url
+
+    def __exit__(self, *args, **kwargs):
+        self.close()
+
+    @cached_property
+    def _fs(self) -> HTTPFileSystem:
+        async def get_client(**kwargs):
+            return RetryClient(
+                connector=aiohttp.TCPConnector(
+                    # Force cleanup of closed SSL transports.
+                    # See https://github.com/iterative/dvc/issues/7414
+                    enable_cleanup_closed=True,
+                ),
+                timeout=aiohttp.ClientTimeout(
+                    total=None,
+                    connect=self._REQUEST_TIMEOUT,
+                    sock_connect=self._REQUEST_TIMEOUT,
+                    sock_read=self._REQUEST_TIMEOUT,
+                ),
+                retry_options=ExponentialRetry(
+                    attempts=self._SESSION_RETRIES,
+                    factor=self._SESSION_BACKOFF_FACTOR,
+                    max_timeout=self._REQUEST_TIMEOUT,
+                    exceptions={aiohttp.ClientError},
+                ),
+                **kwargs,
+            )
+
+        return HTTPFileSystem(get_client=get_client)
+
+    @property
+    def loop(self):
+        return self._fs.loop
+
+    @classmethod
+    def from_git_url(cls, git_url: str) -> "LFSClient":
+        if git_url.startswith(("ssh://", "git@")):
+            return _SSHLFSClient.from_git_url(git_url)
+        if git_url.startswith("https://"):
+            return _HTTPLFSClient.from_git_url(git_url)
+        raise NotImplementedError(f"Unsupported Git URL: {git_url}")
+
+    def close(self):
+        pass
+
+    @abstractmethod
+    def _get_auth_header(self, *, upload: bool) -> dict:
+        ...
+
+    async def _batch_request(
+        self,
+        objects: Iterable[Pointer],
+        upload: bool = False,
+        ref: Optional[str] = None,
+        hash_algo: str = "sha256",
+    ) -> dict[str, Any]:
+        """Send LFS API /objects/batch request."""
+        url = f"{self.url}/objects/batch"
+        body: dict[str, Any] = {
+            "operation": "upload" if upload else "download",
+            "transfers": ["basic"],
+            "objects": [{"oid": obj.oid, "size": obj.size} for obj in objects],
+            "hash_algo": hash_algo,
+        }
+        if ref:
+            body["ref"] = [{"name": ref}]
+        session = await self._fs.set_session()
+        headers = {
+            "Accept": self.JSON_CONTENT_TYPE,
+            "Content-Type": self.JSON_CONTENT_TYPE,
+        }
+        try:
+            async with session.post(
+                url,
+                headers=headers,
+                json=body,
+                raise_for_status=True,
+            ) as resp:
+                data = await resp.json()
+        except aiohttp.ClientResponseError as exc:
+            if exc.status != 401:
+                raise
+            auth_header = self._get_auth_header(upload=upload)
+            if not auth_header:
+                raise
+            async with session.post(
+                url,
+                headers={**headers, **auth_header},
+                json=body,
+                raise_for_status=True,
+            ) as resp:
+                data = await resp.json()
+        return data
+
+    async def _download(
+        self,
+        storage: "LFSStorage",
+        objects: Iterable[Pointer],
+        callback: "Callback" = DEFAULT_CALLBACK,
+        batch_size: Optional[int] = None,
+        **kwargs,
+    ):
+        async def _get_one(from_path: str, to_path: str, **kwargs):
+            with _as_atomic(to_path, create_parents=True) as tmp_file:
+                with callback.branched(from_path, tmp_file) as child:
+                    await self._fs._get_file(
+                        from_path, tmp_file, callback=child, **kwargs
+                    )
+                    callback.relative_update()
+
+        resp_data = await self._batch_request(objects, **kwargs)
+        if resp_data.get("transfer", "basic") != "basic":
+            raise LFSError("Unsupported LFS transfer type")
+        coros = []
+        for data in resp_data.get("objects", []):
+            obj = Pointer(data["oid"], data["size"])
+            download = data.get("actions", {}).get("download", {})
+            url = download.get("href")
+            if not url:
+                logger.debug("No download URL for LFS object '%s'", obj)
+                continue
+            headers = download.get("header", {})
+            to_path = storage.oid_to_path(obj.oid)
+            coros.append(_get_one(url, to_path, headers=headers))
+        for result in await _run_coros_in_chunks(
+            coros, batch_size=batch_size, return_exceptions=True
+        ):
+            if isinstance(result, BaseException):
+                raise result
+
+    download = sync_wrapper(_download)
+
+
+class _HTTPLFSClient(LFSClient):
+    def __init__(self, url: str, git_url: str):
+        """
+        Args:
+            url: LFS server URL.
+            git_url: Git HTTP URL.
+        """
+        super().__init__(url)
+        self.git_url = git_url
+
+    @classmethod
+    def from_git_url(cls, git_url: str) -> "_HTTPLFSClient":
+        if git_url.endswith(".git"):
+            url = f"{git_url}/info/lfs"
+        else:
+            url = f"{git_url}.git/info/lfs"
+        return cls(url, git_url=git_url)
+
+    def _get_auth_header(self, *, upload: bool) -> dict:
+        try:
+            creds = Credential(url=self.git_url).fill()
+            if creds.username and creds.password:
+                return {
+                    aiohttp.hdrs.AUTHORIZATION: aiohttp.BasicAuth(
+                        creds.username, creds.password
+                    ).encode()
+                }
+        except CredentialNotFoundError:
+            pass
+        return {}
+
+
+class _SSHLFSClient(LFSClient):
+    _URL_PATTERN = re.compile(
+        r"(?:ssh://)?git@(?P<host>\S+?)(?::(?P<port>\d+))?(?:[:/])(?P<path>\S+?)\.git"
+    )
+
+    def __init__(self, url: str, host: str, port: int, path: str):
+        """
+        Args:
+            url: LFS server URL.
+            host: Git SSH server host.
+            port: Git SSH server port.
+            path: Git project path.
+        """
+        super().__init__(url)
+        self.host = host
+        self.port = port
+        self.path = path
+        self._ssh = _get_ssh_vendor()
+
+    @classmethod
+    def from_git_url(cls, git_url: str) -> "_SSHLFSClient":
+        result = cls._URL_PATTERN.match(git_url)
+        if not result:
+            raise ValueError(f"Invalid Git SSH URL: {git_url}")
+        host, port, path = result.group("host", "port", "path")
+        url = f"https://{host}/{path}.git/info/lfs"
+        return cls(url, host, int(port or 22), path)
+
+    def _get_auth_header(self, *, upload: bool) -> dict:
+        return self._git_lfs_authenticate(
+            self.host, self.port, f"{self.path}.git", upload=upload
+        ).get("header", {})
+
+    def _git_lfs_authenticate(
+        self, host: str, port: int, path: str, *, upload: bool = False
+    ) -> dict:
+        action = "upload" if upload else "download"
+        return json.loads(
+            self._ssh.run_command(
+                command=f"git-lfs-authenticate {path} {action}",
+                host=host,
+                port=port,
+                username="git",
+            ).read()
+        )
+
+
+@contextmanager
+def _as_atomic(to_info: str, create_parents: bool = False) -> Iterator[str]:
+    parent = os.path.dirname(to_info)
+    if create_parents:
+        os.makedirs(parent, exist_ok=True)
+
+    tmp_file = NamedTemporaryFile(dir=parent, delete=False)
+    tmp_file.close()
+    try:
+        yield tmp_file.name
+    except BaseException:
+        with suppress(FileNotFoundError):
+            os.unlink(tmp_file.name)
+        raise
+    else:
+        shutil.move(tmp_file.name, to_info)

scmrepo-3.2.0/src/scmrepo/git/lfs/progress.py

@@ -0,0 +1,159 @@
+import logging
+import sys
+from typing import Any, BinaryIO, Callable, ClassVar, Optional, Union
+
+from fsspec.callbacks import DEFAULT_CALLBACK, Callback, TqdmCallback
+from tqdm import tqdm
+
+from scmrepo.progress import GitProgressEvent
+
+
+class _Tqdm(tqdm):
+    """
+    maximum-compatibility tqdm-based progressbars
+    """
+
+    BAR_FMT_DEFAULT = (
+        "{percentage:3.0f}% {desc}|{bar}|"
+        "{postfix[info]}{n_fmt}/{total_fmt}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}]"
+    )
+    # nested bars should have fixed bar widths to align nicely
+    BAR_FMT_DEFAULT_NESTED = (
+        "{percentage:3.0f}%|{bar:10}|{desc:{ncols_desc}.{ncols_desc}}"
+        "{postfix[info]}{n_fmt}/{total_fmt}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}]"
+    )
+    BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
+    BYTES_DEFAULTS: ClassVar[dict[str, Any]] = {
+        "unit": "B",
+        "unit_scale": True,
+        "unit_divisor": 1024,
+        "miniters": 1,
+    }
+
+    def __init__(  # noqa: PLR0913
+        self,
+        iterable=None,
+        disable=None,
+        level=logging.ERROR,
+        desc=None,
+        leave=False,
+        bar_format=None,
+        bytes=False,  # noqa: A002
+        file=None,
+        total=None,
+        postfix=None,
+        **kwargs,
+    ):
+        kwargs = kwargs.copy()
+        if bytes:
+            kwargs = {**self.BYTES_DEFAULTS, **kwargs}
+        else:
+            kwargs.setdefault("unit_scale", total > 999 if total else True)
+        if file is None:
+            file = sys.stderr
+        super().__init__(
+            iterable=iterable,
+            disable=disable,
+            leave=leave,
+            desc=desc,
+            bar_format="!",
+            lock_args=(False,),
+            total=total,
+            **kwargs,
+        )
+        self.postfix = postfix or {"info": ""}
+        if bar_format is None:
+            if self.__len__():
+                self.bar_format = (
+                    self.BAR_FMT_DEFAULT_NESTED if self.pos else self.BAR_FMT_DEFAULT
+                )
+            else:
+                self.bar_format = self.BAR_FMT_NOTOTAL
+        else:
+            self.bar_format = bar_format
+        self.refresh()
+
+    def update_to(self, current, total=None):
+        if total:
+            self.total = total
+        self.update(current - self.n)
+
+    def close(self):
+        self.postfix["info"] = ""
+        # remove ETA (either unknown or zero); remove completed bar
+        self.bar_format = self.bar_format.replace("<{remaining}", "").replace(
+            "|{bar:10}|", " "
+        )
+        super().close()
+
+    @property
+    def format_dict(self):
+        """inject `ncols_desc` to fill the display width (`ncols`)"""
+        d = super().format_dict
+        ncols = d["ncols"] or 80
+        # assumes `bar_format` has max one of ("ncols_desc" & "ncols_info")
+
+        meter = self.format_meter(  # type: ignore[call-arg]
+            ncols_desc=1, ncols_info=1, **d
+        )
+        ncols_left = ncols - len(meter) + 1
+        ncols_left = max(ncols_left, 0)
+        if ncols_left:
+            d["ncols_desc"] = d["ncols_info"] = ncols_left
+        else:
+            # work-around for zero-width description
+            d["ncols_desc"] = d["ncols_info"] = 1
+            d["prefix"] = ""
+        return d
+
+
+class LFSCallback(Callback):
+    """Callback subclass to generate Git/LFS style progress."""
+
+    def __init__(
+        self,
+        *args,
+        git_progress: Optional[Callable[[GitProgressEvent], None]] = None,
+        direction: str = "Downloading",
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.direction = direction
+        self.git_progress = git_progress
+
+    def call(self, *args, **kwargs):
+        super().call(*args, **kwargs)
+        self._update_git()
+
+    def _update_git(self):
+        if not self.git_progress:
+            return
+        event = GitProgressEvent(
+            phase=f"{self.direction} LFS objects",
+            completed=self.value,
+            total=self.size,
+        )
+        self.git_progress(event)
+
+    def branched(self, path_1: Union[str, BinaryIO], path_2: str, **kwargs):
+        if self.git_progress:
+            return TqdmCallback(
+                tqdm_kwargs={
+                    "desc": path_1 if isinstance(path_1, str) else path_2,
+                    "bytes": True,
+                },
+                tqdm_cls=_Tqdm,
+            )
+        return DEFAULT_CALLBACK
+
+    @classmethod
+    def as_lfs_callback(
+        cls,
+        git_progress: Optional[Callable[[GitProgressEvent], None]] = None,
+        **kwargs,
+    ):
+        if git_progress is None:
+            return DEFAULT_CALLBACK
+        return cls(git_progress=git_progress, **kwargs)

{scmrepo-3.0.0 → scmrepo-3.2.0}/src/scmrepo/git/lfs/smudge.py

@@ -11,7 +11,10 @@ logger = logging.getLogger(__name__)
 
 
 def smudge(
-    storage: "LFSStorage", fobj: BinaryIO, url: Optional[str] = None
+    storage: "LFSStorage",
+    fobj: BinaryIO,
+    url: Optional[str] = None,
+    batch_size: Optional[int] = None,
 ) -> BinaryIO:
     """Wrap the specified binary IO stream and run LFS smudge if necessary."""
     reader = io.BufferedReader(fobj)  # type: ignore[arg-type]

{scmrepo-3.0.0 → scmrepo-3.2.0}/src/scmrepo/git/lfs/storage.py

@@ -20,13 +20,14 @@ class LFSStorage:
         url: str,
         objects: Collection[Pointer],
         progress: Optional[Callable[["GitProgressEvent"], None]] = None,
+        batch_size: Optional[int] = None,
     ):
         from .client import LFSClient
 
         with LFSCallback.as_lfs_callback(progress) as cb:
             cb.set_size(len(objects))
             with LFSClient.from_git_url(url) as client:
-                client.download(self, objects, callback=cb)
+                client.download(self, objects, callback=cb, batch_size=batch_size)
 
     def oid_to_path(self, oid: str):
         return os.path.join(self.path, "objects", oid[0:2], oid[2:4], oid)
@@ -40,6 +41,7 @@ class LFSStorage:
         self,
         obj: Union[Pointer, str],
         fetch_url: Optional[str] = None,
+        batch_size: Optional[int] = None,
         **kwargs,
     ) -> BinaryIO:
         oid = obj if isinstance(obj, str) else obj.oid
@@ -50,7 +52,7 @@ class LFSStorage:
             if not fetch_url or not isinstance(obj, Pointer):
                 raise
         try:
-            self.fetch(fetch_url, [obj])
+            self.fetch(fetch_url, [obj], batch_size=batch_size)
         except BaseException as exc:  # noqa: BLE001
             raise FileNotFoundError(
                 errno.ENOENT, os.strerror(errno.ENOENT), path

{scmrepo-3.0.0 → scmrepo-3.2.0/src/scmrepo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scmrepo
-Version: 3.0.0
+Version: 3.2.0
 Summary: scmrepo
 Author-email: Iterative <support@dvc.org>
 License: Apache-2.0
@@ -19,13 +19,12 @@ Requires-Dist: gitpython>3
 Requires-Dist: dulwich>=0.21.6
 Requires-Dist: pygit2>=1.14.0
 Requires-Dist: pygtrie>=2.3.2
-Requires-Dist: fsspec>=2024.2.0
+Requires-Dist: fsspec[tqdm]>=2024.2.0
 Requires-Dist: pathspec>=0.9.0
 Requires-Dist: asyncssh<3,>=2.13.1
 Requires-Dist: funcy>=1.14
-Requires-Dist: shortuuid>=0.5.0
-Requires-Dist: dvc-objects<5,>=4
-Requires-Dist: dvc-http>=2.29.0
+Requires-Dist: aiohttp-retry>=2.5.0
+Requires-Dist: tqdm
 Provides-Extra: tests
 Requires-Dist: pytest==7.2.0; extra == "tests"
 Requires-Dist: pytest-sugar==0.9.5; extra == "tests"
@@ -40,6 +39,7 @@ Requires-Dist: paramiko==3.3.1; extra == "tests"
 Requires-Dist: types-certifi==2021.10.8.3; extra == "tests"
 Requires-Dist: types-mock==5.1.0.2; extra == "tests"
 Requires-Dist: types-paramiko==3.4.0.20240120; extra == "tests"
+Requires-Dist: types-tqdm; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: scmrepo[tests]; extra == "dev"
 

{scmrepo-3.0.0 → scmrepo-3.2.0}/src/scmrepo.egg-info/requires.txt

@@ -2,13 +2,12 @@ gitpython>3
 dulwich>=0.21.6
 pygit2>=1.14.0
 pygtrie>=2.3.2
-fsspec>=2024.2.0
+fsspec[tqdm]>=2024.2.0
 pathspec>=0.9.0
 asyncssh<3,>=2.13.1
 funcy>=1.14
-shortuuid>=0.5.0
-dvc-objects<5,>=4
-dvc-http>=2.29.0
+aiohttp-retry>=2.5.0
+tqdm
 
 [dev]
 scmrepo[tests]
@@ -26,6 +25,7 @@ paramiko==3.3.1
 types-certifi==2021.10.8.3
 types-mock==5.1.0.2
 types-paramiko==3.4.0.20240120
+types-tqdm
 
 [tests:python_version < "3.10" and implementation_name != "pypy"]
 pytest-docker==2.2.0

scmrepo-3.0.0/src/scmrepo/git/lfs/client.py

@@ -1,171 +0,0 @@
-import logging
-from collections.abc import Iterable
-from contextlib import AbstractContextManager
-from typing import TYPE_CHECKING, Any, Optional
-
-import aiohttp
-from dvc_http import HTTPFileSystem
-from dvc_objects.executors import batch_coros
-from dvc_objects.fs import localfs
-from dvc_objects.fs.utils import as_atomic
-from fsspec.asyn import sync_wrapper
-from fsspec.callbacks import DEFAULT_CALLBACK
-from funcy import cached_property
-
-from scmrepo.git.credentials import Credential, CredentialNotFoundError
-
-from .exceptions import LFSError
-from .pointer import Pointer
-
-if TYPE_CHECKING:
-    from fsspec.callbacks import Callback
-
-    from .storage import LFSStorage
-
-logger = logging.getLogger(__name__)
-
-
-# pylint: disable=abstract-method
-class _LFSFileSystem(HTTPFileSystem):
-    def _prepare_credentials(self, **config):
-        return {}
-
-
-class LFSClient(AbstractContextManager):
-    """Naive read-only LFS HTTP client."""
-
-    JSON_CONTENT_TYPE = "application/vnd.git-lfs+json"
-
-    def __init__(
-        self,
-        url: str,
-        git_url: Optional[str] = None,
-        headers: Optional[dict[str, str]] = None,
-    ):
-        """
-        Args:
-            url: LFS server URL.
-        """
-        self.url = url
-        self.git_url = git_url
-        self.headers: dict[str, str] = headers or {}
-
-    def __exit__(self, *args, **kwargs):
-        self.close()
-
-    @cached_property
-    def fs(self) -> "_LFSFileSystem":
-        return _LFSFileSystem()
-
-    @property
-    def httpfs(self) -> "HTTPFileSystem":
-        return self.fs.fs
-
-    @property
-    def loop(self):
-        return self.httpfs.loop
-
-    @classmethod
-    def from_git_url(cls, git_url: str) -> "LFSClient":
-        if git_url.endswith(".git"):
-            url = f"{git_url}/info/lfs"
-        else:
-            url = f"{git_url}.git/info/lfs"
-        return cls(url, git_url=git_url)
-
-    def close(self):
-        pass
-
-    def _get_auth(self) -> Optional[aiohttp.BasicAuth]:
-        try:
-            creds = Credential(url=self.git_url).fill()
-            if creds.username and creds.password:
-                return aiohttp.BasicAuth(creds.username, creds.password)
-        except CredentialNotFoundError:
-            pass
-        return None
-
-    async def _set_session(self) -> aiohttp.ClientSession:
-        return await self.fs.fs.set_session()
-
-    async def _batch_request(
-        self,
-        objects: Iterable[Pointer],
-        upload: bool = False,
-        ref: Optional[str] = None,
-        hash_algo: str = "sha256",
-    ) -> dict[str, Any]:
-        """Send LFS API /objects/batch request."""
-        url = f"{self.url}/objects/batch"
-        body: dict[str, Any] = {
-            "operation": "upload" if upload else "download",
-            "transfers": ["basic"],
-            "objects": [{"oid": obj.oid, "size": obj.size} for obj in objects],
-            "hash_algo": hash_algo,
-        }
-        if ref:
-            body["ref"] = [{"name": ref}]
-        session = await self._set_session()
-        headers = dict(self.headers)
-        headers["Accept"] = self.JSON_CONTENT_TYPE
-        headers["Content-Type"] = self.JSON_CONTENT_TYPE
-        try:
-            async with session.post(
-                url,
-                headers=headers,
-                json=body,
-                raise_for_status=True,
-            ) as resp:
-                data = await resp.json()
-        except aiohttp.ClientResponseError as exc:
-            if exc.status != 401:
-                raise
-            auth = self._get_auth()
-            if auth is None:
-                raise
-            async with session.post(
-                url,
-                auth=auth,
-                headers=headers,
-                json=body,
-                raise_for_status=True,
-            ) as resp:
-                data = await resp.json()
-        return data
-
-    async def _download(
-        self,
-        storage: "LFSStorage",
-        objects: Iterable[Pointer],
-        callback: "Callback" = DEFAULT_CALLBACK,
-        **kwargs,
-    ):
-        async def _get_one(from_path: str, to_path: str, **kwargs):
-            with as_atomic(localfs, to_path, create_parents=True) as tmp_file:
-                with callback.branched(from_path, tmp_file) as child:
-                    await self.httpfs._get_file(
-                        from_path, tmp_file, callback=child, **kwargs
-                    )  # pylint: disable=protected-access
-                    callback.relative_update()
-
-        resp_data = await self._batch_request(objects, **kwargs)
-        if resp_data.get("transfer", "basic") != "basic":
-            raise LFSError("Unsupported LFS transfer type")
-        coros = []
-        for data in resp_data.get("objects", []):
-            obj = Pointer(data["oid"], data["size"])
-            download = data.get("actions", {}).get("download", {})
-            url = download.get("href")
-            if not url:
-                logger.debug("No download URL for LFS object '%s'", obj)
-                continue
-            headers = download.get("header", {})
-            to_path = storage.oid_to_path(obj.oid)
-            coros.append(_get_one(url, to_path, headers=headers))
-        for result in await batch_coros(
-            coros, batch_size=self.fs.jobs, return_exceptions=True
-        ):
-            if isinstance(result, BaseException):
-                raise result
-
-    download = sync_wrapper(_download)

scmrepo-3.0.0/src/scmrepo/git/lfs/progress.py

@@ -1,52 +0,0 @@
-from typing import BinaryIO, Callable, Optional, Union
-
-from dvc_objects.fs.callbacks import TqdmCallback
-from fsspec.callbacks import DEFAULT_CALLBACK, Callback
-
-from scmrepo.progress import GitProgressEvent
-
-
-class LFSCallback(Callback):
-    """Callback subclass to generate Git/LFS style progress."""
-
-    def __init__(
-        self,
-        *args,
-        git_progress: Optional[Callable[[GitProgressEvent], None]] = None,
-        direction: str = "Downloading",
-        **kwargs,
-    ):
-        super().__init__(*args, **kwargs)
-        self.direction = direction
-        self.git_progress = git_progress
-
-    def call(self, *args, **kwargs):
-        super().call(*args, **kwargs)
-        self._update_git()
-
-    def _update_git(self):
-        if not self.git_progress:
-            return
-        event = GitProgressEvent(
-            phase=f"{self.direction} LFS objects",
-            completed=self.value,
-            total=self.size,
-        )
-        self.git_progress(event)
-
-    def branched(self, path_1: Union[str, BinaryIO], path_2: str, **kwargs):
-        if self.git_progress:
-            return TqdmCallback(
-                bytes=True, desc=path_1 if isinstance(path_1, str) else path_2
-            )
-        return DEFAULT_CALLBACK
-
-    @classmethod
-    def as_lfs_callback(
-        cls,
-        git_progress: Optional[Callable[[GitProgressEvent], None]] = None,
-        **kwargs,
-    ):
-        if git_progress is None:
-            return DEFAULT_CALLBACK
-        return cls(git_progress=git_progress, **kwargs)