scmrepo 3.0.0__py3-none-any.whl → 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of scmrepo might be problematic. Click here for more details.
- scmrepo/git/backend/dulwich/__init__.py +14 -1
- scmrepo/git/lfs/client.py +160 -56
- scmrepo/git/lfs/progress.py +111 -4
- scmrepo/git/lfs/smudge.py +4 -1
- scmrepo/git/lfs/storage.py +4 -2
- {scmrepo-3.0.0.dist-info → scmrepo-3.2.0.dist-info}/METADATA +5 -5
- {scmrepo-3.0.0.dist-info → scmrepo-3.2.0.dist-info}/RECORD +10 -10
- {scmrepo-3.0.0.dist-info → scmrepo-3.2.0.dist-info}/LICENSE +0 -0
- {scmrepo-3.0.0.dist-info → scmrepo-3.2.0.dist-info}/WHEEL +0 -0
- {scmrepo-3.0.0.dist-info → scmrepo-3.2.0.dist-info}/top_level.txt +0 -0
|
@@ -842,7 +842,7 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
|
|
|
842
842
|
if revision and revision not in rev_mapping:
|
|
843
843
|
rev_mapping[revision] = ref
|
|
844
844
|
for rev in revs:
|
|
845
|
-
results[rev] = rev_mapping.get(rev
|
|
845
|
+
results[rev] = rev_mapping.get(rev)
|
|
846
846
|
return results
|
|
847
847
|
|
|
848
848
|
def diff(self, rev_a: str, rev_b: str, binary=False) -> str:
|
|
@@ -978,3 +978,16 @@ def _parse_identity(identity: str) -> tuple[str, str]:
|
|
|
978
978
|
if not m:
|
|
979
979
|
raise SCMError("Could not parse tagger identity '{identity}'")
|
|
980
980
|
return m.group("name"), m.group("email")
|
|
981
|
+
|
|
982
|
+
|
|
983
|
+
def ls_remote(url: str) -> dict[str, str]:
|
|
984
|
+
from dulwich import porcelain
|
|
985
|
+
from dulwich.client import HTTPUnauthorized
|
|
986
|
+
|
|
987
|
+
try:
|
|
988
|
+
refs = porcelain.ls_remote(url)
|
|
989
|
+
return {os.fsdecode(ref): sha.decode("ascii") for ref, sha in refs.items()}
|
|
990
|
+
except HTTPUnauthorized as exc:
|
|
991
|
+
raise AuthError(url) from exc
|
|
992
|
+
except Exception as exc: # noqa: BLE001
|
|
993
|
+
raise InvalidRemote(url) from exc
|
scmrepo/git/lfs/client.py
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import logging
|
|
2
|
-
|
|
3
|
-
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import shutil
|
|
6
|
+
from abc import abstractmethod
|
|
7
|
+
from collections.abc import Iterable, Iterator
|
|
8
|
+
from contextlib import AbstractContextManager, contextmanager, suppress
|
|
9
|
+
from tempfile import NamedTemporaryFile
|
|
4
10
|
from typing import TYPE_CHECKING, Any, Optional
|
|
5
11
|
|
|
6
12
|
import aiohttp
|
|
7
|
-
from
|
|
8
|
-
from
|
|
9
|
-
from dvc_objects.fs import localfs
|
|
10
|
-
from dvc_objects.fs.utils import as_atomic
|
|
11
|
-
from fsspec.asyn import sync_wrapper
|
|
13
|
+
from aiohttp_retry import ExponentialRetry, RetryClient
|
|
14
|
+
from fsspec.asyn import _run_coros_in_chunks, sync_wrapper
|
|
12
15
|
from fsspec.callbacks import DEFAULT_CALLBACK
|
|
16
|
+
from fsspec.implementations.http import HTTPFileSystem
|
|
13
17
|
from funcy import cached_property
|
|
14
18
|
|
|
19
|
+
from scmrepo.git.backend.dulwich import _get_ssh_vendor
|
|
15
20
|
from scmrepo.git.credentials import Credential, CredentialNotFoundError
|
|
16
21
|
|
|
17
22
|
from .exceptions import LFSError
|
|
@@ -25,68 +30,69 @@ if TYPE_CHECKING:
|
|
|
25
30
|
logger = logging.getLogger(__name__)
|
|
26
31
|
|
|
27
32
|
|
|
28
|
-
# pylint: disable=abstract-method
|
|
29
|
-
class _LFSFileSystem(HTTPFileSystem):
|
|
30
|
-
def _prepare_credentials(self, **config):
|
|
31
|
-
return {}
|
|
32
|
-
|
|
33
|
-
|
|
34
33
|
class LFSClient(AbstractContextManager):
|
|
35
34
|
"""Naive read-only LFS HTTP client."""
|
|
36
35
|
|
|
37
36
|
JSON_CONTENT_TYPE = "application/vnd.git-lfs+json"
|
|
38
37
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
):
|
|
38
|
+
_REQUEST_TIMEOUT = 60
|
|
39
|
+
_SESSION_RETRIES = 5
|
|
40
|
+
_SESSION_BACKOFF_FACTOR = 0.1
|
|
41
|
+
|
|
42
|
+
def __init__(self, url: str):
|
|
45
43
|
"""
|
|
46
44
|
Args:
|
|
47
45
|
url: LFS server URL.
|
|
48
46
|
"""
|
|
49
47
|
self.url = url
|
|
50
|
-
self.git_url = git_url
|
|
51
|
-
self.headers: dict[str, str] = headers or {}
|
|
52
48
|
|
|
53
49
|
def __exit__(self, *args, **kwargs):
|
|
54
50
|
self.close()
|
|
55
51
|
|
|
56
52
|
@cached_property
|
|
57
|
-
def
|
|
58
|
-
|
|
53
|
+
def _fs(self) -> HTTPFileSystem:
|
|
54
|
+
async def get_client(**kwargs):
|
|
55
|
+
return RetryClient(
|
|
56
|
+
connector=aiohttp.TCPConnector(
|
|
57
|
+
# Force cleanup of closed SSL transports.
|
|
58
|
+
# See https://github.com/iterative/dvc/issues/7414
|
|
59
|
+
enable_cleanup_closed=True,
|
|
60
|
+
),
|
|
61
|
+
timeout=aiohttp.ClientTimeout(
|
|
62
|
+
total=None,
|
|
63
|
+
connect=self._REQUEST_TIMEOUT,
|
|
64
|
+
sock_connect=self._REQUEST_TIMEOUT,
|
|
65
|
+
sock_read=self._REQUEST_TIMEOUT,
|
|
66
|
+
),
|
|
67
|
+
retry_options=ExponentialRetry(
|
|
68
|
+
attempts=self._SESSION_RETRIES,
|
|
69
|
+
factor=self._SESSION_BACKOFF_FACTOR,
|
|
70
|
+
max_timeout=self._REQUEST_TIMEOUT,
|
|
71
|
+
exceptions={aiohttp.ClientError},
|
|
72
|
+
),
|
|
73
|
+
**kwargs,
|
|
74
|
+
)
|
|
59
75
|
|
|
60
|
-
|
|
61
|
-
def httpfs(self) -> "HTTPFileSystem":
|
|
62
|
-
return self.fs.fs
|
|
76
|
+
return HTTPFileSystem(get_client=get_client)
|
|
63
77
|
|
|
64
78
|
@property
|
|
65
79
|
def loop(self):
|
|
66
|
-
return self.
|
|
80
|
+
return self._fs.loop
|
|
67
81
|
|
|
68
82
|
@classmethod
|
|
69
83
|
def from_git_url(cls, git_url: str) -> "LFSClient":
|
|
70
|
-
if git_url.
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
84
|
+
if git_url.startswith(("ssh://", "git@")):
|
|
85
|
+
return _SSHLFSClient.from_git_url(git_url)
|
|
86
|
+
if git_url.startswith("https://"):
|
|
87
|
+
return _HTTPLFSClient.from_git_url(git_url)
|
|
88
|
+
raise NotImplementedError(f"Unsupported Git URL: {git_url}")
|
|
75
89
|
|
|
76
90
|
def close(self):
|
|
77
91
|
pass
|
|
78
92
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
if creds.username and creds.password:
|
|
83
|
-
return aiohttp.BasicAuth(creds.username, creds.password)
|
|
84
|
-
except CredentialNotFoundError:
|
|
85
|
-
pass
|
|
86
|
-
return None
|
|
87
|
-
|
|
88
|
-
async def _set_session(self) -> aiohttp.ClientSession:
|
|
89
|
-
return await self.fs.fs.set_session()
|
|
93
|
+
@abstractmethod
|
|
94
|
+
def _get_auth_header(self, *, upload: bool) -> dict:
|
|
95
|
+
...
|
|
90
96
|
|
|
91
97
|
async def _batch_request(
|
|
92
98
|
self,
|
|
@@ -105,10 +111,11 @@ class LFSClient(AbstractContextManager):
|
|
|
105
111
|
}
|
|
106
112
|
if ref:
|
|
107
113
|
body["ref"] = [{"name": ref}]
|
|
108
|
-
session = await self.
|
|
109
|
-
headers =
|
|
110
|
-
|
|
111
|
-
|
|
114
|
+
session = await self._fs.set_session()
|
|
115
|
+
headers = {
|
|
116
|
+
"Accept": self.JSON_CONTENT_TYPE,
|
|
117
|
+
"Content-Type": self.JSON_CONTENT_TYPE,
|
|
118
|
+
}
|
|
112
119
|
try:
|
|
113
120
|
async with session.post(
|
|
114
121
|
url,
|
|
@@ -120,13 +127,12 @@ class LFSClient(AbstractContextManager):
|
|
|
120
127
|
except aiohttp.ClientResponseError as exc:
|
|
121
128
|
if exc.status != 401:
|
|
122
129
|
raise
|
|
123
|
-
|
|
124
|
-
if
|
|
130
|
+
auth_header = self._get_auth_header(upload=upload)
|
|
131
|
+
if not auth_header:
|
|
125
132
|
raise
|
|
126
133
|
async with session.post(
|
|
127
134
|
url,
|
|
128
|
-
|
|
129
|
-
headers=headers,
|
|
135
|
+
headers={**headers, **auth_header},
|
|
130
136
|
json=body,
|
|
131
137
|
raise_for_status=True,
|
|
132
138
|
) as resp:
|
|
@@ -138,14 +144,15 @@ class LFSClient(AbstractContextManager):
|
|
|
138
144
|
storage: "LFSStorage",
|
|
139
145
|
objects: Iterable[Pointer],
|
|
140
146
|
callback: "Callback" = DEFAULT_CALLBACK,
|
|
147
|
+
batch_size: Optional[int] = None,
|
|
141
148
|
**kwargs,
|
|
142
149
|
):
|
|
143
150
|
async def _get_one(from_path: str, to_path: str, **kwargs):
|
|
144
|
-
with
|
|
151
|
+
with _as_atomic(to_path, create_parents=True) as tmp_file:
|
|
145
152
|
with callback.branched(from_path, tmp_file) as child:
|
|
146
|
-
await self.
|
|
153
|
+
await self._fs._get_file(
|
|
147
154
|
from_path, tmp_file, callback=child, **kwargs
|
|
148
|
-
)
|
|
155
|
+
)
|
|
149
156
|
callback.relative_update()
|
|
150
157
|
|
|
151
158
|
resp_data = await self._batch_request(objects, **kwargs)
|
|
@@ -162,10 +169,107 @@ class LFSClient(AbstractContextManager):
|
|
|
162
169
|
headers = download.get("header", {})
|
|
163
170
|
to_path = storage.oid_to_path(obj.oid)
|
|
164
171
|
coros.append(_get_one(url, to_path, headers=headers))
|
|
165
|
-
for result in await
|
|
166
|
-
coros, batch_size=
|
|
172
|
+
for result in await _run_coros_in_chunks(
|
|
173
|
+
coros, batch_size=batch_size, return_exceptions=True
|
|
167
174
|
):
|
|
168
175
|
if isinstance(result, BaseException):
|
|
169
176
|
raise result
|
|
170
177
|
|
|
171
178
|
download = sync_wrapper(_download)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class _HTTPLFSClient(LFSClient):
|
|
182
|
+
def __init__(self, url: str, git_url: str):
|
|
183
|
+
"""
|
|
184
|
+
Args:
|
|
185
|
+
url: LFS server URL.
|
|
186
|
+
git_url: Git HTTP URL.
|
|
187
|
+
"""
|
|
188
|
+
super().__init__(url)
|
|
189
|
+
self.git_url = git_url
|
|
190
|
+
|
|
191
|
+
@classmethod
|
|
192
|
+
def from_git_url(cls, git_url: str) -> "_HTTPLFSClient":
|
|
193
|
+
if git_url.endswith(".git"):
|
|
194
|
+
url = f"{git_url}/info/lfs"
|
|
195
|
+
else:
|
|
196
|
+
url = f"{git_url}.git/info/lfs"
|
|
197
|
+
return cls(url, git_url=git_url)
|
|
198
|
+
|
|
199
|
+
def _get_auth_header(self, *, upload: bool) -> dict:
|
|
200
|
+
try:
|
|
201
|
+
creds = Credential(url=self.git_url).fill()
|
|
202
|
+
if creds.username and creds.password:
|
|
203
|
+
return {
|
|
204
|
+
aiohttp.hdrs.AUTHORIZATION: aiohttp.BasicAuth(
|
|
205
|
+
creds.username, creds.password
|
|
206
|
+
).encode()
|
|
207
|
+
}
|
|
208
|
+
except CredentialNotFoundError:
|
|
209
|
+
pass
|
|
210
|
+
return {}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
class _SSHLFSClient(LFSClient):
|
|
214
|
+
_URL_PATTERN = re.compile(
|
|
215
|
+
r"(?:ssh://)?git@(?P<host>\S+?)(?::(?P<port>\d+))?(?:[:/])(?P<path>\S+?)\.git"
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
def __init__(self, url: str, host: str, port: int, path: str):
|
|
219
|
+
"""
|
|
220
|
+
Args:
|
|
221
|
+
url: LFS server URL.
|
|
222
|
+
host: Git SSH server host.
|
|
223
|
+
port: Git SSH server port.
|
|
224
|
+
path: Git project path.
|
|
225
|
+
"""
|
|
226
|
+
super().__init__(url)
|
|
227
|
+
self.host = host
|
|
228
|
+
self.port = port
|
|
229
|
+
self.path = path
|
|
230
|
+
self._ssh = _get_ssh_vendor()
|
|
231
|
+
|
|
232
|
+
@classmethod
|
|
233
|
+
def from_git_url(cls, git_url: str) -> "_SSHLFSClient":
|
|
234
|
+
result = cls._URL_PATTERN.match(git_url)
|
|
235
|
+
if not result:
|
|
236
|
+
raise ValueError(f"Invalid Git SSH URL: {git_url}")
|
|
237
|
+
host, port, path = result.group("host", "port", "path")
|
|
238
|
+
url = f"https://{host}/{path}.git/info/lfs"
|
|
239
|
+
return cls(url, host, int(port or 22), path)
|
|
240
|
+
|
|
241
|
+
def _get_auth_header(self, *, upload: bool) -> dict:
|
|
242
|
+
return self._git_lfs_authenticate(
|
|
243
|
+
self.host, self.port, f"{self.path}.git", upload=upload
|
|
244
|
+
).get("header", {})
|
|
245
|
+
|
|
246
|
+
def _git_lfs_authenticate(
|
|
247
|
+
self, host: str, port: int, path: str, *, upload: bool = False
|
|
248
|
+
) -> dict:
|
|
249
|
+
action = "upload" if upload else "download"
|
|
250
|
+
return json.loads(
|
|
251
|
+
self._ssh.run_command(
|
|
252
|
+
command=f"git-lfs-authenticate {path} {action}",
|
|
253
|
+
host=host,
|
|
254
|
+
port=port,
|
|
255
|
+
username="git",
|
|
256
|
+
).read()
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
@contextmanager
|
|
261
|
+
def _as_atomic(to_info: str, create_parents: bool = False) -> Iterator[str]:
|
|
262
|
+
parent = os.path.dirname(to_info)
|
|
263
|
+
if create_parents:
|
|
264
|
+
os.makedirs(parent, exist_ok=True)
|
|
265
|
+
|
|
266
|
+
tmp_file = NamedTemporaryFile(dir=parent, delete=False)
|
|
267
|
+
tmp_file.close()
|
|
268
|
+
try:
|
|
269
|
+
yield tmp_file.name
|
|
270
|
+
except BaseException:
|
|
271
|
+
with suppress(FileNotFoundError):
|
|
272
|
+
os.unlink(tmp_file.name)
|
|
273
|
+
raise
|
|
274
|
+
else:
|
|
275
|
+
shutil.move(tmp_file.name, to_info)
|
scmrepo/git/lfs/progress.py
CHANGED
|
@@ -1,11 +1,114 @@
|
|
|
1
|
-
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
from typing import Any, BinaryIO, Callable, ClassVar, Optional, Union
|
|
2
4
|
|
|
3
|
-
from
|
|
4
|
-
from
|
|
5
|
+
from fsspec.callbacks import DEFAULT_CALLBACK, Callback, TqdmCallback
|
|
6
|
+
from tqdm import tqdm
|
|
5
7
|
|
|
6
8
|
from scmrepo.progress import GitProgressEvent
|
|
7
9
|
|
|
8
10
|
|
|
11
|
+
class _Tqdm(tqdm):
|
|
12
|
+
"""
|
|
13
|
+
maximum-compatibility tqdm-based progressbars
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
BAR_FMT_DEFAULT = (
|
|
17
|
+
"{percentage:3.0f}% {desc}|{bar}|"
|
|
18
|
+
"{postfix[info]}{n_fmt}/{total_fmt}"
|
|
19
|
+
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
|
|
20
|
+
)
|
|
21
|
+
# nested bars should have fixed bar widths to align nicely
|
|
22
|
+
BAR_FMT_DEFAULT_NESTED = (
|
|
23
|
+
"{percentage:3.0f}%|{bar:10}|{desc:{ncols_desc}.{ncols_desc}}"
|
|
24
|
+
"{postfix[info]}{n_fmt}/{total_fmt}"
|
|
25
|
+
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
|
|
26
|
+
)
|
|
27
|
+
BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
|
|
28
|
+
BYTES_DEFAULTS: ClassVar[dict[str, Any]] = {
|
|
29
|
+
"unit": "B",
|
|
30
|
+
"unit_scale": True,
|
|
31
|
+
"unit_divisor": 1024,
|
|
32
|
+
"miniters": 1,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
def __init__( # noqa: PLR0913
|
|
36
|
+
self,
|
|
37
|
+
iterable=None,
|
|
38
|
+
disable=None,
|
|
39
|
+
level=logging.ERROR,
|
|
40
|
+
desc=None,
|
|
41
|
+
leave=False,
|
|
42
|
+
bar_format=None,
|
|
43
|
+
bytes=False, # noqa: A002
|
|
44
|
+
file=None,
|
|
45
|
+
total=None,
|
|
46
|
+
postfix=None,
|
|
47
|
+
**kwargs,
|
|
48
|
+
):
|
|
49
|
+
kwargs = kwargs.copy()
|
|
50
|
+
if bytes:
|
|
51
|
+
kwargs = {**self.BYTES_DEFAULTS, **kwargs}
|
|
52
|
+
else:
|
|
53
|
+
kwargs.setdefault("unit_scale", total > 999 if total else True)
|
|
54
|
+
if file is None:
|
|
55
|
+
file = sys.stderr
|
|
56
|
+
super().__init__(
|
|
57
|
+
iterable=iterable,
|
|
58
|
+
disable=disable,
|
|
59
|
+
leave=leave,
|
|
60
|
+
desc=desc,
|
|
61
|
+
bar_format="!",
|
|
62
|
+
lock_args=(False,),
|
|
63
|
+
total=total,
|
|
64
|
+
**kwargs,
|
|
65
|
+
)
|
|
66
|
+
self.postfix = postfix or {"info": ""}
|
|
67
|
+
if bar_format is None:
|
|
68
|
+
if self.__len__():
|
|
69
|
+
self.bar_format = (
|
|
70
|
+
self.BAR_FMT_DEFAULT_NESTED if self.pos else self.BAR_FMT_DEFAULT
|
|
71
|
+
)
|
|
72
|
+
else:
|
|
73
|
+
self.bar_format = self.BAR_FMT_NOTOTAL
|
|
74
|
+
else:
|
|
75
|
+
self.bar_format = bar_format
|
|
76
|
+
self.refresh()
|
|
77
|
+
|
|
78
|
+
def update_to(self, current, total=None):
|
|
79
|
+
if total:
|
|
80
|
+
self.total = total
|
|
81
|
+
self.update(current - self.n)
|
|
82
|
+
|
|
83
|
+
def close(self):
|
|
84
|
+
self.postfix["info"] = ""
|
|
85
|
+
# remove ETA (either unknown or zero); remove completed bar
|
|
86
|
+
self.bar_format = self.bar_format.replace("<{remaining}", "").replace(
|
|
87
|
+
"|{bar:10}|", " "
|
|
88
|
+
)
|
|
89
|
+
super().close()
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def format_dict(self):
|
|
93
|
+
"""inject `ncols_desc` to fill the display width (`ncols`)"""
|
|
94
|
+
d = super().format_dict
|
|
95
|
+
ncols = d["ncols"] or 80
|
|
96
|
+
# assumes `bar_format` has max one of ("ncols_desc" & "ncols_info")
|
|
97
|
+
|
|
98
|
+
meter = self.format_meter( # type: ignore[call-arg]
|
|
99
|
+
ncols_desc=1, ncols_info=1, **d
|
|
100
|
+
)
|
|
101
|
+
ncols_left = ncols - len(meter) + 1
|
|
102
|
+
ncols_left = max(ncols_left, 0)
|
|
103
|
+
if ncols_left:
|
|
104
|
+
d["ncols_desc"] = d["ncols_info"] = ncols_left
|
|
105
|
+
else:
|
|
106
|
+
# work-around for zero-width description
|
|
107
|
+
d["ncols_desc"] = d["ncols_info"] = 1
|
|
108
|
+
d["prefix"] = ""
|
|
109
|
+
return d
|
|
110
|
+
|
|
111
|
+
|
|
9
112
|
class LFSCallback(Callback):
|
|
10
113
|
"""Callback subclass to generate Git/LFS style progress."""
|
|
11
114
|
|
|
@@ -37,7 +140,11 @@ class LFSCallback(Callback):
|
|
|
37
140
|
def branched(self, path_1: Union[str, BinaryIO], path_2: str, **kwargs):
|
|
38
141
|
if self.git_progress:
|
|
39
142
|
return TqdmCallback(
|
|
40
|
-
|
|
143
|
+
tqdm_kwargs={
|
|
144
|
+
"desc": path_1 if isinstance(path_1, str) else path_2,
|
|
145
|
+
"bytes": True,
|
|
146
|
+
},
|
|
147
|
+
tqdm_cls=_Tqdm,
|
|
41
148
|
)
|
|
42
149
|
return DEFAULT_CALLBACK
|
|
43
150
|
|
scmrepo/git/lfs/smudge.py
CHANGED
|
@@ -11,7 +11,10 @@ logger = logging.getLogger(__name__)
|
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def smudge(
|
|
14
|
-
storage: "LFSStorage",
|
|
14
|
+
storage: "LFSStorage",
|
|
15
|
+
fobj: BinaryIO,
|
|
16
|
+
url: Optional[str] = None,
|
|
17
|
+
batch_size: Optional[int] = None,
|
|
15
18
|
) -> BinaryIO:
|
|
16
19
|
"""Wrap the specified binary IO stream and run LFS smudge if necessary."""
|
|
17
20
|
reader = io.BufferedReader(fobj) # type: ignore[arg-type]
|
scmrepo/git/lfs/storage.py
CHANGED
|
@@ -20,13 +20,14 @@ class LFSStorage:
|
|
|
20
20
|
url: str,
|
|
21
21
|
objects: Collection[Pointer],
|
|
22
22
|
progress: Optional[Callable[["GitProgressEvent"], None]] = None,
|
|
23
|
+
batch_size: Optional[int] = None,
|
|
23
24
|
):
|
|
24
25
|
from .client import LFSClient
|
|
25
26
|
|
|
26
27
|
with LFSCallback.as_lfs_callback(progress) as cb:
|
|
27
28
|
cb.set_size(len(objects))
|
|
28
29
|
with LFSClient.from_git_url(url) as client:
|
|
29
|
-
client.download(self, objects, callback=cb)
|
|
30
|
+
client.download(self, objects, callback=cb, batch_size=batch_size)
|
|
30
31
|
|
|
31
32
|
def oid_to_path(self, oid: str):
|
|
32
33
|
return os.path.join(self.path, "objects", oid[0:2], oid[2:4], oid)
|
|
@@ -40,6 +41,7 @@ class LFSStorage:
|
|
|
40
41
|
self,
|
|
41
42
|
obj: Union[Pointer, str],
|
|
42
43
|
fetch_url: Optional[str] = None,
|
|
44
|
+
batch_size: Optional[int] = None,
|
|
43
45
|
**kwargs,
|
|
44
46
|
) -> BinaryIO:
|
|
45
47
|
oid = obj if isinstance(obj, str) else obj.oid
|
|
@@ -50,7 +52,7 @@ class LFSStorage:
|
|
|
50
52
|
if not fetch_url or not isinstance(obj, Pointer):
|
|
51
53
|
raise
|
|
52
54
|
try:
|
|
53
|
-
self.fetch(fetch_url, [obj])
|
|
55
|
+
self.fetch(fetch_url, [obj], batch_size=batch_size)
|
|
54
56
|
except BaseException as exc: # noqa: BLE001
|
|
55
57
|
raise FileNotFoundError(
|
|
56
58
|
errno.ENOENT, os.strerror(errno.ENOENT), path
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: scmrepo
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.2.0
|
|
4
4
|
Summary: scmrepo
|
|
5
5
|
Author-email: Iterative <support@dvc.org>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -19,13 +19,12 @@ Requires-Dist: gitpython >3
|
|
|
19
19
|
Requires-Dist: dulwich >=0.21.6
|
|
20
20
|
Requires-Dist: pygit2 >=1.14.0
|
|
21
21
|
Requires-Dist: pygtrie >=2.3.2
|
|
22
|
-
Requires-Dist: fsspec >=2024.2.0
|
|
22
|
+
Requires-Dist: fsspec[tqdm] >=2024.2.0
|
|
23
23
|
Requires-Dist: pathspec >=0.9.0
|
|
24
24
|
Requires-Dist: asyncssh <3,>=2.13.1
|
|
25
25
|
Requires-Dist: funcy >=1.14
|
|
26
|
-
Requires-Dist:
|
|
27
|
-
Requires-Dist:
|
|
28
|
-
Requires-Dist: dvc-http >=2.29.0
|
|
26
|
+
Requires-Dist: aiohttp-retry >=2.5.0
|
|
27
|
+
Requires-Dist: tqdm
|
|
29
28
|
Provides-Extra: dev
|
|
30
29
|
Requires-Dist: scmrepo[tests] ; extra == 'dev'
|
|
31
30
|
Provides-Extra: tests
|
|
@@ -41,6 +40,7 @@ Requires-Dist: paramiko ==3.3.1 ; extra == 'tests'
|
|
|
41
40
|
Requires-Dist: types-certifi ==2021.10.8.3 ; extra == 'tests'
|
|
42
41
|
Requires-Dist: types-mock ==5.1.0.2 ; extra == 'tests'
|
|
43
42
|
Requires-Dist: types-paramiko ==3.4.0.20240120 ; extra == 'tests'
|
|
43
|
+
Requires-Dist: types-tqdm ; extra == 'tests'
|
|
44
44
|
Requires-Dist: pytest-docker ==2.2.0 ; (python_version < "3.10" and implementation_name != "pypy") and extra == 'tests'
|
|
45
45
|
|
|
46
46
|
scmrepo
|
|
@@ -15,23 +15,23 @@ scmrepo/git/stash.py,sha256=rnZDeOsO9P-k2e7ulCLUmZKSxSCxaRKl3XJlh97F084,2801
|
|
|
15
15
|
scmrepo/git/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
16
|
scmrepo/git/backend/base.py,sha256=acxuSQ0Z-UGNkGraCdLQxBxDHbTdYWi-FzXtwdb-1O8,13535
|
|
17
17
|
scmrepo/git/backend/gitpython.py,sha256=6L47iX1SmqfM04_Ghwd_DEHegOKewCLu-5MTkBZO_Zo,25312
|
|
18
|
-
scmrepo/git/backend/dulwich/__init__.py,sha256=
|
|
18
|
+
scmrepo/git/backend/dulwich/__init__.py,sha256=oEajaQDbmVs5Sl90tM4F8rwdvPZagXbgx6aAAL-jvKg,34782
|
|
19
19
|
scmrepo/git/backend/dulwich/asyncssh_vendor.py,sha256=OuZ_bWe5-LiZCIMwBRaX_uj03oEcrRgr1uf9i2Xv4Fk,11497
|
|
20
20
|
scmrepo/git/backend/dulwich/client.py,sha256=bcDroljSvNz6s5WWv9UVvZHKkOJOVTK_zU7YCq62TN4,2360
|
|
21
21
|
scmrepo/git/backend/pygit2/__init__.py,sha256=tapIRAh--nzGUcVGijaMSal2ZPscab9c1mHdqiUIxBU,37004
|
|
22
22
|
scmrepo/git/backend/pygit2/callbacks.py,sha256=Ky4YmUPhv9xjU_44ypBYIcaVHJixzaGb6t9HIeUmBP4,2751
|
|
23
23
|
scmrepo/git/backend/pygit2/filter.py,sha256=2NlWfQ7soXN1H7Es6-LctE74hpj3QKQTlYqXRH83VpM,2128
|
|
24
24
|
scmrepo/git/lfs/__init__.py,sha256=at5blRIKnKpg_g5dLRDsGWBFi6SbucRlF_DX6aAkGtE,257
|
|
25
|
-
scmrepo/git/lfs/client.py,sha256=
|
|
25
|
+
scmrepo/git/lfs/client.py,sha256=I3HX3X3-2ZR3wetb6h19bGcV5rx-Wp2zLwECc-mlKhs,8926
|
|
26
26
|
scmrepo/git/lfs/exceptions.py,sha256=cLlImmPXWJJUl44S4xcRBa2T9wYRkWTaKQGwJylwOhA,77
|
|
27
27
|
scmrepo/git/lfs/fetch.py,sha256=ADNpskbDrvMI7ru4AiOf_c1gfw8TQ7Wct0EiN2Pq-qc,4683
|
|
28
28
|
scmrepo/git/lfs/object.py,sha256=rAYY_z9EYoHPfbpF1QHwL7ecYgaETPyCl-zBx0E1oIQ,337
|
|
29
29
|
scmrepo/git/lfs/pointer.py,sha256=BcVbtjoOUG9cEzyJSJDeweqehGZvq43P6NNLDYUGYEI,3181
|
|
30
|
-
scmrepo/git/lfs/progress.py,sha256=
|
|
31
|
-
scmrepo/git/lfs/smudge.py,sha256=
|
|
32
|
-
scmrepo/git/lfs/storage.py,sha256=
|
|
33
|
-
scmrepo-3.
|
|
34
|
-
scmrepo-3.
|
|
35
|
-
scmrepo-3.
|
|
36
|
-
scmrepo-3.
|
|
37
|
-
scmrepo-3.
|
|
30
|
+
scmrepo/git/lfs/progress.py,sha256=ELlBs2SeXhAcnPDN23w3FTeBRgB9RGqBD2CFMS6n9Xs,4750
|
|
31
|
+
scmrepo/git/lfs/smudge.py,sha256=1O_fznptWo4CKXqcJgUoWP6cgWWhvGAZ3d87kasG3cQ,1610
|
|
32
|
+
scmrepo/git/lfs/storage.py,sha256=x31GQRtrZH1SBoLc_m_IaLmR-mUBw_VC01HVkWgwHvI,2371
|
|
33
|
+
scmrepo-3.2.0.dist-info/LICENSE,sha256=-1jhbPjoIVHR0cEgahL4Zhct75Ff4MzYCR_jOaJDPq8,11340
|
|
34
|
+
scmrepo-3.2.0.dist-info/METADATA,sha256=-sK4txnW8FAqOTeww2mbq9HMlUp_1dKSdpM3a6i3jPI,4841
|
|
35
|
+
scmrepo-3.2.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
36
|
+
scmrepo-3.2.0.dist-info/top_level.txt,sha256=iunjod6w3GogERsAYfLRupnANXnqzX3jbIfbeIQG5cc,8
|
|
37
|
+
scmrepo-3.2.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|