hfget 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hfget/__init__.py +52 -0
- hfget/api.py +233 -0
- hfget/cli.py +318 -0
- hfget/curl.py +177 -0
- hfget/display.py +316 -0
- hfget/downloader.py +317 -0
- hfget/errors.py +9 -0
- hfget/frames.py +524 -0
- hfget/hub.py +285 -0
- hfget/layout.py +280 -0
- hfget/progress.py +225 -0
- hfget/py.typed +0 -0
- hfget/speed.py +128 -0
- hfget/style.py +428 -0
- hfget/utils.py +57 -0
- hfget-0.1.0.dist-info/METADATA +255 -0
- hfget-0.1.0.dist-info/RECORD +20 -0
- hfget-0.1.0.dist-info/WHEEL +4 -0
- hfget-0.1.0.dist-info/entry_points.txt +3 -0
- hfget-0.1.0.dist-info/licenses/LICENSE +21 -0
hfget/__init__.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""hfget: download Hugging Face Hub files with curl, continuing partial files.
|
|
2
|
+
|
|
3
|
+
`download` fetches files from a Hub repo with curl and continues partial files from where
|
|
4
|
+
they stopped. `plan_download` shows
|
|
5
|
+
what a download would do without downloading, and `execute_plan` runs such a plan.
|
|
6
|
+
|
|
7
|
+
The functions log messages through the `logging` module, under the `hfget`
|
|
8
|
+
logger, send `DownloadProgress` snapshots to an optional `ProgressCallback`, and raise
|
|
9
|
+
`DownloadError` or `StoppedError` when they cannot finish. They never print.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
14
|
+
|
|
15
|
+
from .api import DownloadPlan, download, execute_plan, plan_download
|
|
16
|
+
from .errors import DownloadError, StoppedError
|
|
17
|
+
from .hub import RemoteFile
|
|
18
|
+
from .layout import Layout
|
|
19
|
+
from .progress import (
|
|
20
|
+
DownloadProgress,
|
|
21
|
+
FileProgress,
|
|
22
|
+
FileState,
|
|
23
|
+
ProgressCallback,
|
|
24
|
+
RetryInfo,
|
|
25
|
+
is_progress_event,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
__version__ = version("hfget")
|
|
30
|
+
except PackageNotFoundError: # running from a source tree that is not installed
|
|
31
|
+
__version__ = "0+unknown"
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"DownloadError",
|
|
35
|
+
"DownloadPlan",
|
|
36
|
+
"DownloadProgress",
|
|
37
|
+
"FileProgress",
|
|
38
|
+
"FileState",
|
|
39
|
+
"Layout",
|
|
40
|
+
"ProgressCallback",
|
|
41
|
+
"RemoteFile",
|
|
42
|
+
"RetryInfo",
|
|
43
|
+
"StoppedError",
|
|
44
|
+
"__version__",
|
|
45
|
+
"download",
|
|
46
|
+
"execute_plan",
|
|
47
|
+
"is_progress_event",
|
|
48
|
+
"plan_download",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
# A library adds no output handler of its own: the application decides where logs go.
|
|
52
|
+
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
hfget/api.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
"""The public functions: plan a download, run a plan, or do both in one call.
|
|
2
|
+
|
|
3
|
+
The functions report what they do through the `logging` module (logger names start with
|
|
4
|
+
`hfget`) and raise exceptions instead of exiting.
|
|
5
|
+
|
|
6
|
+
Example:
|
|
7
|
+
>>> from hfget import download
|
|
8
|
+
>>> download("Qwen/Qwen3-0.6B", filenames=["config.json"], local_dir="models/qwen")
|
|
9
|
+
PosixPath('models/qwen/config.json')
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from huggingface_hub import get_token
|
|
18
|
+
|
|
19
|
+
from .curl import Curl
|
|
20
|
+
from .downloader import Downloader
|
|
21
|
+
from .hub import RemoteFile, list_repo_files, select_files, split_repo_id
|
|
22
|
+
from .layout import CacheLayout, Layout, LocalDirLayout, repo_cache_folder
|
|
23
|
+
from .progress import PROGRESS_EVENT, ProgressCallback
|
|
24
|
+
from .utils import ensure_free_space, file_size
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class DownloadPlan:
|
|
31
|
+
"""What a download will do. `--dry-run` shows this.
|
|
32
|
+
|
|
33
|
+
Attributes:
|
|
34
|
+
repo_id: The repo, without a type prefix.
|
|
35
|
+
repo_type: "model", "dataset", "space" or "kernel".
|
|
36
|
+
commit: The commit the files come from.
|
|
37
|
+
files: Every selected file.
|
|
38
|
+
pending: The selected files that still need to be downloaded.
|
|
39
|
+
bytes_to_download: The bytes still missing for the pending files. Bytes already in
|
|
40
|
+
partial files are not counted.
|
|
41
|
+
path: What `download` returns: the requested file if exactly one file was asked
|
|
42
|
+
for by name, otherwise the folder.
|
|
43
|
+
layout: Where the files go on disk.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
repo_id: str
|
|
47
|
+
repo_type: str
|
|
48
|
+
commit: str
|
|
49
|
+
files: tuple[RemoteFile, ...]
|
|
50
|
+
pending: tuple[RemoteFile, ...]
|
|
51
|
+
bytes_to_download: int
|
|
52
|
+
path: Path
|
|
53
|
+
layout: Layout
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def plan_download( # noqa: PLR0913 - the same keyword options as `hf download`
|
|
57
|
+
repo_id: str,
|
|
58
|
+
*,
|
|
59
|
+
filenames: Sequence[str] | None = None,
|
|
60
|
+
repo_type: str = "model",
|
|
61
|
+
revision: str | None = None,
|
|
62
|
+
include: Sequence[str] | None = None,
|
|
63
|
+
exclude: Sequence[str] | None = None,
|
|
64
|
+
cache_dir: str | Path | None = None,
|
|
65
|
+
local_dir: str | Path | None = None,
|
|
66
|
+
force_download: bool = False,
|
|
67
|
+
token: str | None = None,
|
|
68
|
+
) -> DownloadPlan:
|
|
69
|
+
"""List the repo, select the files and find the ones that still need to be downloaded.
|
|
70
|
+
|
|
71
|
+
Nothing is downloaded. With `local_dir`, the bookkeeping folder
|
|
72
|
+
`<local_dir>/.cache/huggingface` is created, as `hf download` does.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
repo_id: The repo, for example "Qwen/Qwen3-0.6B". The forms `datasets/org/name`,
|
|
76
|
+
`spaces/org/name` and `hf://...` also work.
|
|
77
|
+
filenames: Only these files. A name that ends with "/" means a whole folder.
|
|
78
|
+
repo_type: "model", "dataset", "space" or "kernel". A prefix in `repo_id` wins.
|
|
79
|
+
revision: A branch, tag or commit. None means the default branch.
|
|
80
|
+
include: Glob patterns of files to download. Ignored when `filenames` is set.
|
|
81
|
+
exclude: Glob patterns of files to skip. Ignored when `filenames` is set.
|
|
82
|
+
cache_dir: The Hugging Face cache folder, used when `local_dir` is not set.
|
|
83
|
+
None means the default cache.
|
|
84
|
+
local_dir: Save the files in this folder instead of the cache.
|
|
85
|
+
force_download: Download files again even if they are already complete. Partial
|
|
86
|
+
files are still continued.
|
|
87
|
+
token: A Hugging Face access token. None means the saved token, if any.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
The plan.
|
|
91
|
+
|
|
92
|
+
Raises:
|
|
93
|
+
DownloadError: If the repo cannot be listed or a requested file is not in it.
|
|
94
|
+
"""
|
|
95
|
+
repo_id, repo_type = split_repo_id(repo_id, repo_type)
|
|
96
|
+
commit, files = list_repo_files(repo_id, repo_type, revision, token)
|
|
97
|
+
names = list(filenames or [])
|
|
98
|
+
selected = select_files(files, names, include, exclude)
|
|
99
|
+
|
|
100
|
+
layout: Layout
|
|
101
|
+
if local_dir:
|
|
102
|
+
layout = LocalDirLayout(Path(local_dir), commit)
|
|
103
|
+
else:
|
|
104
|
+
layout = CacheLayout(repo_cache_folder(repo_id, repo_type, cache_dir), commit, revision)
|
|
105
|
+
layout.prepare()
|
|
106
|
+
|
|
107
|
+
pending = [file for file in selected if force_download or not layout.is_done(file)]
|
|
108
|
+
return DownloadPlan(
|
|
109
|
+
repo_id=repo_id,
|
|
110
|
+
repo_type=repo_type,
|
|
111
|
+
commit=commit,
|
|
112
|
+
files=tuple(selected),
|
|
113
|
+
pending=tuple(pending),
|
|
114
|
+
bytes_to_download=sum(file.size - file_size(layout.partial_path(file)) for file in pending),
|
|
115
|
+
path=_result_path(layout.root, names, selected),
|
|
116
|
+
layout=layout,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def execute_plan(
|
|
121
|
+
plan: DownloadPlan,
|
|
122
|
+
*,
|
|
123
|
+
token: str | None = None,
|
|
124
|
+
max_workers: int = 8,
|
|
125
|
+
on_progress: ProgressCallback | None = None,
|
|
126
|
+
) -> Path:
|
|
127
|
+
"""Download the pending files of a plan and complete the disk layout.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
plan: The plan from `plan_download`.
|
|
131
|
+
token: A Hugging Face access token. None means the saved token, if any.
|
|
132
|
+
max_workers: How many files download at the same time.
|
|
133
|
+
on_progress: Called about every 0.25 s with a `DownloadProgress` snapshot of the
|
|
134
|
+
pending files.
|
|
135
|
+
|
|
136
|
+
Returns:
|
|
137
|
+
The downloaded file if exactly one file was asked for by name, otherwise the folder.
|
|
138
|
+
|
|
139
|
+
Raises:
|
|
140
|
+
DownloadError: If there is not enough disk space or a file cannot be downloaded.
|
|
141
|
+
Partial files are kept, so a new call continues them.
|
|
142
|
+
StoppedError: If the download was stopped.
|
|
143
|
+
"""
|
|
144
|
+
if plan.pending:
|
|
145
|
+
ensure_free_space(plan.layout.root, plan.bytes_to_download)
|
|
146
|
+
logger.info(
|
|
147
|
+
"Downloading %d of %d files from %s (%s).",
|
|
148
|
+
len(plan.pending),
|
|
149
|
+
len(plan.files),
|
|
150
|
+
plan.repo_id,
|
|
151
|
+
plan.commit[:8],
|
|
152
|
+
extra=PROGRESS_EVENT,
|
|
153
|
+
)
|
|
154
|
+
curl = Curl(token or get_token())
|
|
155
|
+
Downloader(plan.layout, curl, plan.pending, on_progress).download_all(max_workers)
|
|
156
|
+
else:
|
|
157
|
+
logger.info("All files are already downloaded.")
|
|
158
|
+
plan.layout.finalize(plan.files)
|
|
159
|
+
return plan.path
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def download( # noqa: PLR0913 - the same keyword options as `hf download`
|
|
163
|
+
repo_id: str,
|
|
164
|
+
*,
|
|
165
|
+
filenames: Sequence[str] | None = None,
|
|
166
|
+
repo_type: str = "model",
|
|
167
|
+
revision: str | None = None,
|
|
168
|
+
include: Sequence[str] | None = None,
|
|
169
|
+
exclude: Sequence[str] | None = None,
|
|
170
|
+
cache_dir: str | Path | None = None,
|
|
171
|
+
local_dir: str | Path | None = None,
|
|
172
|
+
force_download: bool = False,
|
|
173
|
+
token: str | None = None,
|
|
174
|
+
max_workers: int = 8,
|
|
175
|
+
on_progress: ProgressCallback | None = None,
|
|
176
|
+
) -> Path:
|
|
177
|
+
"""Download files from a Hub repo, continuing partial files from where they stopped.
|
|
178
|
+
|
|
179
|
+
This is `plan_download` followed by `execute_plan`.
|
|
180
|
+
|
|
181
|
+
Args:
|
|
182
|
+
repo_id: The repo, for example "Qwen/Qwen3-0.6B". The forms `datasets/org/name`,
|
|
183
|
+
`spaces/org/name` and `hf://...` also work.
|
|
184
|
+
filenames: Only these files. A name that ends with "/" means a whole folder.
|
|
185
|
+
repo_type: "model", "dataset", "space" or "kernel". A prefix in `repo_id` wins.
|
|
186
|
+
revision: A branch, tag or commit. None means the default branch.
|
|
187
|
+
include: Glob patterns of files to download. Ignored when `filenames` is set.
|
|
188
|
+
exclude: Glob patterns of files to skip. Ignored when `filenames` is set.
|
|
189
|
+
cache_dir: The Hugging Face cache folder, used when `local_dir` is not set.
|
|
190
|
+
None means the default cache.
|
|
191
|
+
local_dir: Save the files in this folder instead of the cache.
|
|
192
|
+
force_download: Download files again even if they are already complete. Partial
|
|
193
|
+
files are still continued.
|
|
194
|
+
token: A Hugging Face access token. None means the saved token, if any.
|
|
195
|
+
max_workers: How many files download at the same time.
|
|
196
|
+
on_progress: Called about every 0.25 s with a `DownloadProgress` snapshot of the
|
|
197
|
+
pending files.
|
|
198
|
+
|
|
199
|
+
Returns:
|
|
200
|
+
The downloaded file if exactly one file was asked for by name, otherwise the folder.
|
|
201
|
+
|
|
202
|
+
Raises:
|
|
203
|
+
DownloadError: If a problem cannot be fixed by retrying. Partial files are kept.
|
|
204
|
+
StoppedError: If the download was stopped.
|
|
205
|
+
"""
|
|
206
|
+
plan = plan_download(
|
|
207
|
+
repo_id,
|
|
208
|
+
filenames=filenames,
|
|
209
|
+
repo_type=repo_type,
|
|
210
|
+
revision=revision,
|
|
211
|
+
include=include,
|
|
212
|
+
exclude=exclude,
|
|
213
|
+
cache_dir=cache_dir,
|
|
214
|
+
local_dir=local_dir,
|
|
215
|
+
force_download=force_download,
|
|
216
|
+
token=token,
|
|
217
|
+
)
|
|
218
|
+
return execute_plan(plan, token=token, max_workers=max_workers, on_progress=on_progress)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _result_path(root: Path, filenames: Sequence[str], selected: Sequence[RemoteFile]) -> Path:
|
|
222
|
+
"""Choose the path a download returns.
|
|
223
|
+
|
|
224
|
+
Args:
|
|
225
|
+
root: The folder where the files appear under their repo paths.
|
|
226
|
+
filenames: The file names the user asked for.
|
|
227
|
+
selected: The selected files.
|
|
228
|
+
|
|
229
|
+
Returns:
|
|
230
|
+
The file if one exact file name was asked for and it matched one file, else `root`.
|
|
231
|
+
"""
|
|
232
|
+
single_file = len(filenames) == 1 and len(selected) == 1 and not filenames[0].endswith("/")
|
|
233
|
+
return root / selected[0].path if single_file else root
|
hfget/cli.py
ADDED
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
"""The `hfget` command: a thin command line layer over the `api` module.
|
|
2
|
+
|
|
3
|
+
It takes the same arguments as `hf download`. It prints the final path on stdout, sends
|
|
4
|
+
messages to stderr, and turns errors into exit messages.
|
|
5
|
+
|
|
6
|
+
Usage:
|
|
7
|
+
hfget Qwen/Qwen3.5-4B --local-dir models/Qwen3.5-4B
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
import sys
|
|
14
|
+
from collections.abc import Generator, Sequence
|
|
15
|
+
from contextlib import closing, contextmanager
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from . import __version__
|
|
19
|
+
from .api import DownloadPlan, execute_plan, plan_download
|
|
20
|
+
from .display import Display, LiveDisplay, PlainDisplay
|
|
21
|
+
from .errors import DownloadError, StoppedError
|
|
22
|
+
from .hub import REPO_TYPES
|
|
23
|
+
from .progress import is_progress_event
|
|
24
|
+
from .style import Glyphs, Theme
|
|
25
|
+
from .utils import human_size
|
|
26
|
+
|
|
27
|
+
OUTPUT_FORMATS = ("auto", "human", "agent", "json", "quiet")
|
|
28
|
+
VERBOSE_FORMATS = ("auto", "human", "agent") # the formats that show info messages
|
|
29
|
+
STOPPED_MESSAGE = "\nStopped. Finished files are kept. Run the same command again to continue."
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class _ConsoleHandler(logging.Handler):
|
|
33
|
+
"""Writes log messages to stderr, or through the progress display while one is shown.
|
|
34
|
+
|
|
35
|
+
Attributes:
|
|
36
|
+
display: The display that receives the messages, or None to write to stderr.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(self) -> None:
|
|
40
|
+
"""Create the handler with no display attached."""
|
|
41
|
+
super().__init__()
|
|
42
|
+
self.display: Display | None = None
|
|
43
|
+
|
|
44
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
45
|
+
"""Write one log record.
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
record: The record to write.
|
|
49
|
+
"""
|
|
50
|
+
try:
|
|
51
|
+
message = self.format(record)
|
|
52
|
+
display = self.display
|
|
53
|
+
if display is None:
|
|
54
|
+
sys.stderr.write(f"{message}\n")
|
|
55
|
+
sys.stderr.flush()
|
|
56
|
+
else:
|
|
57
|
+
display.show_message(message, progress_event=is_progress_event(record))
|
|
58
|
+
except Exception: # the same handling as logging.StreamHandler
|
|
59
|
+
self.handleError(record)
|
|
60
|
+
|
|
61
|
+
@contextmanager
|
|
62
|
+
def routed_to(self, display: Display) -> Generator[None, None, None]:
|
|
63
|
+
"""Send messages through a display while the `with` block runs.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
display: The display.
|
|
67
|
+
|
|
68
|
+
Yields:
|
|
69
|
+
Nothing.
|
|
70
|
+
"""
|
|
71
|
+
self.display = display
|
|
72
|
+
try:
|
|
73
|
+
yield
|
|
74
|
+
finally:
|
|
75
|
+
self.display = None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def main(argv: Sequence[str] | None = None) -> None:
|
|
79
|
+
"""Run the `hfget` command.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
argv: The command line arguments. None means `sys.argv[1:]`.
|
|
83
|
+
|
|
84
|
+
Raises:
|
|
85
|
+
SystemExit: With an error message on stderr if the download fails or is stopped.
|
|
86
|
+
"""
|
|
87
|
+
args = _parse_args(argv)
|
|
88
|
+
output = _output_format(args)
|
|
89
|
+
handler = _configure_logging(output)
|
|
90
|
+
try:
|
|
91
|
+
path = _run(args, output, handler)
|
|
92
|
+
except DownloadError as e:
|
|
93
|
+
sys.exit(f"Error: {e}")
|
|
94
|
+
except (KeyboardInterrupt, StoppedError):
|
|
95
|
+
sys.exit(STOPPED_MESSAGE)
|
|
96
|
+
if args.dry_run:
|
|
97
|
+
return # `_run` already printed the plan
|
|
98
|
+
if output == "json":
|
|
99
|
+
print(json.dumps({"path": str(path)}))
|
|
100
|
+
else:
|
|
101
|
+
print(path)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _run(args: argparse.Namespace, output: str, handler: _ConsoleHandler) -> Path:
|
|
105
|
+
"""Plan the download, then print the plan (dry run) or run it with a progress display.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
args: The parsed command line.
|
|
109
|
+
output: The output format.
|
|
110
|
+
handler: The log handler, which is routed through the display while it is shown.
|
|
111
|
+
|
|
112
|
+
Returns:
|
|
113
|
+
The path to print: the requested file, or the folder.
|
|
114
|
+
"""
|
|
115
|
+
plan = plan_download(
|
|
116
|
+
args.repo_id,
|
|
117
|
+
filenames=args.filenames,
|
|
118
|
+
repo_type=args.repo_type,
|
|
119
|
+
revision=args.revision,
|
|
120
|
+
include=args.include,
|
|
121
|
+
exclude=args.exclude,
|
|
122
|
+
cache_dir=args.cache_dir,
|
|
123
|
+
local_dir=args.local_dir,
|
|
124
|
+
force_download=args.force_download,
|
|
125
|
+
token=args.token,
|
|
126
|
+
)
|
|
127
|
+
if args.dry_run:
|
|
128
|
+
if output == "json":
|
|
129
|
+
_print_dry_run_json(plan)
|
|
130
|
+
else:
|
|
131
|
+
_print_dry_run(plan)
|
|
132
|
+
return plan.path
|
|
133
|
+
display = _make_display(output, plan)
|
|
134
|
+
if display is None:
|
|
135
|
+
return execute_plan(plan, token=args.token, max_workers=args.max_workers)
|
|
136
|
+
with closing(display), handler.routed_to(display):
|
|
137
|
+
return execute_plan(
|
|
138
|
+
plan, token=args.token, max_workers=args.max_workers, on_progress=display.update
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _make_display(output: str, plan: DownloadPlan) -> Display | None:
|
|
143
|
+
"""Choose the progress display for an output format.
|
|
144
|
+
|
|
145
|
+
Args:
|
|
146
|
+
output: The output format.
|
|
147
|
+
plan: The plan being run.
|
|
148
|
+
|
|
149
|
+
Returns:
|
|
150
|
+
A live display for `human`, and for `auto` on a terminal; plain progress lines for
|
|
151
|
+
`agent`, and for `auto` when stderr is not a terminal; None for `json` and `quiet`.
|
|
152
|
+
"""
|
|
153
|
+
if output == "human" or (output == "auto" and sys.stderr.isatty()):
|
|
154
|
+
return LiveDisplay(sys.stderr, plan, Theme.for_stream(sys.stderr))
|
|
155
|
+
if output in ("auto", "agent"):
|
|
156
|
+
return PlainDisplay(sys.stderr, Glyphs.for_stream(sys.stderr))
|
|
157
|
+
return None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _print_dry_run(plan: DownloadPlan) -> None:
|
|
161
|
+
"""Print the files a download would fetch, in the format of `hf download --dry-run`.
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
plan: The download plan.
|
|
165
|
+
"""
|
|
166
|
+
print(
|
|
167
|
+
f"[dry-run] Will download {len(plan.pending)} files (out of {len(plan.files)}) "
|
|
168
|
+
f"totalling {human_size(plan.bytes_to_download)}."
|
|
169
|
+
)
|
|
170
|
+
if plan.pending:
|
|
171
|
+
print("file\tsize")
|
|
172
|
+
for file in plan.pending:
|
|
173
|
+
print(f"{file.path}\t{human_size(file.size)}")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _print_dry_run_json(plan: DownloadPlan) -> None:
|
|
177
|
+
"""Print the plan of a dry run as one JSON object, so the output stays valid JSON.
|
|
178
|
+
|
|
179
|
+
Args:
|
|
180
|
+
plan: The download plan.
|
|
181
|
+
"""
|
|
182
|
+
print(
|
|
183
|
+
json.dumps(
|
|
184
|
+
{
|
|
185
|
+
"path": str(plan.path),
|
|
186
|
+
"dry_run": True,
|
|
187
|
+
"bytes_to_download": plan.bytes_to_download,
|
|
188
|
+
"files": [{"path": file.path, "size": file.size} for file in plan.pending],
|
|
189
|
+
}
|
|
190
|
+
)
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _output_format(args: argparse.Namespace) -> str:
|
|
195
|
+
"""Resolve the output format from `--format`, `--json` and `--quiet`.
|
|
196
|
+
|
|
197
|
+
Args:
|
|
198
|
+
args: The parsed command line.
|
|
199
|
+
|
|
200
|
+
Returns:
|
|
201
|
+
One of `OUTPUT_FORMATS`.
|
|
202
|
+
"""
|
|
203
|
+
if args.json:
|
|
204
|
+
return "json"
|
|
205
|
+
if args.quiet:
|
|
206
|
+
return "quiet"
|
|
207
|
+
return args.format
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _configure_logging(output: str) -> _ConsoleHandler:
|
|
211
|
+
"""Send the package's log messages to stderr.
|
|
212
|
+
|
|
213
|
+
Info messages are shown only in the verbose formats. Warnings are always shown.
|
|
214
|
+
|
|
215
|
+
Args:
|
|
216
|
+
output: The output format.
|
|
217
|
+
|
|
218
|
+
Returns:
|
|
219
|
+
The handler, so a progress display can take over its output.
|
|
220
|
+
"""
|
|
221
|
+
handler = _ConsoleHandler()
|
|
222
|
+
handler.setFormatter(logging.Formatter("%(message)s"))
|
|
223
|
+
package_logger = logging.getLogger(__package__)
|
|
224
|
+
package_logger.addHandler(handler)
|
|
225
|
+
package_logger.setLevel(logging.INFO if output in VERBOSE_FORMATS else logging.WARNING)
|
|
226
|
+
package_logger.propagate = False
|
|
227
|
+
return handler
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace:
|
|
231
|
+
"""Parse the command line. The arguments and help texts match `hf download`.
|
|
232
|
+
|
|
233
|
+
Args:
|
|
234
|
+
argv: The command line arguments. None means `sys.argv[1:]`.
|
|
235
|
+
|
|
236
|
+
Returns:
|
|
237
|
+
The parsed arguments.
|
|
238
|
+
"""
|
|
239
|
+
p = argparse.ArgumentParser(
|
|
240
|
+
prog="hfget",
|
|
241
|
+
description="Download files from the Hub. Same arguments as `hf download`, "
|
|
242
|
+
"but partial files continue where they stopped.",
|
|
243
|
+
)
|
|
244
|
+
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
245
|
+
p.add_argument(
|
|
246
|
+
"repo_id",
|
|
247
|
+
help="The ID of the repo (e.g. `username/repo-name` or `spaces/username/repo-name`).",
|
|
248
|
+
)
|
|
249
|
+
p.add_argument(
|
|
250
|
+
"filenames",
|
|
251
|
+
nargs="*",
|
|
252
|
+
help="Files to download (e.g. `config.json`, `data/metadata.jsonl`).",
|
|
253
|
+
)
|
|
254
|
+
p.add_argument(
|
|
255
|
+
"--type",
|
|
256
|
+
"--repo-type",
|
|
257
|
+
dest="repo_type",
|
|
258
|
+
choices=REPO_TYPES,
|
|
259
|
+
default="model",
|
|
260
|
+
help="The type of repository (model, dataset, space, or kernel).",
|
|
261
|
+
)
|
|
262
|
+
p.add_argument(
|
|
263
|
+
"--revision", help="Git revision id which can be a branch name, a tag, or a commit hash."
|
|
264
|
+
)
|
|
265
|
+
p.add_argument(
|
|
266
|
+
"--include",
|
|
267
|
+
action="append",
|
|
268
|
+
help="Glob patterns to include from files to download. eg: *.json",
|
|
269
|
+
)
|
|
270
|
+
p.add_argument(
|
|
271
|
+
"--exclude", action="append", help="Glob patterns to exclude from files to download."
|
|
272
|
+
)
|
|
273
|
+
p.add_argument("--cache-dir", help="Directory where to save files.")
|
|
274
|
+
p.add_argument(
|
|
275
|
+
"--local-dir", help="If set, the downloaded file will be placed under this directory."
|
|
276
|
+
)
|
|
277
|
+
p.add_argument(
|
|
278
|
+
"--force-download",
|
|
279
|
+
action=argparse.BooleanOptionalAction,
|
|
280
|
+
default=False,
|
|
281
|
+
help="If True, the files will be downloaded even if they are already cached.",
|
|
282
|
+
)
|
|
283
|
+
p.add_argument(
|
|
284
|
+
"--dry-run",
|
|
285
|
+
action=argparse.BooleanOptionalAction,
|
|
286
|
+
default=False,
|
|
287
|
+
help="If True, perform a dry run without actually downloading the file.",
|
|
288
|
+
)
|
|
289
|
+
p.add_argument(
|
|
290
|
+
"--token", help="A User Access Token generated from https://huggingface.co/settings/tokens."
|
|
291
|
+
)
|
|
292
|
+
p.add_argument(
|
|
293
|
+
"--max-workers",
|
|
294
|
+
type=int,
|
|
295
|
+
default=8,
|
|
296
|
+
help="Maximum number of workers to use for downloading files. Default is 8.",
|
|
297
|
+
)
|
|
298
|
+
fmt = p.add_argument_group("Formatting options")
|
|
299
|
+
fmt.add_argument(
|
|
300
|
+
"--format",
|
|
301
|
+
choices=OUTPUT_FORMATS,
|
|
302
|
+
default="auto",
|
|
303
|
+
help="Output format. Defaults to 'auto'.",
|
|
304
|
+
)
|
|
305
|
+
fmt.add_argument(
|
|
306
|
+
"--json", action="store_true", help="JSON output. Equivalent to '--format json'."
|
|
307
|
+
)
|
|
308
|
+
fmt.add_argument(
|
|
309
|
+
"-q", "--quiet", action="store_true", help="Quiet output. Equivalent to '--format quiet'."
|
|
310
|
+
)
|
|
311
|
+
fmt.add_argument(
|
|
312
|
+
"--no-truncate", action="store_true", help="Accepted for compatibility. Has no effect."
|
|
313
|
+
)
|
|
314
|
+
return p.parse_args(argv)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
if __name__ == "__main__":
|
|
318
|
+
main()
|