hfget 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hfget/__init__.py ADDED
@@ -0,0 +1,52 @@
1
+ """hfget: download Hugging Face Hub files with curl, continuing partial files.
2
+
3
+ `download` fetches files from a Hub repo with curl and continues partial files from where
4
+ they stopped. `plan_download` shows
5
+ what a download would do without downloading, and `execute_plan` runs such a plan.
6
+
7
+ The functions log messages through the `logging` module, under the `hfget`
8
+ logger, send `DownloadProgress` snapshots to an optional `ProgressCallback`, and raise
9
+ `DownloadError` or `StoppedError` when they cannot finish. They never print.
10
+ """
11
+
12
+ import logging
13
+ from importlib.metadata import PackageNotFoundError, version
14
+
15
+ from .api import DownloadPlan, download, execute_plan, plan_download
16
+ from .errors import DownloadError, StoppedError
17
+ from .hub import RemoteFile
18
+ from .layout import Layout
19
+ from .progress import (
20
+ DownloadProgress,
21
+ FileProgress,
22
+ FileState,
23
+ ProgressCallback,
24
+ RetryInfo,
25
+ is_progress_event,
26
+ )
27
+
28
+ try:
29
+ __version__ = version("hfget")
30
+ except PackageNotFoundError: # running from a source tree that is not installed
31
+ __version__ = "0+unknown"
32
+
33
+ __all__ = [
34
+ "DownloadError",
35
+ "DownloadPlan",
36
+ "DownloadProgress",
37
+ "FileProgress",
38
+ "FileState",
39
+ "Layout",
40
+ "ProgressCallback",
41
+ "RemoteFile",
42
+ "RetryInfo",
43
+ "StoppedError",
44
+ "__version__",
45
+ "download",
46
+ "execute_plan",
47
+ "is_progress_event",
48
+ "plan_download",
49
+ ]
50
+
51
+ # A library adds no output handler of its own: the application decides where logs go.
52
+ logging.getLogger(__name__).addHandler(logging.NullHandler())
hfget/api.py ADDED
@@ -0,0 +1,233 @@
1
+ """The public functions: plan a download, run a plan, or do both in one call.
2
+
3
+ The functions report what they do through the `logging` module (logger names start with
4
+ `hfget`) and raise exceptions instead of exiting.
5
+
6
+ Example:
7
+ >>> from hfget import download
8
+ >>> download("Qwen/Qwen3-0.6B", filenames=["config.json"], local_dir="models/qwen")
9
+ PosixPath('models/qwen/config.json')
10
+ """
11
+
12
+ import logging
13
+ from collections.abc import Sequence
14
+ from dataclasses import dataclass
15
+ from pathlib import Path
16
+
17
+ from huggingface_hub import get_token
18
+
19
+ from .curl import Curl
20
+ from .downloader import Downloader
21
+ from .hub import RemoteFile, list_repo_files, select_files, split_repo_id
22
+ from .layout import CacheLayout, Layout, LocalDirLayout, repo_cache_folder
23
+ from .progress import PROGRESS_EVENT, ProgressCallback
24
+ from .utils import ensure_free_space, file_size
25
+
26
+ logger = logging.getLogger(__name__)
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class DownloadPlan:
31
+ """What a download will do. `--dry-run` shows this.
32
+
33
+ Attributes:
34
+ repo_id: The repo, without a type prefix.
35
+ repo_type: "model", "dataset", "space" or "kernel".
36
+ commit: The commit the files come from.
37
+ files: Every selected file.
38
+ pending: The selected files that still need to be downloaded.
39
+ bytes_to_download: The bytes still missing for the pending files. Bytes already in
40
+ partial files are not counted.
41
+ path: What `download` returns: the requested file if exactly one file was asked
42
+ for by name, otherwise the folder.
43
+ layout: Where the files go on disk.
44
+ """
45
+
46
+ repo_id: str
47
+ repo_type: str
48
+ commit: str
49
+ files: tuple[RemoteFile, ...]
50
+ pending: tuple[RemoteFile, ...]
51
+ bytes_to_download: int
52
+ path: Path
53
+ layout: Layout
54
+
55
+
56
+ def plan_download( # noqa: PLR0913 - the same keyword options as `hf download`
57
+ repo_id: str,
58
+ *,
59
+ filenames: Sequence[str] | None = None,
60
+ repo_type: str = "model",
61
+ revision: str | None = None,
62
+ include: Sequence[str] | None = None,
63
+ exclude: Sequence[str] | None = None,
64
+ cache_dir: str | Path | None = None,
65
+ local_dir: str | Path | None = None,
66
+ force_download: bool = False,
67
+ token: str | None = None,
68
+ ) -> DownloadPlan:
69
+ """List the repo, select the files and find the ones that still need to be downloaded.
70
+
71
+ Nothing is downloaded. With `local_dir`, the bookkeeping folder
72
+ `<local_dir>/.cache/huggingface` is created, as `hf download` does.
73
+
74
+ Args:
75
+ repo_id: The repo, for example "Qwen/Qwen3-0.6B". The forms `datasets/org/name`,
76
+ `spaces/org/name` and `hf://...` also work.
77
+ filenames: Only these files. A name that ends with "/" means a whole folder.
78
+ repo_type: "model", "dataset", "space" or "kernel". A prefix in `repo_id` wins.
79
+ revision: A branch, tag or commit. None means the default branch.
80
+ include: Glob patterns of files to download. Ignored when `filenames` is set.
81
+ exclude: Glob patterns of files to skip. Ignored when `filenames` is set.
82
+ cache_dir: The Hugging Face cache folder, used when `local_dir` is not set.
83
+ None means the default cache.
84
+ local_dir: Save the files in this folder instead of the cache.
85
+ force_download: Download files again even if they are already complete. Partial
86
+ files are still continued.
87
+ token: A Hugging Face access token. None means the saved token, if any.
88
+
89
+ Returns:
90
+ The plan.
91
+
92
+ Raises:
93
+ DownloadError: If the repo cannot be listed or a requested file is not in it.
94
+ """
95
+ repo_id, repo_type = split_repo_id(repo_id, repo_type)
96
+ commit, files = list_repo_files(repo_id, repo_type, revision, token)
97
+ names = list(filenames or [])
98
+ selected = select_files(files, names, include, exclude)
99
+
100
+ layout: Layout
101
+ if local_dir:
102
+ layout = LocalDirLayout(Path(local_dir), commit)
103
+ else:
104
+ layout = CacheLayout(repo_cache_folder(repo_id, repo_type, cache_dir), commit, revision)
105
+ layout.prepare()
106
+
107
+ pending = [file for file in selected if force_download or not layout.is_done(file)]
108
+ return DownloadPlan(
109
+ repo_id=repo_id,
110
+ repo_type=repo_type,
111
+ commit=commit,
112
+ files=tuple(selected),
113
+ pending=tuple(pending),
114
+ bytes_to_download=sum(file.size - file_size(layout.partial_path(file)) for file in pending),
115
+ path=_result_path(layout.root, names, selected),
116
+ layout=layout,
117
+ )
118
+
119
+
120
+ def execute_plan(
121
+ plan: DownloadPlan,
122
+ *,
123
+ token: str | None = None,
124
+ max_workers: int = 8,
125
+ on_progress: ProgressCallback | None = None,
126
+ ) -> Path:
127
+ """Download the pending files of a plan and complete the disk layout.
128
+
129
+ Args:
130
+ plan: The plan from `plan_download`.
131
+ token: A Hugging Face access token. None means the saved token, if any.
132
+ max_workers: How many files download at the same time.
133
+ on_progress: Called about every 0.25 s with a `DownloadProgress` snapshot of the
134
+ pending files.
135
+
136
+ Returns:
137
+ The downloaded file if exactly one file was asked for by name, otherwise the folder.
138
+
139
+ Raises:
140
+ DownloadError: If there is not enough disk space or a file cannot be downloaded.
141
+ Partial files are kept, so a new call continues them.
142
+ StoppedError: If the download was stopped.
143
+ """
144
+ if plan.pending:
145
+ ensure_free_space(plan.layout.root, plan.bytes_to_download)
146
+ logger.info(
147
+ "Downloading %d of %d files from %s (%s).",
148
+ len(plan.pending),
149
+ len(plan.files),
150
+ plan.repo_id,
151
+ plan.commit[:8],
152
+ extra=PROGRESS_EVENT,
153
+ )
154
+ curl = Curl(token or get_token())
155
+ Downloader(plan.layout, curl, plan.pending, on_progress).download_all(max_workers)
156
+ else:
157
+ logger.info("All files are already downloaded.")
158
+ plan.layout.finalize(plan.files)
159
+ return plan.path
160
+
161
+
162
+ def download( # noqa: PLR0913 - the same keyword options as `hf download`
163
+ repo_id: str,
164
+ *,
165
+ filenames: Sequence[str] | None = None,
166
+ repo_type: str = "model",
167
+ revision: str | None = None,
168
+ include: Sequence[str] | None = None,
169
+ exclude: Sequence[str] | None = None,
170
+ cache_dir: str | Path | None = None,
171
+ local_dir: str | Path | None = None,
172
+ force_download: bool = False,
173
+ token: str | None = None,
174
+ max_workers: int = 8,
175
+ on_progress: ProgressCallback | None = None,
176
+ ) -> Path:
177
+ """Download files from a Hub repo, continuing partial files from where they stopped.
178
+
179
+ This is `plan_download` followed by `execute_plan`.
180
+
181
+ Args:
182
+ repo_id: The repo, for example "Qwen/Qwen3-0.6B". The forms `datasets/org/name`,
183
+ `spaces/org/name` and `hf://...` also work.
184
+ filenames: Only these files. A name that ends with "/" means a whole folder.
185
+ repo_type: "model", "dataset", "space" or "kernel". A prefix in `repo_id` wins.
186
+ revision: A branch, tag or commit. None means the default branch.
187
+ include: Glob patterns of files to download. Ignored when `filenames` is set.
188
+ exclude: Glob patterns of files to skip. Ignored when `filenames` is set.
189
+ cache_dir: The Hugging Face cache folder, used when `local_dir` is not set.
190
+ None means the default cache.
191
+ local_dir: Save the files in this folder instead of the cache.
192
+ force_download: Download files again even if they are already complete. Partial
193
+ files are still continued.
194
+ token: A Hugging Face access token. None means the saved token, if any.
195
+ max_workers: How many files download at the same time.
196
+ on_progress: Called about every 0.25 s with a `DownloadProgress` snapshot of the
197
+ pending files.
198
+
199
+ Returns:
200
+ The downloaded file if exactly one file was asked for by name, otherwise the folder.
201
+
202
+ Raises:
203
+ DownloadError: If a problem cannot be fixed by retrying. Partial files are kept.
204
+ StoppedError: If the download was stopped.
205
+ """
206
+ plan = plan_download(
207
+ repo_id,
208
+ filenames=filenames,
209
+ repo_type=repo_type,
210
+ revision=revision,
211
+ include=include,
212
+ exclude=exclude,
213
+ cache_dir=cache_dir,
214
+ local_dir=local_dir,
215
+ force_download=force_download,
216
+ token=token,
217
+ )
218
+ return execute_plan(plan, token=token, max_workers=max_workers, on_progress=on_progress)
219
+
220
+
221
+ def _result_path(root: Path, filenames: Sequence[str], selected: Sequence[RemoteFile]) -> Path:
222
+ """Choose the path a download returns.
223
+
224
+ Args:
225
+ root: The folder where the files appear under their repo paths.
226
+ filenames: The file names the user asked for.
227
+ selected: The selected files.
228
+
229
+ Returns:
230
+ The file if one exact file name was asked for and it matched one file, else `root`.
231
+ """
232
+ single_file = len(filenames) == 1 and len(selected) == 1 and not filenames[0].endswith("/")
233
+ return root / selected[0].path if single_file else root
hfget/cli.py ADDED
@@ -0,0 +1,318 @@
1
+ """The `hfget` command: a thin command line layer over the `api` module.
2
+
3
+ It takes the same arguments as `hf download`. It prints the final path on stdout, sends
4
+ messages to stderr, and turns errors into exit messages.
5
+
6
+ Usage:
7
+ hfget Qwen/Qwen3.5-4B --local-dir models/Qwen3.5-4B
8
+ """
9
+
10
+ import argparse
11
+ import json
12
+ import logging
13
+ import sys
14
+ from collections.abc import Generator, Sequence
15
+ from contextlib import closing, contextmanager
16
+ from pathlib import Path
17
+
18
+ from . import __version__
19
+ from .api import DownloadPlan, execute_plan, plan_download
20
+ from .display import Display, LiveDisplay, PlainDisplay
21
+ from .errors import DownloadError, StoppedError
22
+ from .hub import REPO_TYPES
23
+ from .progress import is_progress_event
24
+ from .style import Glyphs, Theme
25
+ from .utils import human_size
26
+
27
+ OUTPUT_FORMATS = ("auto", "human", "agent", "json", "quiet")
28
+ VERBOSE_FORMATS = ("auto", "human", "agent") # the formats that show info messages
29
+ STOPPED_MESSAGE = "\nStopped. Finished files are kept. Run the same command again to continue."
30
+
31
+
32
+ class _ConsoleHandler(logging.Handler):
33
+ """Writes log messages to stderr, or through the progress display while one is shown.
34
+
35
+ Attributes:
36
+ display: The display that receives the messages, or None to write to stderr.
37
+ """
38
+
39
+ def __init__(self) -> None:
40
+ """Create the handler with no display attached."""
41
+ super().__init__()
42
+ self.display: Display | None = None
43
+
44
+ def emit(self, record: logging.LogRecord) -> None:
45
+ """Write one log record.
46
+
47
+ Args:
48
+ record: The record to write.
49
+ """
50
+ try:
51
+ message = self.format(record)
52
+ display = self.display
53
+ if display is None:
54
+ sys.stderr.write(f"{message}\n")
55
+ sys.stderr.flush()
56
+ else:
57
+ display.show_message(message, progress_event=is_progress_event(record))
58
+ except Exception: # the same handling as logging.StreamHandler
59
+ self.handleError(record)
60
+
61
+ @contextmanager
62
+ def routed_to(self, display: Display) -> Generator[None, None, None]:
63
+ """Send messages through a display while the `with` block runs.
64
+
65
+ Args:
66
+ display: The display.
67
+
68
+ Yields:
69
+ Nothing.
70
+ """
71
+ self.display = display
72
+ try:
73
+ yield
74
+ finally:
75
+ self.display = None
76
+
77
+
78
+ def main(argv: Sequence[str] | None = None) -> None:
79
+ """Run the `hfget` command.
80
+
81
+ Args:
82
+ argv: The command line arguments. None means `sys.argv[1:]`.
83
+
84
+ Raises:
85
+ SystemExit: With an error message on stderr if the download fails or is stopped.
86
+ """
87
+ args = _parse_args(argv)
88
+ output = _output_format(args)
89
+ handler = _configure_logging(output)
90
+ try:
91
+ path = _run(args, output, handler)
92
+ except DownloadError as e:
93
+ sys.exit(f"Error: {e}")
94
+ except (KeyboardInterrupt, StoppedError):
95
+ sys.exit(STOPPED_MESSAGE)
96
+ if args.dry_run:
97
+ return # `_run` already printed the plan
98
+ if output == "json":
99
+ print(json.dumps({"path": str(path)}))
100
+ else:
101
+ print(path)
102
+
103
+
104
+ def _run(args: argparse.Namespace, output: str, handler: _ConsoleHandler) -> Path:
105
+ """Plan the download, then print the plan (dry run) or run it with a progress display.
106
+
107
+ Args:
108
+ args: The parsed command line.
109
+ output: The output format.
110
+ handler: The log handler, which is routed through the display while it is shown.
111
+
112
+ Returns:
113
+ The path to print: the requested file, or the folder.
114
+ """
115
+ plan = plan_download(
116
+ args.repo_id,
117
+ filenames=args.filenames,
118
+ repo_type=args.repo_type,
119
+ revision=args.revision,
120
+ include=args.include,
121
+ exclude=args.exclude,
122
+ cache_dir=args.cache_dir,
123
+ local_dir=args.local_dir,
124
+ force_download=args.force_download,
125
+ token=args.token,
126
+ )
127
+ if args.dry_run:
128
+ if output == "json":
129
+ _print_dry_run_json(plan)
130
+ else:
131
+ _print_dry_run(plan)
132
+ return plan.path
133
+ display = _make_display(output, plan)
134
+ if display is None:
135
+ return execute_plan(plan, token=args.token, max_workers=args.max_workers)
136
+ with closing(display), handler.routed_to(display):
137
+ return execute_plan(
138
+ plan, token=args.token, max_workers=args.max_workers, on_progress=display.update
139
+ )
140
+
141
+
142
+ def _make_display(output: str, plan: DownloadPlan) -> Display | None:
143
+ """Choose the progress display for an output format.
144
+
145
+ Args:
146
+ output: The output format.
147
+ plan: The plan being run.
148
+
149
+ Returns:
150
+ A live display for `human`, and for `auto` on a terminal; plain progress lines for
151
+ `agent`, and for `auto` when stderr is not a terminal; None for `json` and `quiet`.
152
+ """
153
+ if output == "human" or (output == "auto" and sys.stderr.isatty()):
154
+ return LiveDisplay(sys.stderr, plan, Theme.for_stream(sys.stderr))
155
+ if output in ("auto", "agent"):
156
+ return PlainDisplay(sys.stderr, Glyphs.for_stream(sys.stderr))
157
+ return None
158
+
159
+
160
+ def _print_dry_run(plan: DownloadPlan) -> None:
161
+ """Print the files a download would fetch, in the format of `hf download --dry-run`.
162
+
163
+ Args:
164
+ plan: The download plan.
165
+ """
166
+ print(
167
+ f"[dry-run] Will download {len(plan.pending)} files (out of {len(plan.files)}) "
168
+ f"totalling {human_size(plan.bytes_to_download)}."
169
+ )
170
+ if plan.pending:
171
+ print("file\tsize")
172
+ for file in plan.pending:
173
+ print(f"{file.path}\t{human_size(file.size)}")
174
+
175
+
176
+ def _print_dry_run_json(plan: DownloadPlan) -> None:
177
+ """Print the plan of a dry run as one JSON object, so the output stays valid JSON.
178
+
179
+ Args:
180
+ plan: The download plan.
181
+ """
182
+ print(
183
+ json.dumps(
184
+ {
185
+ "path": str(plan.path),
186
+ "dry_run": True,
187
+ "bytes_to_download": plan.bytes_to_download,
188
+ "files": [{"path": file.path, "size": file.size} for file in plan.pending],
189
+ }
190
+ )
191
+ )
192
+
193
+
194
+ def _output_format(args: argparse.Namespace) -> str:
195
+ """Resolve the output format from `--format`, `--json` and `--quiet`.
196
+
197
+ Args:
198
+ args: The parsed command line.
199
+
200
+ Returns:
201
+ One of `OUTPUT_FORMATS`.
202
+ """
203
+ if args.json:
204
+ return "json"
205
+ if args.quiet:
206
+ return "quiet"
207
+ return args.format
208
+
209
+
210
+ def _configure_logging(output: str) -> _ConsoleHandler:
211
+ """Send the package's log messages to stderr.
212
+
213
+ Info messages are shown only in the verbose formats. Warnings are always shown.
214
+
215
+ Args:
216
+ output: The output format.
217
+
218
+ Returns:
219
+ The handler, so a progress display can take over its output.
220
+ """
221
+ handler = _ConsoleHandler()
222
+ handler.setFormatter(logging.Formatter("%(message)s"))
223
+ package_logger = logging.getLogger(__package__)
224
+ package_logger.addHandler(handler)
225
+ package_logger.setLevel(logging.INFO if output in VERBOSE_FORMATS else logging.WARNING)
226
+ package_logger.propagate = False
227
+ return handler
228
+
229
+
230
+ def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace:
231
+ """Parse the command line. The arguments and help texts match `hf download`.
232
+
233
+ Args:
234
+ argv: The command line arguments. None means `sys.argv[1:]`.
235
+
236
+ Returns:
237
+ The parsed arguments.
238
+ """
239
+ p = argparse.ArgumentParser(
240
+ prog="hfget",
241
+ description="Download files from the Hub. Same arguments as `hf download`, "
242
+ "but partial files continue where they stopped.",
243
+ )
244
+ p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
245
+ p.add_argument(
246
+ "repo_id",
247
+ help="The ID of the repo (e.g. `username/repo-name` or `spaces/username/repo-name`).",
248
+ )
249
+ p.add_argument(
250
+ "filenames",
251
+ nargs="*",
252
+ help="Files to download (e.g. `config.json`, `data/metadata.jsonl`).",
253
+ )
254
+ p.add_argument(
255
+ "--type",
256
+ "--repo-type",
257
+ dest="repo_type",
258
+ choices=REPO_TYPES,
259
+ default="model",
260
+ help="The type of repository (model, dataset, space, or kernel).",
261
+ )
262
+ p.add_argument(
263
+ "--revision", help="Git revision id which can be a branch name, a tag, or a commit hash."
264
+ )
265
+ p.add_argument(
266
+ "--include",
267
+ action="append",
268
+ help="Glob patterns to include from files to download. eg: *.json",
269
+ )
270
+ p.add_argument(
271
+ "--exclude", action="append", help="Glob patterns to exclude from files to download."
272
+ )
273
+ p.add_argument("--cache-dir", help="Directory where to save files.")
274
+ p.add_argument(
275
+ "--local-dir", help="If set, the downloaded file will be placed under this directory."
276
+ )
277
+ p.add_argument(
278
+ "--force-download",
279
+ action=argparse.BooleanOptionalAction,
280
+ default=False,
281
+ help="If True, the files will be downloaded even if they are already cached.",
282
+ )
283
+ p.add_argument(
284
+ "--dry-run",
285
+ action=argparse.BooleanOptionalAction,
286
+ default=False,
287
+ help="If True, perform a dry run without actually downloading the file.",
288
+ )
289
+ p.add_argument(
290
+ "--token", help="A User Access Token generated from https://huggingface.co/settings/tokens."
291
+ )
292
+ p.add_argument(
293
+ "--max-workers",
294
+ type=int,
295
+ default=8,
296
+ help="Maximum number of workers to use for downloading files. Default is 8.",
297
+ )
298
+ fmt = p.add_argument_group("Formatting options")
299
+ fmt.add_argument(
300
+ "--format",
301
+ choices=OUTPUT_FORMATS,
302
+ default="auto",
303
+ help="Output format. Defaults to 'auto'.",
304
+ )
305
+ fmt.add_argument(
306
+ "--json", action="store_true", help="JSON output. Equivalent to '--format json'."
307
+ )
308
+ fmt.add_argument(
309
+ "-q", "--quiet", action="store_true", help="Quiet output. Equivalent to '--format quiet'."
310
+ )
311
+ fmt.add_argument(
312
+ "--no-truncate", action="store_true", help="Accepted for compatibility. Has no effect."
313
+ )
314
+ return p.parse_args(argv)
315
+
316
+
317
+ if __name__ == "__main__":
318
+ main()