hfget 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hfget-0.1.0/LICENSE +21 -0
- hfget-0.1.0/PKG-INFO +255 -0
- hfget-0.1.0/README.md +222 -0
- hfget-0.1.0/pyproject.toml +123 -0
- hfget-0.1.0/pyproject.toml.orig +118 -0
- hfget-0.1.0/src/hfget/__init__.py +52 -0
- hfget-0.1.0/src/hfget/api.py +233 -0
- hfget-0.1.0/src/hfget/cli.py +318 -0
- hfget-0.1.0/src/hfget/curl.py +177 -0
- hfget-0.1.0/src/hfget/display.py +316 -0
- hfget-0.1.0/src/hfget/downloader.py +317 -0
- hfget-0.1.0/src/hfget/errors.py +9 -0
- hfget-0.1.0/src/hfget/frames.py +524 -0
- hfget-0.1.0/src/hfget/hub.py +285 -0
- hfget-0.1.0/src/hfget/layout.py +280 -0
- hfget-0.1.0/src/hfget/progress.py +225 -0
- hfget-0.1.0/src/hfget/py.typed +0 -0
- hfget-0.1.0/src/hfget/speed.py +128 -0
- hfget-0.1.0/src/hfget/style.py +428 -0
- hfget-0.1.0/src/hfget/utils.py +57 -0
hfget-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mohammad Sharara
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
hfget-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hfget
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Download Hugging Face models and datasets with curl, continuing partial files where they stopped.
|
|
5
|
+
Keywords: huggingface,hub,download,resume,curl,models,datasets
|
|
6
|
+
Author: Mohammad Sharara
|
|
7
|
+
Author-email: Mohammad Sharara <ms198mohammad@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Operating System :: MacOS
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Dist: huggingface-hub>=2.4,<3
|
|
27
|
+
Requires-Python: >=3.10
|
|
28
|
+
Project-URL: Homepage, https://github.com/msharara1998/hfget
|
|
29
|
+
Project-URL: Repository, https://github.com/msharara1998/hfget
|
|
30
|
+
Project-URL: Issues, https://github.com/msharara1998/hfget/issues
|
|
31
|
+
Project-URL: Changelog, https://github.com/msharara1998/hfget/blob/main/CHANGELOG.md
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# hfget
|
|
35
|
+
|
|
36
|
+
[](https://pypi.org/project/hfget/)
|
|
37
|
+
[](https://pypi.org/project/hfget/)
|
|
38
|
+
[](https://github.com/msharara1998/hfget/blob/main/LICENSE)
|
|
39
|
+
[](https://github.com/msharara1998/hfget/actions/workflows/ci.yml)
|
|
40
|
+
|
|
41
|
+
Download models and datasets from the Hugging Face Hub, and **continue partial files from
|
|
42
|
+
where they stopped**.
|
|
43
|
+
|
|
44
|
+
`hfget` takes the same arguments as `hf download` and saves files in the same layout, so
|
|
45
|
+
`transformers` and the other Hugging Face tools find them. The difference: when a download is
|
|
46
|
+
interrupted, `hfget` keeps the partial file and continues it, byte for byte.
|
|
47
|
+
|
|
48
|
+
## Why
|
|
49
|
+
|
|
50
|
+
`hf download` (huggingface_hub 2.4) deletes a partial file when the download is interrupted.
|
|
51
|
+
Running it again skips the files that finished, but a large file that was half done starts
|
|
52
|
+
again from zero. On a slow or unstable connection, a 5 GB weight file can then take hours more.
|
|
53
|
+
|
|
54
|
+
`hfget` keeps every byte it has received:
|
|
55
|
+
|
|
56
|
+
| What happens | What `hfget` does |
|
|
57
|
+
| --- | --- |
|
|
58
|
+
| The network drops, or no data arrives for 60 seconds | Waits and continues by itself. The wait doubles after each failed attempt, up to 60 seconds. |
|
|
59
|
+
| You press Ctrl+C, close the terminal, or the computer restarts | Keeps the partial file. Run the same command again to continue. |
|
|
60
|
+
| More than 10 attempts in a row download nothing new | Stops with a message. Run the download again to continue. |
|
|
61
|
+
| The repo is gated or private, or not found | Stops at once with a message, because retrying will not help. |
|
|
62
|
+
|
|
63
|
+
## Install
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
pip install hfget
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Or as a command-line tool with [uv](https://docs.astral.sh/uv/):
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
uv tool install hfget # installs the `hfget` command
|
|
73
|
+
uvx hfget --help # or run it once without installing
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Requirements: Python 3.10 or newer, and `curl` on your `PATH` (it does the downloading).
|
|
77
|
+
`curl` is preinstalled on macOS and most Linux systems. Otherwise:
|
|
78
|
+
`sudo apt install curl` (Debian/Ubuntu) or `brew install curl` (macOS).
|
|
79
|
+
|
|
80
|
+
## Quick start
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
# Download a model into a folder
|
|
84
|
+
hfget Qwen/Qwen3-0.6B --local-dir models/Qwen3-0.6B
|
|
85
|
+
|
|
86
|
+
# Or into the Hugging Face cache, where transformers looks for models
|
|
87
|
+
hfget Qwen/Qwen3-0.6B
|
|
88
|
+
|
|
89
|
+
# See what would be downloaded, without downloading
|
|
90
|
+
hfget Qwen/Qwen3-0.6B --local-dir models/Qwen3-0.6B --dry-run
|
|
91
|
+
|
|
92
|
+
# Only some files
|
|
93
|
+
hfget Qwen/Qwen3-0.6B config.json tokenizer.json --local-dir models/Qwen3-0.6B
|
|
94
|
+
hfget Qwen/Qwen3-0.6B --include "*.json" --local-dir models/Qwen3-0.6B
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
If the download stops, run the same command again. To check a folder later, also run the same
|
|
98
|
+
command again: it prints `All files are already downloaded.` or downloads only what is
|
|
99
|
+
missing.
|
|
100
|
+
|
|
101
|
+
## What you see while downloading
|
|
102
|
+
|
|
103
|
+
In a terminal, two lines say what is downloaded and where. Below them, a live area shows one
|
|
104
|
+
row per active file and a total row:
|
|
105
|
+
|
|
106
|
+
```
|
|
107
|
+
Qwen/Qwen3.5-4B @ 851bf6e8 → models/Qwen3.5-4B
|
|
108
|
+
2 files to download (2 resuming, 3.98 GB already on disk) · 12 already complete
|
|
109
|
+
✓ config.json 3.20 kB
|
|
110
|
+
↓ model.safetensors-00001-of-00002.safetensors ━━━━━━━━━━━━━━╸───── 71% 3.80 / 5.30 GB 1.9 MB/s
|
|
111
|
+
↻ model.safetensors-00002-of-00002.safetensors retry 2 in 4 s · no data for 60 s
|
|
112
|
+
Total 1/2 ━━━━━━━━━━━━━╸────── 65% 6.10 / 9.32 GB 2.1 MB/s ETA 26 min elapsed 41 min
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
- `↓` downloads, `↻` waits before a retry, and a spinner means the checksum is being checked.
|
|
116
|
+
- A finished file leaves the live area and stays above it as a `✓` line.
|
|
117
|
+
- Speeds cover the last 10 seconds and leave out bytes from earlier runs. ETA is the bytes
|
|
118
|
+
left divided by the speed.
|
|
119
|
+
- When the window gets narrower, the speed, then the byte counts, then the bar are dropped.
|
|
120
|
+
Long names are shortened in the middle, so the ending stays visible. The live area redraws
|
|
121
|
+
correctly after the terminal rewraps lines when you resize it.
|
|
122
|
+
- At the end, a summary line replaces the live area, for example
|
|
123
|
+
`✓ Downloaded 2 files (9.32 GB) in 1 h 12 min · average 2.1 MB/s`.
|
|
124
|
+
- Colors follow `NO_COLOR`. Plain ASCII characters are used if the terminal cannot show the
|
|
125
|
+
Unicode ones.
|
|
126
|
+
|
|
127
|
+
When stderr is not a terminal (a log file or a pipe), or with `--format agent`, you get plain
|
|
128
|
+
lines instead, about every 10 % or 30 seconds:
|
|
129
|
+
|
|
130
|
+
```
|
|
131
|
+
[ 42%] 3.91 / 9.32 GB · 2.1 MB/s · ETA 43 min
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
`--quiet` and `--json` show no progress.
|
|
135
|
+
|
|
136
|
+
## Use it from Python
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
from hfget import download
|
|
140
|
+
|
|
141
|
+
path = download("Qwen/Qwen3-0.6B", local_dir="models/Qwen3-0.6B")
|
|
142
|
+
print(path) # models/Qwen3-0.6B
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
`download` takes the same options as the command line: `filenames`, `repo_type`, `revision`,
|
|
146
|
+
`include`, `exclude`, `cache_dir`, `local_dir`, `force_download`, `token`, `max_workers` and
|
|
147
|
+
`on_progress`. It returns the folder, or the file if you asked for exactly one file by name.
|
|
148
|
+
|
|
149
|
+
To look before you download, split it in two steps:
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
from hfget import execute_plan, plan_download
|
|
153
|
+
|
|
154
|
+
plan = plan_download("Qwen/Qwen3-0.6B", local_dir="models/Qwen3-0.6B")
|
|
155
|
+
print(f"{len(plan.pending)} of {len(plan.files)} files, {plan.bytes_to_download} bytes to download")
|
|
156
|
+
execute_plan(plan)
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
To follow the progress, pass a callback. It receives a `DownloadProgress` snapshot about four
|
|
160
|
+
times a second:
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
from hfget import DownloadProgress, download
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def show(progress: DownloadProgress) -> None:
|
|
167
|
+
print(f"{progress.done_bytes} / {progress.total_bytes} bytes")
|
|
168
|
+
for file in progress.files:
|
|
169
|
+
print(f" {file.path}: {file.state.name} {file.bytes_on_disk} / {file.size}")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
download("Qwen/Qwen3-0.6B", filenames=["config.json"], local_dir="models/qwen", on_progress=show)
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
The library never prints. It logs through the standard `logging` module under the `hfget`
|
|
176
|
+
logger, and raises `hfget.DownloadError` (a problem retrying will not fix) or
|
|
177
|
+
`hfget.StoppedError` (the download was stopped). Partial files are always kept, so calling
|
|
178
|
+
`download` again continues them.
|
|
179
|
+
|
|
180
|
+
## Command-line options
|
|
181
|
+
|
|
182
|
+
The options match `hf download`:
|
|
183
|
+
|
|
184
|
+
| Option | Meaning |
|
|
185
|
+
| --- | --- |
|
|
186
|
+
| `REPO_ID` | The repo, for example `Qwen/Qwen3-0.6B`. `datasets/...`, `spaces/...` and `hf://...` forms also work. |
|
|
187
|
+
| `FILENAMES...` | Only these files. A name that ends with `/` means a whole folder. |
|
|
188
|
+
| `--type`, `--repo-type` | `model` (default), `dataset`, `space` or `kernel` |
|
|
189
|
+
| `--revision` | A branch, tag or commit. Default: `main`. |
|
|
190
|
+
| `--include` / `--exclude` | Glob patterns, for example `--include "*.json"`. Repeat the option for more patterns. |
|
|
191
|
+
| `--local-dir` | Save the files in this folder. |
|
|
192
|
+
| `--cache-dir` | Hugging Face cache folder, used when `--local-dir` is not set. Default: `~/.cache/huggingface/hub`. |
|
|
193
|
+
| `--force-download` | Download files again even if they are already complete. Partial files are still continued. |
|
|
194
|
+
| `--dry-run` | Only show what would be downloaded. |
|
|
195
|
+
| `--token` | Hugging Face access token. Default: your saved login (`hf auth login`) or `HF_TOKEN`. |
|
|
196
|
+
| `--max-workers` | Number of files downloaded at the same time. Default: 8. |
|
|
197
|
+
| `--format`, `--json`, `-q` / `--quiet` | Output format. `human` shows the live progress area, `agent` prints plain progress lines, `auto` (default) picks `human` in a terminal and `agent` otherwise. |
|
|
198
|
+
| `--no-truncate` | Accepted so `hf download` commands work unchanged. It has no effect. |
|
|
199
|
+
| `--version` | Print the version. |
|
|
200
|
+
|
|
201
|
+
## How it works
|
|
202
|
+
|
|
203
|
+
1. **Find the files.** The `huggingface_hub` library gets the commit of the requested revision
|
|
204
|
+
and lists every file with its size and checksum.
|
|
205
|
+
2. **Choose the files.** File names and `--include` / `--exclude` patterns are matched with the
|
|
206
|
+
library's own matching function, so the selection is the same as `hf download`.
|
|
207
|
+
3. **Skip finished files.** A file is skipped when it already exists with the right size and
|
|
208
|
+
checksum record.
|
|
209
|
+
4. **Download.** `curl --continue-at -` downloads each file into an `.incomplete` file. After
|
|
210
|
+
an interruption, curl asks the server only for the bytes that are still missing.
|
|
211
|
+
5. **Check.** When a file is complete, its checksum is compared with the one from the Hub
|
|
212
|
+
(sha256 for large files, the git blob id for small ones). A file that does not match is
|
|
213
|
+
downloaded again.
|
|
214
|
+
6. **Move into place.** The file is moved to its final name and its record is written.
|
|
215
|
+
|
|
216
|
+
Your access token is passed to curl on stdin, so it never shows in the process list.
|
|
217
|
+
|
|
218
|
+
## Where the files go
|
|
219
|
+
|
|
220
|
+
With `--local-dir models/Qwen3-0.6B`:
|
|
221
|
+
|
|
222
|
+
```
|
|
223
|
+
models/Qwen3-0.6B/
|
|
224
|
+
├── config.json, tokenizer.json, ... finished files
|
|
225
|
+
└── .cache/huggingface/download/
|
|
226
|
+
├── <file>.metadata record: commit, checksum, time
|
|
227
|
+
└── <file>.incomplete partial file while downloading
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
Without `--local-dir`, files go to the Hugging Face cache (`blobs/`, `snapshots/`, `refs/`).
|
|
231
|
+
Both layouts are the same as `hf download` writes them, so you can mix the two tools.
|
|
232
|
+
|
|
233
|
+
## Platform support
|
|
234
|
+
|
|
235
|
+
- Tested on Linux and macOS, with Python 3.10 to 3.14.
|
|
236
|
+
- Windows is not tested yet. The cache layout uses symlinks, which need Developer Mode on
|
|
237
|
+
Windows.
|
|
238
|
+
- The live display is built for terminals that rewrap lines when you resize them, as most
|
|
239
|
+
modern terminals do. It is tested in tmux. In terminals that cut long lines instead, such as
|
|
240
|
+
classic xterm, shrinking the window may erase a few lines above the live area.
|
|
241
|
+
|
|
242
|
+
## Development
|
|
243
|
+
|
|
244
|
+
```bash
|
|
245
|
+
git clone https://github.com/msharara1998/hfget
|
|
246
|
+
cd hfget
|
|
247
|
+
uv sync
|
|
248
|
+
uv run ruff format && uv run ruff check && uv run ty check
|
|
249
|
+
uv run pytest # offline tests
|
|
250
|
+
uv run pytest -m network # also downloads a small real file from the Hub
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
## License
|
|
254
|
+
|
|
255
|
+
MIT. See [LICENSE](https://github.com/msharara1998/hfget/blob/main/LICENSE).
|
hfget-0.1.0/README.md
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
# hfget
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/hfget/)
|
|
4
|
+
[](https://pypi.org/project/hfget/)
|
|
5
|
+
[](https://github.com/msharara1998/hfget/blob/main/LICENSE)
|
|
6
|
+
[](https://github.com/msharara1998/hfget/actions/workflows/ci.yml)
|
|
7
|
+
|
|
8
|
+
Download models and datasets from the Hugging Face Hub, and **continue partial files from
|
|
9
|
+
where they stopped**.
|
|
10
|
+
|
|
11
|
+
`hfget` takes the same arguments as `hf download` and saves files in the same layout, so
|
|
12
|
+
`transformers` and the other Hugging Face tools find them. The difference: when a download is
|
|
13
|
+
interrupted, `hfget` keeps the partial file and continues it, byte for byte.
|
|
14
|
+
|
|
15
|
+
## Why
|
|
16
|
+
|
|
17
|
+
`hf download` (huggingface_hub 2.4) deletes a partial file when the download is interrupted.
|
|
18
|
+
Running it again skips the files that finished, but a large file that was half done starts
|
|
19
|
+
again from zero. On a slow or unstable connection, a 5 GB weight file can then take hours more.
|
|
20
|
+
|
|
21
|
+
`hfget` keeps every byte it has received:
|
|
22
|
+
|
|
23
|
+
| What happens | What `hfget` does |
|
|
24
|
+
| --- | --- |
|
|
25
|
+
| The network drops, or no data arrives for 60 seconds | Waits and continues by itself. The wait doubles after each failed attempt, up to 60 seconds. |
|
|
26
|
+
| You press Ctrl+C, close the terminal, or the computer restarts | Keeps the partial file. Run the same command again to continue. |
|
|
27
|
+
| More than 10 attempts in a row download nothing new | Stops with a message. Run the download again to continue. |
|
|
28
|
+
| The repo is gated or private, or not found | Stops at once with a message, because retrying will not help. |
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install hfget
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Or as a command-line tool with [uv](https://docs.astral.sh/uv/):
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
uv tool install hfget # installs the `hfget` command
|
|
40
|
+
uvx hfget --help # or run it once without installing
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Requirements: Python 3.10 or newer, and `curl` on your `PATH` (it does the downloading).
|
|
44
|
+
`curl` is preinstalled on macOS and most Linux systems. Otherwise:
|
|
45
|
+
`sudo apt install curl` (Debian/Ubuntu) or `brew install curl` (macOS).
|
|
46
|
+
|
|
47
|
+
## Quick start
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
# Download a model into a folder
|
|
51
|
+
hfget Qwen/Qwen3-0.6B --local-dir models/Qwen3-0.6B
|
|
52
|
+
|
|
53
|
+
# Or into the Hugging Face cache, where transformers looks for models
|
|
54
|
+
hfget Qwen/Qwen3-0.6B
|
|
55
|
+
|
|
56
|
+
# See what would be downloaded, without downloading
|
|
57
|
+
hfget Qwen/Qwen3-0.6B --local-dir models/Qwen3-0.6B --dry-run
|
|
58
|
+
|
|
59
|
+
# Only some files
|
|
60
|
+
hfget Qwen/Qwen3-0.6B config.json tokenizer.json --local-dir models/Qwen3-0.6B
|
|
61
|
+
hfget Qwen/Qwen3-0.6B --include "*.json" --local-dir models/Qwen3-0.6B
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
If the download stops, run the same command again. To check a folder later, also run the same
|
|
65
|
+
command again: it prints `All files are already downloaded.` or downloads only what is
|
|
66
|
+
missing.
|
|
67
|
+
|
|
68
|
+
## What you see while downloading
|
|
69
|
+
|
|
70
|
+
In a terminal, two lines say what is downloaded and where. Below them, a live area shows one
|
|
71
|
+
row per active file and a total row:
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
Qwen/Qwen3.5-4B @ 851bf6e8 → models/Qwen3.5-4B
|
|
75
|
+
2 files to download (2 resuming, 3.98 GB already on disk) · 12 already complete
|
|
76
|
+
✓ config.json 3.20 kB
|
|
77
|
+
↓ model.safetensors-00001-of-00002.safetensors ━━━━━━━━━━━━━━╸───── 71% 3.80 / 5.30 GB 1.9 MB/s
|
|
78
|
+
↻ model.safetensors-00002-of-00002.safetensors retry 2 in 4 s · no data for 60 s
|
|
79
|
+
Total 1/2 ━━━━━━━━━━━━━╸────── 65% 6.10 / 9.32 GB 2.1 MB/s ETA 26 min elapsed 41 min
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
- `↓` downloads, `↻` waits before a retry, and a spinner means the checksum is being checked.
|
|
83
|
+
- A finished file leaves the live area and stays above it as a `✓` line.
|
|
84
|
+
- Speeds cover the last 10 seconds and leave out bytes from earlier runs. ETA is the bytes
|
|
85
|
+
left divided by the speed.
|
|
86
|
+
- When the window gets narrower, the speed, then the byte counts, then the bar are dropped.
|
|
87
|
+
Long names are shortened in the middle, so the ending stays visible. The live area redraws
|
|
88
|
+
correctly after the terminal rewraps lines when you resize it.
|
|
89
|
+
- At the end, a summary line replaces the live area, for example
|
|
90
|
+
`✓ Downloaded 2 files (9.32 GB) in 1 h 12 min · average 2.1 MB/s`.
|
|
91
|
+
- Colors follow `NO_COLOR`. Plain ASCII characters are used if the terminal cannot show the
|
|
92
|
+
Unicode ones.
|
|
93
|
+
|
|
94
|
+
When stderr is not a terminal (a log file or a pipe), or with `--format agent`, you get plain
|
|
95
|
+
lines instead, about every 10 % or 30 seconds:
|
|
96
|
+
|
|
97
|
+
```
|
|
98
|
+
[ 42%] 3.91 / 9.32 GB · 2.1 MB/s · ETA 43 min
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
`--quiet` and `--json` show no progress.
|
|
102
|
+
|
|
103
|
+
## Use it from Python
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from hfget import download
|
|
107
|
+
|
|
108
|
+
path = download("Qwen/Qwen3-0.6B", local_dir="models/Qwen3-0.6B")
|
|
109
|
+
print(path) # models/Qwen3-0.6B
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
`download` takes the same options as the command line: `filenames`, `repo_type`, `revision`,
|
|
113
|
+
`include`, `exclude`, `cache_dir`, `local_dir`, `force_download`, `token`, `max_workers` and
|
|
114
|
+
`on_progress`. It returns the folder, or the file if you asked for exactly one file by name.
|
|
115
|
+
|
|
116
|
+
To look before you download, split it in two steps:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from hfget import execute_plan, plan_download
|
|
120
|
+
|
|
121
|
+
plan = plan_download("Qwen/Qwen3-0.6B", local_dir="models/Qwen3-0.6B")
|
|
122
|
+
print(f"{len(plan.pending)} of {len(plan.files)} files, {plan.bytes_to_download} bytes to download")
|
|
123
|
+
execute_plan(plan)
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
To follow the progress, pass a callback. It receives a `DownloadProgress` snapshot about four
|
|
127
|
+
times a second:
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from hfget import DownloadProgress, download
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def show(progress: DownloadProgress) -> None:
|
|
134
|
+
print(f"{progress.done_bytes} / {progress.total_bytes} bytes")
|
|
135
|
+
for file in progress.files:
|
|
136
|
+
print(f" {file.path}: {file.state.name} {file.bytes_on_disk} / {file.size}")
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
download("Qwen/Qwen3-0.6B", filenames=["config.json"], local_dir="models/qwen", on_progress=show)
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
The library never prints. It logs through the standard `logging` module under the `hfget`
|
|
143
|
+
logger, and raises `hfget.DownloadError` (a problem retrying will not fix) or
|
|
144
|
+
`hfget.StoppedError` (the download was stopped). Partial files are always kept, so calling
|
|
145
|
+
`download` again continues them.
|
|
146
|
+
|
|
147
|
+
## Command-line options
|
|
148
|
+
|
|
149
|
+
The options match `hf download`:
|
|
150
|
+
|
|
151
|
+
| Option | Meaning |
|
|
152
|
+
| --- | --- |
|
|
153
|
+
| `REPO_ID` | The repo, for example `Qwen/Qwen3-0.6B`. `datasets/...`, `spaces/...` and `hf://...` forms also work. |
|
|
154
|
+
| `FILENAMES...` | Only these files. A name that ends with `/` means a whole folder. |
|
|
155
|
+
| `--type`, `--repo-type` | `model` (default), `dataset`, `space` or `kernel` |
|
|
156
|
+
| `--revision` | A branch, tag or commit. Default: `main`. |
|
|
157
|
+
| `--include` / `--exclude` | Glob patterns, for example `--include "*.json"`. Repeat the option for more patterns. |
|
|
158
|
+
| `--local-dir` | Save the files in this folder. |
|
|
159
|
+
| `--cache-dir` | Hugging Face cache folder, used when `--local-dir` is not set. Default: `~/.cache/huggingface/hub`. |
|
|
160
|
+
| `--force-download` | Download files again even if they are already complete. Partial files are still continued. |
|
|
161
|
+
| `--dry-run` | Only show what would be downloaded. |
|
|
162
|
+
| `--token` | Hugging Face access token. Default: your saved login (`hf auth login`) or `HF_TOKEN`. |
|
|
163
|
+
| `--max-workers` | Number of files downloaded at the same time. Default: 8. |
|
|
164
|
+
| `--format`, `--json`, `-q` / `--quiet` | Output format. `human` shows the live progress area, `agent` prints plain progress lines, `auto` (default) picks `human` in a terminal and `agent` otherwise. |
|
|
165
|
+
| `--no-truncate` | Accepted so `hf download` commands work unchanged. It has no effect. |
|
|
166
|
+
| `--version` | Print the version. |
|
|
167
|
+
|
|
168
|
+
## How it works
|
|
169
|
+
|
|
170
|
+
1. **Find the files.** The `huggingface_hub` library gets the commit of the requested revision
|
|
171
|
+
and lists every file with its size and checksum.
|
|
172
|
+
2. **Choose the files.** File names and `--include` / `--exclude` patterns are matched with the
|
|
173
|
+
library's own matching function, so the selection is the same as `hf download`.
|
|
174
|
+
3. **Skip finished files.** A file is skipped when it already exists with the right size and
|
|
175
|
+
checksum record.
|
|
176
|
+
4. **Download.** `curl --continue-at -` downloads each file into an `.incomplete` file. After
|
|
177
|
+
an interruption, curl asks the server only for the bytes that are still missing.
|
|
178
|
+
5. **Check.** When a file is complete, its checksum is compared with the one from the Hub
|
|
179
|
+
(sha256 for large files, the git blob id for small ones). A file that does not match is
|
|
180
|
+
downloaded again.
|
|
181
|
+
6. **Move into place.** The file is moved to its final name and its record is written.
|
|
182
|
+
|
|
183
|
+
Your access token is passed to curl on stdin, so it never shows in the process list.
|
|
184
|
+
|
|
185
|
+
## Where the files go
|
|
186
|
+
|
|
187
|
+
With `--local-dir models/Qwen3-0.6B`:
|
|
188
|
+
|
|
189
|
+
```
|
|
190
|
+
models/Qwen3-0.6B/
|
|
191
|
+
├── config.json, tokenizer.json, ... finished files
|
|
192
|
+
└── .cache/huggingface/download/
|
|
193
|
+
├── <file>.metadata record: commit, checksum, time
|
|
194
|
+
└── <file>.incomplete partial file while downloading
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Without `--local-dir`, files go to the Hugging Face cache (`blobs/`, `snapshots/`, `refs/`).
|
|
198
|
+
Both layouts are the same as `hf download` writes them, so you can mix the two tools.
|
|
199
|
+
|
|
200
|
+
## Platform support
|
|
201
|
+
|
|
202
|
+
- Tested on Linux and macOS, with Python 3.10 to 3.14.
|
|
203
|
+
- Windows is not tested yet. The cache layout uses symlinks, which need Developer Mode on
|
|
204
|
+
Windows.
|
|
205
|
+
- The live display is built for terminals that rewrap lines when you resize them, as most
|
|
206
|
+
modern terminals do. It is tested in tmux. In terminals that cut long lines instead, such as
|
|
207
|
+
classic xterm, shrinking the window may erase a few lines above the live area.
|
|
208
|
+
|
|
209
|
+
## Development
|
|
210
|
+
|
|
211
|
+
```bash
|
|
212
|
+
git clone https://github.com/msharara1998/hfget
|
|
213
|
+
cd hfget
|
|
214
|
+
uv sync
|
|
215
|
+
uv run ruff format && uv run ruff check && uv run ty check
|
|
216
|
+
uv run pytest # offline tests
|
|
217
|
+
uv run pytest -m network # also downloads a small real file from the Hub
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
## License
|
|
221
|
+
|
|
222
|
+
MIT. See [LICENSE](https://github.com/msharara1998/hfget/blob/main/LICENSE).
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "hfget"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Download Hugging Face models and datasets with curl, continuing partial files where they stopped."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
keywords = [
|
|
9
|
+
"huggingface",
|
|
10
|
+
"hub",
|
|
11
|
+
"download",
|
|
12
|
+
"resume",
|
|
13
|
+
"curl",
|
|
14
|
+
"models",
|
|
15
|
+
"datasets",
|
|
16
|
+
]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 3 - Alpha",
|
|
19
|
+
"Environment :: Console",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"Intended Audience :: Science/Research",
|
|
22
|
+
"Operating System :: MacOS",
|
|
23
|
+
"Operating System :: POSIX :: Linux",
|
|
24
|
+
"Programming Language :: Python :: 3",
|
|
25
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
26
|
+
"Programming Language :: Python :: 3.10",
|
|
27
|
+
"Programming Language :: Python :: 3.11",
|
|
28
|
+
"Programming Language :: Python :: 3.12",
|
|
29
|
+
"Programming Language :: Python :: 3.13",
|
|
30
|
+
"Programming Language :: Python :: 3.14",
|
|
31
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
32
|
+
"Topic :: Internet :: WWW/HTTP",
|
|
33
|
+
"Typing :: Typed",
|
|
34
|
+
]
|
|
35
|
+
requires-python = ">=3.10"
|
|
36
|
+
dependencies = ["huggingface-hub>=2.4,<3"]
|
|
37
|
+
|
|
38
|
+
[[project.authors]]
|
|
39
|
+
name = "Mohammad Sharara"
|
|
40
|
+
email = "ms198mohammad@gmail.com"
|
|
41
|
+
|
|
42
|
+
[project.urls]
|
|
43
|
+
Homepage = "https://github.com/msharara1998/hfget"
|
|
44
|
+
Repository = "https://github.com/msharara1998/hfget"
|
|
45
|
+
Issues = "https://github.com/msharara1998/hfget/issues"
|
|
46
|
+
Changelog = "https://github.com/msharara1998/hfget/blob/main/CHANGELOG.md"
|
|
47
|
+
|
|
48
|
+
[project.scripts]
|
|
49
|
+
hfget = "hfget.cli:main"
|
|
50
|
+
|
|
51
|
+
[build-system]
|
|
52
|
+
requires = ["uv_build>=0.13.9,<0.14.0"]
|
|
53
|
+
build-backend = "uv_build"
|
|
54
|
+
|
|
55
|
+
[tool.uv]
|
|
56
|
+
required-version = ">=0.13.9"
|
|
57
|
+
|
|
58
|
+
[tool.uv.build-backend]
|
|
59
|
+
module-name = "hfget"
|
|
60
|
+
|
|
61
|
+
[tool.pytest.ini_options]
|
|
62
|
+
testpaths = ["tests"]
|
|
63
|
+
addopts = "-m 'not network'"
|
|
64
|
+
markers = ["network: downloads real files from the Hugging Face Hub (run with `-m network`)"]
|
|
65
|
+
|
|
66
|
+
[tool.ruff]
|
|
67
|
+
line-length = 100
|
|
68
|
+
target-version = "py310"
|
|
69
|
+
|
|
70
|
+
[tool.ruff.lint]
|
|
71
|
+
select = [
|
|
72
|
+
"E",
|
|
73
|
+
"W",
|
|
74
|
+
"F",
|
|
75
|
+
"I",
|
|
76
|
+
"UP",
|
|
77
|
+
"B",
|
|
78
|
+
"SIM",
|
|
79
|
+
"PTH",
|
|
80
|
+
"RUF",
|
|
81
|
+
"D",
|
|
82
|
+
"N",
|
|
83
|
+
"ANN",
|
|
84
|
+
"C90",
|
|
85
|
+
"PLR09",
|
|
86
|
+
"RET",
|
|
87
|
+
"C4",
|
|
88
|
+
"A",
|
|
89
|
+
"ERA",
|
|
90
|
+
"T20",
|
|
91
|
+
"G",
|
|
92
|
+
"LOG",
|
|
93
|
+
"FBT",
|
|
94
|
+
"PIE",
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
[tool.ruff.lint.pydocstyle]
|
|
98
|
+
convention = "google"
|
|
99
|
+
|
|
100
|
+
[tool.ruff.lint.mccabe]
|
|
101
|
+
max-complexity = 10
|
|
102
|
+
|
|
103
|
+
[tool.ruff.lint.per-file-ignores]
|
|
104
|
+
"src/*/cli.py" = ["T201"]
|
|
105
|
+
"tests/**" = [
|
|
106
|
+
"D",
|
|
107
|
+
"ANN",
|
|
108
|
+
"PLR09",
|
|
109
|
+
"FBT",
|
|
110
|
+
]
|
|
111
|
+
|
|
112
|
+
[tool.ty.environment]
|
|
113
|
+
python-version = "3.10"
|
|
114
|
+
|
|
115
|
+
[tool.ty.terminal]
|
|
116
|
+
error-on-warning = true
|
|
117
|
+
|
|
118
|
+
[dependency-groups]
|
|
119
|
+
dev = [
|
|
120
|
+
"pytest>=8",
|
|
121
|
+
"ruff>=0.17.5",
|
|
122
|
+
"ty>=0.0.105",
|
|
123
|
+
]
|