hfget 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hfget-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Mohammad Sharara
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
hfget-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,255 @@
1
+ Metadata-Version: 2.4
2
+ Name: hfget
3
+ Version: 0.1.0
4
+ Summary: Download Hugging Face models and datasets with curl, continuing partial files where they stopped.
5
+ Keywords: huggingface,hub,download,resume,curl,models,datasets
6
+ Author: Mohammad Sharara
7
+ Author-email: Mohammad Sharara <ms198mohammad@gmail.com>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Environment :: Console
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Operating System :: MacOS
15
+ Classifier: Operating System :: POSIX :: Linux
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3 :: Only
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Programming Language :: Python :: 3.14
23
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
24
+ Classifier: Topic :: Internet :: WWW/HTTP
25
+ Classifier: Typing :: Typed
26
+ Requires-Dist: huggingface-hub>=2.4,<3
27
+ Requires-Python: >=3.10
28
+ Project-URL: Homepage, https://github.com/msharara1998/hfget
29
+ Project-URL: Repository, https://github.com/msharara1998/hfget
30
+ Project-URL: Issues, https://github.com/msharara1998/hfget/issues
31
+ Project-URL: Changelog, https://github.com/msharara1998/hfget/blob/main/CHANGELOG.md
32
+ Description-Content-Type: text/markdown
33
+
34
+ # hfget
35
+
36
+ [![PyPI](https://img.shields.io/pypi/v/hfget)](https://pypi.org/project/hfget/)
37
+ [![Python versions](https://img.shields.io/pypi/pyversions/hfget)](https://pypi.org/project/hfget/)
38
+ [![License: MIT](https://img.shields.io/pypi/l/hfget)](https://github.com/msharara1998/hfget/blob/main/LICENSE)
39
+ [![CI](https://github.com/msharara1998/hfget/actions/workflows/ci.yml/badge.svg)](https://github.com/msharara1998/hfget/actions/workflows/ci.yml)
40
+
41
+ Download models and datasets from the Hugging Face Hub, and **continue partial files from
42
+ where they stopped**.
43
+
44
+ `hfget` takes the same arguments as `hf download` and saves files in the same layout, so
45
+ `transformers` and the other Hugging Face tools find them. The difference: when a download is
46
+ interrupted, `hfget` keeps the partial file and continues it, byte for byte.
47
+
48
+ ## Why
49
+
50
+ `hf download` (huggingface_hub 2.4) deletes a partial file when the download is interrupted.
51
+ Running it again skips the files that finished, but a large file that was half done starts
52
+ again from zero. On a slow or unstable connection, a 5 GB weight file can then take hours more.
53
+
54
+ `hfget` keeps every byte it has received:
55
+
56
+ | What happens | What `hfget` does |
57
+ | --- | --- |
58
+ | The network drops, or no data arrives for 60 seconds | Waits and continues by itself. The wait doubles after each failed attempt, up to 60 seconds. |
59
+ | You press Ctrl+C, close the terminal, or the computer restarts | Keeps the partial file. Run the same command again to continue. |
60
+ | More than 10 attempts in a row download nothing new | Stops with a message. Run the download again to continue. |
61
+ | The repo is gated or private, or not found | Stops at once with a message, because retrying will not help. |
62
+
63
+ ## Install
64
+
65
+ ```bash
66
+ pip install hfget
67
+ ```
68
+
69
+ Or as a command-line tool with [uv](https://docs.astral.sh/uv/):
70
+
71
+ ```bash
72
+ uv tool install hfget # installs the `hfget` command
73
+ uvx hfget --help # or run it once without installing
74
+ ```
75
+
76
+ Requirements: Python 3.10 or newer, and `curl` on your `PATH` (it does the downloading).
77
+ `curl` is preinstalled on macOS and most Linux systems. Otherwise:
78
+ `sudo apt install curl` (Debian/Ubuntu) or `brew install curl` (macOS).
79
+
80
+ ## Quick start
81
+
82
+ ```bash
83
+ # Download a model into a folder
84
+ hfget Qwen/Qwen3-0.6B --local-dir models/Qwen3-0.6B
85
+
86
+ # Or into the Hugging Face cache, where transformers looks for models
87
+ hfget Qwen/Qwen3-0.6B
88
+
89
+ # See what would be downloaded, without downloading
90
+ hfget Qwen/Qwen3-0.6B --local-dir models/Qwen3-0.6B --dry-run
91
+
92
+ # Only some files
93
+ hfget Qwen/Qwen3-0.6B config.json tokenizer.json --local-dir models/Qwen3-0.6B
94
+ hfget Qwen/Qwen3-0.6B --include "*.json" --local-dir models/Qwen3-0.6B
95
+ ```
96
+
97
+ If the download stops, run the same command again. To check a folder later, also run the same
98
+ command again: it prints `All files are already downloaded.` or downloads only what is
99
+ missing.
100
+
101
+ ## What you see while downloading
102
+
103
+ In a terminal, two lines say what is downloaded and where. Below them, a live area shows one
104
+ row per active file and a total row:
105
+
106
+ ```
107
+ Qwen/Qwen3.5-4B @ 851bf6e8 → models/Qwen3.5-4B
108
+ 2 files to download (2 resuming, 3.98 GB already on disk) · 12 already complete
109
+ ✓ config.json 3.20 kB
110
+ ↓ model.safetensors-00001-of-00002.safetensors ━━━━━━━━━━━━━━╸───── 71% 3.80 / 5.30 GB 1.9 MB/s
111
+ ↻ model.safetensors-00002-of-00002.safetensors retry 2 in 4 s · no data for 60 s
112
+ Total 1/2 ━━━━━━━━━━━━━╸────── 65% 6.10 / 9.32 GB 2.1 MB/s ETA 26 min elapsed 41 min
113
+ ```
114
+
115
+ - `↓` downloads, `↻` waits before a retry, and a spinner means the checksum is being checked.
116
+ - A finished file leaves the live area and stays above it as a `✓` line.
117
+ - Speeds cover the last 10 seconds and leave out bytes from earlier runs. ETA is the bytes
118
+ left divided by the speed.
119
+ - When the window gets narrower, the speed, then the byte counts, then the bar are dropped.
120
+ Long names are shortened in the middle, so the ending stays visible. The live area redraws
121
+ correctly after the terminal rewraps lines when you resize it.
122
+ - At the end, a summary line replaces the live area, for example
123
+ `✓ Downloaded 2 files (9.32 GB) in 1 h 12 min · average 2.1 MB/s`.
124
+ - Colors follow `NO_COLOR`. Plain ASCII characters are used if the terminal cannot show the
125
+ Unicode ones.
126
+
127
+ When stderr is not a terminal (a log file or a pipe), or with `--format agent`, you get plain
128
+ lines instead, about every 10 % or 30 seconds:
129
+
130
+ ```
131
+ [ 42%] 3.91 / 9.32 GB · 2.1 MB/s · ETA 43 min
132
+ ```
133
+
134
+ `--quiet` and `--json` show no progress.
135
+
136
+ ## Use it from Python
137
+
138
+ ```python
139
+ from hfget import download
140
+
141
+ path = download("Qwen/Qwen3-0.6B", local_dir="models/Qwen3-0.6B")
142
+ print(path) # models/Qwen3-0.6B
143
+ ```
144
+
145
+ `download` takes the same options as the command line: `filenames`, `repo_type`, `revision`,
146
+ `include`, `exclude`, `cache_dir`, `local_dir`, `force_download`, `token`, `max_workers` and
147
+ `on_progress`. It returns the folder, or the file if you asked for exactly one file by name.
148
+
149
+ To look before you download, split it in two steps:
150
+
151
+ ```python
152
+ from hfget import execute_plan, plan_download
153
+
154
+ plan = plan_download("Qwen/Qwen3-0.6B", local_dir="models/Qwen3-0.6B")
155
+ print(f"{len(plan.pending)} of {len(plan.files)} files, {plan.bytes_to_download} bytes to download")
156
+ execute_plan(plan)
157
+ ```
158
+
159
+ To follow the progress, pass a callback. It receives a `DownloadProgress` snapshot about four
160
+ times a second:
161
+
162
+ ```python
163
+ from hfget import DownloadProgress, download
164
+
165
+
166
+ def show(progress: DownloadProgress) -> None:
167
+ print(f"{progress.done_bytes} / {progress.total_bytes} bytes")
168
+ for file in progress.files:
169
+ print(f" {file.path}: {file.state.name} {file.bytes_on_disk} / {file.size}")
170
+
171
+
172
+ download("Qwen/Qwen3-0.6B", filenames=["config.json"], local_dir="models/qwen", on_progress=show)
173
+ ```
174
+
175
+ The library never prints. It logs through the standard `logging` module under the `hfget`
176
+ logger, and raises `hfget.DownloadError` (a problem retrying will not fix) or
177
+ `hfget.StoppedError` (the download was stopped). Partial files are always kept, so calling
178
+ `download` again continues them.
179
+
180
+ ## Command-line options
181
+
182
+ The options match `hf download`:
183
+
184
+ | Option | Meaning |
185
+ | --- | --- |
186
+ | `REPO_ID` | The repo, for example `Qwen/Qwen3-0.6B`. `datasets/...`, `spaces/...` and `hf://...` forms also work. |
187
+ | `FILENAMES...` | Only these files. A name that ends with `/` means a whole folder. |
188
+ | `--type`, `--repo-type` | `model` (default), `dataset`, `space` or `kernel` |
189
+ | `--revision` | A branch, tag or commit. Default: `main`. |
190
+ | `--include` / `--exclude` | Glob patterns, for example `--include "*.json"`. Repeat the option for more patterns. |
191
+ | `--local-dir` | Save the files in this folder. |
192
+ | `--cache-dir` | Hugging Face cache folder, used when `--local-dir` is not set. Default: `~/.cache/huggingface/hub`. |
193
+ | `--force-download` | Download files again even if they are already complete. Partial files are still continued. |
194
+ | `--dry-run` | Only show what would be downloaded. |
195
+ | `--token` | Hugging Face access token. Default: your saved login (`hf auth login`) or `HF_TOKEN`. |
196
+ | `--max-workers` | Number of files downloaded at the same time. Default: 8. |
197
+ | `--format`, `--json`, `-q` / `--quiet` | Output format. `human` shows the live progress area, `agent` prints plain progress lines, `auto` (default) picks `human` in a terminal and `agent` otherwise. |
198
+ | `--no-truncate` | Accepted so `hf download` commands work unchanged. It has no effect. |
199
+ | `--version` | Print the version. |
200
+
201
+ ## How it works
202
+
203
+ 1. **Find the files.** The `huggingface_hub` library gets the commit of the requested revision
204
+ and lists every file with its size and checksum.
205
+ 2. **Choose the files.** File names and `--include` / `--exclude` patterns are matched with the
206
+ library's own matching function, so the selection is the same as `hf download`.
207
+ 3. **Skip finished files.** A file is skipped when it already exists with the right size and
208
+ checksum record.
209
+ 4. **Download.** `curl --continue-at -` downloads each file into an `.incomplete` file. After
210
+ an interruption, curl asks the server only for the bytes that are still missing.
211
+ 5. **Check.** When a file is complete, its checksum is compared with the one from the Hub
212
+ (sha256 for large files, the git blob id for small ones). A file that does not match is
213
+ downloaded again.
214
+ 6. **Move into place.** The file is moved to its final name and its record is written.
215
+
216
+ Your access token is passed to curl on stdin, so it never shows in the process list.
217
+
218
+ ## Where the files go
219
+
220
+ With `--local-dir models/Qwen3-0.6B`:
221
+
222
+ ```
223
+ models/Qwen3-0.6B/
224
+ ├── config.json, tokenizer.json, ... finished files
225
+ └── .cache/huggingface/download/
226
+ ├── <file>.metadata record: commit, checksum, time
227
+ └── <file>.incomplete partial file while downloading
228
+ ```
229
+
230
+ Without `--local-dir`, files go to the Hugging Face cache (`blobs/`, `snapshots/`, `refs/`).
231
+ Both layouts are the same as `hf download` writes them, so you can mix the two tools.
232
+
233
+ ## Platform support
234
+
235
+ - Tested on Linux and macOS, with Python 3.10 to 3.14.
236
+ - Windows is not tested yet. The cache layout uses symlinks, which need Developer Mode on
237
+ Windows.
238
+ - The live display is built for terminals that rewrap lines when you resize them, as most
239
+ modern terminals do. It is tested in tmux. In terminals that cut long lines instead, such as
240
+ classic xterm, shrinking the window may erase a few lines above the live area.
241
+
242
+ ## Development
243
+
244
+ ```bash
245
+ git clone https://github.com/msharara1998/hfget
246
+ cd hfget
247
+ uv sync
248
+ uv run ruff format && uv run ruff check && uv run ty check
249
+ uv run pytest # offline tests
250
+ uv run pytest -m network # also downloads a small real file from the Hub
251
+ ```
252
+
253
+ ## License
254
+
255
+ MIT. See [LICENSE](https://github.com/msharara1998/hfget/blob/main/LICENSE).
hfget-0.1.0/README.md ADDED
@@ -0,0 +1,222 @@
1
+ # hfget
2
+
3
+ [![PyPI](https://img.shields.io/pypi/v/hfget)](https://pypi.org/project/hfget/)
4
+ [![Python versions](https://img.shields.io/pypi/pyversions/hfget)](https://pypi.org/project/hfget/)
5
+ [![License: MIT](https://img.shields.io/pypi/l/hfget)](https://github.com/msharara1998/hfget/blob/main/LICENSE)
6
+ [![CI](https://github.com/msharara1998/hfget/actions/workflows/ci.yml/badge.svg)](https://github.com/msharara1998/hfget/actions/workflows/ci.yml)
7
+
8
+ Download models and datasets from the Hugging Face Hub, and **continue partial files from
9
+ where they stopped**.
10
+
11
+ `hfget` takes the same arguments as `hf download` and saves files in the same layout, so
12
+ `transformers` and the other Hugging Face tools find them. The difference: when a download is
13
+ interrupted, `hfget` keeps the partial file and continues it, byte for byte.
14
+
15
+ ## Why
16
+
17
+ `hf download` (huggingface_hub 2.4) deletes a partial file when the download is interrupted.
18
+ Running it again skips the files that finished, but a large file that was half done starts
19
+ again from zero. On a slow or unstable connection, a 5 GB weight file can then take hours more.
20
+
21
+ `hfget` keeps every byte it has received:
22
+
23
+ | What happens | What `hfget` does |
24
+ | --- | --- |
25
+ | The network drops, or no data arrives for 60 seconds | Waits and continues by itself. The wait doubles after each failed attempt, up to 60 seconds. |
26
+ | You press Ctrl+C, close the terminal, or the computer restarts | Keeps the partial file. Run the same command again to continue. |
27
+ | More than 10 attempts in a row download nothing new | Stops with a message. Run the download again to continue. |
28
+ | The repo is gated or private, or not found | Stops at once with a message, because retrying will not help. |
29
+
30
+ ## Install
31
+
32
+ ```bash
33
+ pip install hfget
34
+ ```
35
+
36
+ Or as a command-line tool with [uv](https://docs.astral.sh/uv/):
37
+
38
+ ```bash
39
+ uv tool install hfget # installs the `hfget` command
40
+ uvx hfget --help # or run it once without installing
41
+ ```
42
+
43
+ Requirements: Python 3.10 or newer, and `curl` on your `PATH` (it does the downloading).
44
+ `curl` is preinstalled on macOS and most Linux systems. Otherwise:
45
+ `sudo apt install curl` (Debian/Ubuntu) or `brew install curl` (macOS).
46
+
47
+ ## Quick start
48
+
49
+ ```bash
50
+ # Download a model into a folder
51
+ hfget Qwen/Qwen3-0.6B --local-dir models/Qwen3-0.6B
52
+
53
+ # Or into the Hugging Face cache, where transformers looks for models
54
+ hfget Qwen/Qwen3-0.6B
55
+
56
+ # See what would be downloaded, without downloading
57
+ hfget Qwen/Qwen3-0.6B --local-dir models/Qwen3-0.6B --dry-run
58
+
59
+ # Only some files
60
+ hfget Qwen/Qwen3-0.6B config.json tokenizer.json --local-dir models/Qwen3-0.6B
61
+ hfget Qwen/Qwen3-0.6B --include "*.json" --local-dir models/Qwen3-0.6B
62
+ ```
63
+
64
+ If the download stops, run the same command again. To check a folder later, also run the same
65
+ command again: it prints `All files are already downloaded.` or downloads only what is
66
+ missing.
67
+
68
+ ## What you see while downloading
69
+
70
+ In a terminal, two lines say what is downloaded and where. Below them, a live area shows one
71
+ row per active file and a total row:
72
+
73
+ ```
74
+ Qwen/Qwen3.5-4B @ 851bf6e8 → models/Qwen3.5-4B
75
+ 2 files to download (2 resuming, 3.98 GB already on disk) · 12 already complete
76
+ ✓ config.json 3.20 kB
77
+ ↓ model.safetensors-00001-of-00002.safetensors ━━━━━━━━━━━━━━╸───── 71% 3.80 / 5.30 GB 1.9 MB/s
78
+ ↻ model.safetensors-00002-of-00002.safetensors retry 2 in 4 s · no data for 60 s
79
+ Total 1/2 ━━━━━━━━━━━━━╸────── 65% 6.10 / 9.32 GB 2.1 MB/s ETA 26 min elapsed 41 min
80
+ ```
81
+
82
+ - `↓` downloads, `↻` waits before a retry, and a spinner means the checksum is being checked.
83
+ - A finished file leaves the live area and stays above it as a `✓` line.
84
+ - Speeds cover the last 10 seconds and leave out bytes from earlier runs. ETA is the bytes
85
+ left divided by the speed.
86
+ - When the window gets narrower, the speed, then the byte counts, then the bar are dropped.
87
+ Long names are shortened in the middle, so the ending stays visible. The live area redraws
88
+ correctly after the terminal rewraps lines when you resize it.
89
+ - At the end, a summary line replaces the live area, for example
90
+ `✓ Downloaded 2 files (9.32 GB) in 1 h 12 min · average 2.1 MB/s`.
91
+ - Colors follow `NO_COLOR`. Plain ASCII characters are used if the terminal cannot show the
92
+ Unicode ones.
93
+
94
+ When stderr is not a terminal (a log file or a pipe), or with `--format agent`, you get plain
95
+ lines instead, about every 10 % or 30 seconds:
96
+
97
+ ```
98
+ [ 42%] 3.91 / 9.32 GB · 2.1 MB/s · ETA 43 min
99
+ ```
100
+
101
+ `--quiet` and `--json` show no progress.
102
+
103
+ ## Use it from Python
104
+
105
+ ```python
106
+ from hfget import download
107
+
108
+ path = download("Qwen/Qwen3-0.6B", local_dir="models/Qwen3-0.6B")
109
+ print(path) # models/Qwen3-0.6B
110
+ ```
111
+
112
+ `download` takes the same options as the command line: `filenames`, `repo_type`, `revision`,
113
+ `include`, `exclude`, `cache_dir`, `local_dir`, `force_download`, `token`, `max_workers` and
114
+ `on_progress`. It returns the folder, or the file if you asked for exactly one file by name.
115
+
116
+ To look before you download, split it in two steps:
117
+
118
+ ```python
119
+ from hfget import execute_plan, plan_download
120
+
121
+ plan = plan_download("Qwen/Qwen3-0.6B", local_dir="models/Qwen3-0.6B")
122
+ print(f"{len(plan.pending)} of {len(plan.files)} files, {plan.bytes_to_download} bytes to download")
123
+ execute_plan(plan)
124
+ ```
125
+
126
+ To follow the progress, pass a callback. It receives a `DownloadProgress` snapshot about four
127
+ times a second:
128
+
129
+ ```python
130
+ from hfget import DownloadProgress, download
131
+
132
+
133
+ def show(progress: DownloadProgress) -> None:
134
+ print(f"{progress.done_bytes} / {progress.total_bytes} bytes")
135
+ for file in progress.files:
136
+ print(f" {file.path}: {file.state.name} {file.bytes_on_disk} / {file.size}")
137
+
138
+
139
+ download("Qwen/Qwen3-0.6B", filenames=["config.json"], local_dir="models/qwen", on_progress=show)
140
+ ```
141
+
142
+ The library never prints. It logs through the standard `logging` module under the `hfget`
143
+ logger, and raises `hfget.DownloadError` (a problem retrying will not fix) or
144
+ `hfget.StoppedError` (the download was stopped). Partial files are always kept, so calling
145
+ `download` again continues them.
146
+
147
+ ## Command-line options
148
+
149
+ The options match `hf download`:
150
+
151
+ | Option | Meaning |
152
+ | --- | --- |
153
+ | `REPO_ID` | The repo, for example `Qwen/Qwen3-0.6B`. `datasets/...`, `spaces/...` and `hf://...` forms also work. |
154
+ | `FILENAMES...` | Only these files. A name that ends with `/` means a whole folder. |
155
+ | `--type`, `--repo-type` | `model` (default), `dataset`, `space` or `kernel` |
156
+ | `--revision` | A branch, tag or commit. Default: `main`. |
157
+ | `--include` / `--exclude` | Glob patterns, for example `--include "*.json"`. Repeat the option for more patterns. |
158
+ | `--local-dir` | Save the files in this folder. |
159
+ | `--cache-dir` | Hugging Face cache folder, used when `--local-dir` is not set. Default: `~/.cache/huggingface/hub`. |
160
+ | `--force-download` | Download files again even if they are already complete. Partial files are still continued. |
161
+ | `--dry-run` | Only show what would be downloaded. |
162
+ | `--token` | Hugging Face access token. Default: your saved login (`hf auth login`) or `HF_TOKEN`. |
163
+ | `--max-workers` | Number of files downloaded at the same time. Default: 8. |
164
+ | `--format`, `--json`, `-q` / `--quiet` | Output format. `human` shows the live progress area, `agent` prints plain progress lines, `auto` (default) picks `human` in a terminal and `agent` otherwise. |
165
+ | `--no-truncate` | Accepted so `hf download` commands work unchanged. It has no effect. |
166
+ | `--version` | Print the version. |
167
+
168
+ ## How it works
169
+
170
+ 1. **Find the files.** The `huggingface_hub` library gets the commit of the requested revision
171
+ and lists every file with its size and checksum.
172
+ 2. **Choose the files.** File names and `--include` / `--exclude` patterns are matched with the
173
+ library's own matching function, so the selection is the same as `hf download`.
174
+ 3. **Skip finished files.** A file is skipped when it already exists with the right size and
175
+ checksum record.
176
+ 4. **Download.** `curl --continue-at -` downloads each file into an `.incomplete` file. After
177
+ an interruption, curl asks the server only for the bytes that are still missing.
178
+ 5. **Check.** When a file is complete, its checksum is compared with the one from the Hub
179
+ (sha256 for large files, the git blob id for small ones). A file that does not match is
180
+ downloaded again.
181
+ 6. **Move into place.** The file is moved to its final name and its record is written.
182
+
183
+ Your access token is passed to curl on stdin, so it never shows in the process list.
184
+
185
+ ## Where the files go
186
+
187
+ With `--local-dir models/Qwen3-0.6B`:
188
+
189
+ ```
190
+ models/Qwen3-0.6B/
191
+ ├── config.json, tokenizer.json, ... finished files
192
+ └── .cache/huggingface/download/
193
+ ├── <file>.metadata record: commit, checksum, time
194
+ └── <file>.incomplete partial file while downloading
195
+ ```
196
+
197
+ Without `--local-dir`, files go to the Hugging Face cache (`blobs/`, `snapshots/`, `refs/`).
198
+ Both layouts are the same as `hf download` writes them, so you can mix the two tools.
199
+
200
+ ## Platform support
201
+
202
+ - Tested on Linux and macOS, with Python 3.10 to 3.14.
203
+ - Windows is not tested yet. The cache layout uses symlinks, which need Developer Mode on
204
+ Windows.
205
+ - The live display is built for terminals that rewrap lines when you resize them, as most
206
+ modern terminals do. It is tested in tmux. In terminals that cut long lines instead, such as
207
+ classic xterm, shrinking the window may erase a few lines above the live area.
208
+
209
+ ## Development
210
+
211
+ ```bash
212
+ git clone https://github.com/msharara1998/hfget
213
+ cd hfget
214
+ uv sync
215
+ uv run ruff format && uv run ruff check && uv run ty check
216
+ uv run pytest # offline tests
217
+ uv run pytest -m network # also downloads a small real file from the Hub
218
+ ```
219
+
220
+ ## License
221
+
222
+ MIT. See [LICENSE](https://github.com/msharara1998/hfget/blob/main/LICENSE).
@@ -0,0 +1,123 @@
1
+ [project]
2
+ name = "hfget"
3
+ version = "0.1.0"
4
+ description = "Download Hugging Face models and datasets with curl, continuing partial files where they stopped."
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ license-files = ["LICENSE"]
8
+ keywords = [
9
+ "huggingface",
10
+ "hub",
11
+ "download",
12
+ "resume",
13
+ "curl",
14
+ "models",
15
+ "datasets",
16
+ ]
17
+ classifiers = [
18
+ "Development Status :: 3 - Alpha",
19
+ "Environment :: Console",
20
+ "Intended Audience :: Developers",
21
+ "Intended Audience :: Science/Research",
22
+ "Operating System :: MacOS",
23
+ "Operating System :: POSIX :: Linux",
24
+ "Programming Language :: Python :: 3",
25
+ "Programming Language :: Python :: 3 :: Only",
26
+ "Programming Language :: Python :: 3.10",
27
+ "Programming Language :: Python :: 3.11",
28
+ "Programming Language :: Python :: 3.12",
29
+ "Programming Language :: Python :: 3.13",
30
+ "Programming Language :: Python :: 3.14",
31
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
32
+ "Topic :: Internet :: WWW/HTTP",
33
+ "Typing :: Typed",
34
+ ]
35
+ requires-python = ">=3.10"
36
+ dependencies = ["huggingface-hub>=2.4,<3"]
37
+
38
+ [[project.authors]]
39
+ name = "Mohammad Sharara"
40
+ email = "ms198mohammad@gmail.com"
41
+
42
+ [project.urls]
43
+ Homepage = "https://github.com/msharara1998/hfget"
44
+ Repository = "https://github.com/msharara1998/hfget"
45
+ Issues = "https://github.com/msharara1998/hfget/issues"
46
+ Changelog = "https://github.com/msharara1998/hfget/blob/main/CHANGELOG.md"
47
+
48
+ [project.scripts]
49
+ hfget = "hfget.cli:main"
50
+
51
+ [build-system]
52
+ requires = ["uv_build>=0.13.9,<0.14.0"]
53
+ build-backend = "uv_build"
54
+
55
+ [tool.uv]
56
+ required-version = ">=0.13.9"
57
+
58
+ [tool.uv.build-backend]
59
+ module-name = "hfget"
60
+
61
+ [tool.pytest.ini_options]
62
+ testpaths = ["tests"]
63
+ addopts = "-m 'not network'"
64
+ markers = ["network: downloads real files from the Hugging Face Hub (run with `-m network`)"]
65
+
66
+ [tool.ruff]
67
+ line-length = 100
68
+ target-version = "py310"
69
+
70
+ [tool.ruff.lint]
71
+ select = [
72
+ "E",
73
+ "W",
74
+ "F",
75
+ "I",
76
+ "UP",
77
+ "B",
78
+ "SIM",
79
+ "PTH",
80
+ "RUF",
81
+ "D",
82
+ "N",
83
+ "ANN",
84
+ "C90",
85
+ "PLR09",
86
+ "RET",
87
+ "C4",
88
+ "A",
89
+ "ERA",
90
+ "T20",
91
+ "G",
92
+ "LOG",
93
+ "FBT",
94
+ "PIE",
95
+ ]
96
+
97
+ [tool.ruff.lint.pydocstyle]
98
+ convention = "google"
99
+
100
+ [tool.ruff.lint.mccabe]
101
+ max-complexity = 10
102
+
103
+ [tool.ruff.lint.per-file-ignores]
104
+ "src/*/cli.py" = ["T201"]
105
+ "tests/**" = [
106
+ "D",
107
+ "ANN",
108
+ "PLR09",
109
+ "FBT",
110
+ ]
111
+
112
+ [tool.ty.environment]
113
+ python-version = "3.10"
114
+
115
+ [tool.ty.terminal]
116
+ error-on-warning = true
117
+
118
+ [dependency-groups]
119
+ dev = [
120
+ "pytest>=8",
121
+ "ruff>=0.17.5",
122
+ "ty>=0.0.105",
123
+ ]