nercone-fastget 6.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nercone_fastget-6.0.0/PKG-INFO +135 -0
- nercone_fastget-6.0.0/README.md +119 -0
- nercone_fastget-6.0.0/pyproject.toml +27 -0
- nercone_fastget-6.0.0/src/nercone_fastget/__init__.py +0 -0
- nercone_fastget-6.0.0/src/nercone_fastget/__main__.py +199 -0
- nercone_fastget-6.0.0/src/nercone_fastget/fastget.py +259 -0
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: nercone-fastget
|
|
3
|
+
Version: 6.0.0
|
|
4
|
+
Summary: High-speed File Downloading Tool
|
|
5
|
+
Author: Nercone
|
|
6
|
+
Author-email: Nercone <nercone@diamondgotcat.net>
|
|
7
|
+
License: MIT
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Dist: rich
|
|
12
|
+
Requires-Dist: httpx[http2]
|
|
13
|
+
Requires-Dist: uvloop ; sys_platform != 'win32'
|
|
14
|
+
Requires-Python: >=3.8
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
<img width="1920" alt="fastget" src="https://github.com/user-attachments/assets/fb4b6013-f65b-4c34-ae86-848011e7bfcf" />
|
|
19
|
+
|
|
20
|
+
# FastGet
|
|
21
|
+
High-speed File Downloading Tool
|
|
22
|
+
|
|
23
|
+
## How
|
|
24
|
+
It's using Multiple Thread Download.
|
|
25
|
+
|
|
26
|
+
This is need Range-select feature, in Server-side.
|
|
27
|
+
|
|
28
|
+
## Requiments
|
|
29
|
+
- CPython 3.9+
|
|
30
|
+
- `uv` [PyPI↗︎](https://pypi.org/project/uv/) or `pip3` [PyPI↗︎](https://pypi.org/project/pip/)
|
|
31
|
+
- `nercone-modern` [PyPI↗︎](https://pypi.org/project/nercone-modern/) for Modern CLI Visual
|
|
32
|
+
- `httpx[http2]` [PyPI↗︎](https://pypi.org/project/httpx/) for HTTP Connection
|
|
33
|
+
- `uvloop` [PyPI↗︎](https://pypi.org/project/uvloop/) for Fast Asynchronous Processing (non-Windows only)
|
|
34
|
+
|
|
35
|
+
## Installation
|
|
36
|
+
|
|
37
|
+
### uv (recommended)
|
|
38
|
+
```
|
|
39
|
+
uv tool install nercone-fastget
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### pip3
|
|
43
|
+
|
|
44
|
+
**System Python:**
|
|
45
|
+
```
|
|
46
|
+
pip3 install nercone-fastget --break-system-packages
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
**Venv Python:**
|
|
50
|
+
```
|
|
51
|
+
pip3 install nercone-fastget
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Update
|
|
55
|
+
|
|
56
|
+
### uv
|
|
57
|
+
```
|
|
58
|
+
uv tool install nercone-fastget --upgrade
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### pip3
|
|
62
|
+
|
|
63
|
+
**System Python:**
|
|
64
|
+
```
|
|
65
|
+
pip3 install nercone-fastget --upgrade --break-system-packages
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
**Venv Python:**
|
|
69
|
+
```
|
|
70
|
+
pip3 install nercone-fastget --upgrade
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Usage
|
|
74
|
+
|
|
75
|
+
### Show helps
|
|
76
|
+
```
|
|
77
|
+
fastget [-h] [--help]
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```
|
|
81
|
+
nercone@demo ~/Downloads> fastget -h
|
|
82
|
+
usage: fastget [-h] [-o OUTPUT] [-X {GET,POST}] [-d DATA] [-H HEADER] [-t THREADS] [-p] [-s] [-m] [--no-verify] [--no-info] [--no-http1] [--no-http2] url
|
|
83
|
+
|
|
84
|
+
Modern High-Performance Downloader
|
|
85
|
+
|
|
86
|
+
positional arguments:
|
|
87
|
+
url Target URL
|
|
88
|
+
|
|
89
|
+
options:
|
|
90
|
+
-h, --help show this help message and exit
|
|
91
|
+
-o OUTPUT, --output OUTPUT
|
|
92
|
+
File destination
|
|
93
|
+
-X {GET,POST}, --method {GET,POST}
|
|
94
|
+
HTTP method (GET/POST)
|
|
95
|
+
-d DATA, --data DATA Data for POST method
|
|
96
|
+
-H HEADER, --header HEADER
|
|
97
|
+
Custom Headers
|
|
98
|
+
-t THREADS, --threads THREADS
|
|
99
|
+
Number of threads to use for downloading
|
|
100
|
+
-p, --print Output data directly to stdout without saving to a file
|
|
101
|
+
-s, --storage, --low-memory
|
|
102
|
+
Utilize storage efficiently to reduce memory usage during internal processes such as downloading and merging.
|
|
103
|
+
-m, --memory, --low-storage
|
|
104
|
+
Utilize memory efficiently to reduce maximum concurrent storage usage during internal processes such as downloading and merging.
|
|
105
|
+
--no-verify In the case of HTTPS, if a secure connection cannot be established, the system will continue to operate normally.
|
|
106
|
+
--no-info Suppresses all displays such as progress bars. If --print is used, only data is output to stdout.
|
|
107
|
+
--no-http1 Do not use HTTP/1 or HTTP/1.1
|
|
108
|
+
--no-http2 Do not use HTTP/2
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Download
|
|
112
|
+
```
|
|
113
|
+
fastget [-o OUTPUT] [-X {GET,POST}] [-d DATA] [-H HEADER] [-t THREADS] [-p] [-s] [-m] [--no-verify] [--no-info] [--no-http1] [--no-http2] url
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
```
|
|
117
|
+
nercone@demo ~> fastget https://download.fedoraproject.org/pub/fedora/linux/releases/43/Server/aarch64/iso/Fedora-Server-netinst-aarch64-43-1.6.iso
|
|
118
|
+
[2025-12-05T17:24:31Z INFO fastget] File size: 1,198,647,296 bytes
|
|
119
|
+
[2025-12-05T17:24:31Z INFO fastget] Connection Type: HTTPS (HTTP/1.1, TLS, Verified)
|
|
120
|
+
[2025-12-05T17:24:31Z INFO fastget] Threads: 8
|
|
121
|
+
[-------------------------------------] Total DONE No Message
|
|
122
|
+
[-------------------------------------] DL #1 DONE No Message
|
|
123
|
+
[-------------------------------------] DL #2 DONE No Message
|
|
124
|
+
[-------------------------------------] DL #3 DONE No Message
|
|
125
|
+
[-------------------------------------] DL #4 DONE No Message
|
|
126
|
+
[-------------------------------------] DL #5 DONE No Message
|
|
127
|
+
[-------------------------------------] DL #6 DONE No Message
|
|
128
|
+
[-------------------------------------] DL #7 DONE No Message
|
|
129
|
+
[-------------------------------------] DL #8 DONE No Message
|
|
130
|
+
[-------------------------------------] Marge DONE No Message
|
|
131
|
+
[2025-12-05T17:24:46Z INFO fastget] Completed in 16222.00ms
|
|
132
|
+
[2025-12-05T17:24:46Z INFO fastget] Saved to: Fedora-Server-netinst-aarch64-43-1.6.iso
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+

|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
|
|
2
|
+
<img width="1920" alt="fastget" src="https://github.com/user-attachments/assets/fb4b6013-f65b-4c34-ae86-848011e7bfcf" />
|
|
3
|
+
|
|
4
|
+
# FastGet
|
|
5
|
+
High-speed File Downloading Tool
|
|
6
|
+
|
|
7
|
+
## How
|
|
8
|
+
It's using Multiple Thread Download.
|
|
9
|
+
|
|
10
|
+
This is need Range-select feature, in Server-side.
|
|
11
|
+
|
|
12
|
+
## Requiments
|
|
13
|
+
- CPython 3.9+
|
|
14
|
+
- `uv` [PyPI↗︎](https://pypi.org/project/uv/) or `pip3` [PyPI↗︎](https://pypi.org/project/pip/)
|
|
15
|
+
- `nercone-modern` [PyPI↗︎](https://pypi.org/project/nercone-modern/) for Modern CLI Visual
|
|
16
|
+
- `httpx[http2]` [PyPI↗︎](https://pypi.org/project/httpx/) for HTTP Connection
|
|
17
|
+
- `uvloop` [PyPI↗︎](https://pypi.org/project/uvloop/) for Fast Asynchronous Processing (non-Windows only)
|
|
18
|
+
|
|
19
|
+
## Installation
|
|
20
|
+
|
|
21
|
+
### uv (recommended)
|
|
22
|
+
```
|
|
23
|
+
uv tool install nercone-fastget
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
### pip3
|
|
27
|
+
|
|
28
|
+
**System Python:**
|
|
29
|
+
```
|
|
30
|
+
pip3 install nercone-fastget --break-system-packages
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
**Venv Python:**
|
|
34
|
+
```
|
|
35
|
+
pip3 install nercone-fastget
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Update
|
|
39
|
+
|
|
40
|
+
### uv
|
|
41
|
+
```
|
|
42
|
+
uv tool install nercone-fastget --upgrade
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
### pip3
|
|
46
|
+
|
|
47
|
+
**System Python:**
|
|
48
|
+
```
|
|
49
|
+
pip3 install nercone-fastget --upgrade --break-system-packages
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
**Venv Python:**
|
|
53
|
+
```
|
|
54
|
+
pip3 install nercone-fastget --upgrade
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Usage
|
|
58
|
+
|
|
59
|
+
### Show helps
|
|
60
|
+
```
|
|
61
|
+
fastget [-h] [--help]
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
nercone@demo ~/Downloads> fastget -h
|
|
66
|
+
usage: fastget [-h] [-o OUTPUT] [-X {GET,POST}] [-d DATA] [-H HEADER] [-t THREADS] [-p] [-s] [-m] [--no-verify] [--no-info] [--no-http1] [--no-http2] url
|
|
67
|
+
|
|
68
|
+
Modern High-Performance Downloader
|
|
69
|
+
|
|
70
|
+
positional arguments:
|
|
71
|
+
url Target URL
|
|
72
|
+
|
|
73
|
+
options:
|
|
74
|
+
-h, --help show this help message and exit
|
|
75
|
+
-o OUTPUT, --output OUTPUT
|
|
76
|
+
File destination
|
|
77
|
+
-X {GET,POST}, --method {GET,POST}
|
|
78
|
+
HTTP method (GET/POST)
|
|
79
|
+
-d DATA, --data DATA Data for POST method
|
|
80
|
+
-H HEADER, --header HEADER
|
|
81
|
+
Custom Headers
|
|
82
|
+
-t THREADS, --threads THREADS
|
|
83
|
+
Number of threads to use for downloading
|
|
84
|
+
-p, --print Output data directly to stdout without saving to a file
|
|
85
|
+
-s, --storage, --low-memory
|
|
86
|
+
Utilize storage efficiently to reduce memory usage during internal processes such as downloading and merging.
|
|
87
|
+
-m, --memory, --low-storage
|
|
88
|
+
Utilize memory efficiently to reduce maximum concurrent storage usage during internal processes such as downloading and merging.
|
|
89
|
+
--no-verify In the case of HTTPS, if a secure connection cannot be established, the system will continue to operate normally.
|
|
90
|
+
--no-info Suppresses all displays such as progress bars. If --print is used, only data is output to stdout.
|
|
91
|
+
--no-http1 Do not use HTTP/1 or HTTP/1.1
|
|
92
|
+
--no-http2 Do not use HTTP/2
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Download
|
|
96
|
+
```
|
|
97
|
+
fastget [-o OUTPUT] [-X {GET,POST}] [-d DATA] [-H HEADER] [-t THREADS] [-p] [-s] [-m] [--no-verify] [--no-info] [--no-http1] [--no-http2] url
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
nercone@demo ~> fastget https://download.fedoraproject.org/pub/fedora/linux/releases/43/Server/aarch64/iso/Fedora-Server-netinst-aarch64-43-1.6.iso
|
|
102
|
+
[2025-12-05T17:24:31Z INFO fastget] File size: 1,198,647,296 bytes
|
|
103
|
+
[2025-12-05T17:24:31Z INFO fastget] Connection Type: HTTPS (HTTP/1.1, TLS, Verified)
|
|
104
|
+
[2025-12-05T17:24:31Z INFO fastget] Threads: 8
|
|
105
|
+
[-------------------------------------] Total DONE No Message
|
|
106
|
+
[-------------------------------------] DL #1 DONE No Message
|
|
107
|
+
[-------------------------------------] DL #2 DONE No Message
|
|
108
|
+
[-------------------------------------] DL #3 DONE No Message
|
|
109
|
+
[-------------------------------------] DL #4 DONE No Message
|
|
110
|
+
[-------------------------------------] DL #5 DONE No Message
|
|
111
|
+
[-------------------------------------] DL #6 DONE No Message
|
|
112
|
+
[-------------------------------------] DL #7 DONE No Message
|
|
113
|
+
[-------------------------------------] DL #8 DONE No Message
|
|
114
|
+
[-------------------------------------] Marge DONE No Message
|
|
115
|
+
[2025-12-05T17:24:46Z INFO fastget] Completed in 16222.00ms
|
|
116
|
+
[2025-12-05T17:24:46Z INFO fastget] Saved to: Fedora-Server-netinst-aarch64-43-1.6.iso
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+

|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.9.5,<0.10.0"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "nercone-fastget"
|
|
7
|
+
version = "6.0.0"
|
|
8
|
+
description = "High-speed File Downloading Tool"
|
|
9
|
+
readme = { file = "README.md", content-type = "text/markdown" }
|
|
10
|
+
authors = [
|
|
11
|
+
{ name = "Nercone", email = "nercone@diamondgotcat.net" }
|
|
12
|
+
]
|
|
13
|
+
license = { text = "MIT" }
|
|
14
|
+
requires-python = ">=3.8"
|
|
15
|
+
dependencies = [
|
|
16
|
+
"rich",
|
|
17
|
+
"httpx[http2]",
|
|
18
|
+
"uvloop; platform_system != 'Windows'"
|
|
19
|
+
]
|
|
20
|
+
classifiers = [
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"License :: OSI Approved :: MIT License",
|
|
23
|
+
"Operating System :: OS Independent"
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
fastget = "nercone_fastget.__main__:main"
|
|
File without changes
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# ╭──────────────────────────────────────╮
|
|
2
|
+
# │ __main__.py on nercone-fastget │
|
|
3
|
+
# │ Nercone <nercone@diamondgotcat.net> │
|
|
4
|
+
# │ Made by Nercone / MIT License │
|
|
5
|
+
# │ Copyright (c) 2025 DiamondGotCat │
|
|
6
|
+
# ╰──────────────────────────────────────╯
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import asyncio
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
from urllib.parse import urlparse, unquote
|
|
13
|
+
from typing import Dict, List, Optional
|
|
14
|
+
|
|
15
|
+
from rich.console import Console
|
|
16
|
+
from rich.panel import Panel
|
|
17
|
+
from rich.filesize import decimal
|
|
18
|
+
from rich.progress import (
|
|
19
|
+
Progress,
|
|
20
|
+
BarColumn,
|
|
21
|
+
TextColumn,
|
|
22
|
+
DownloadColumn,
|
|
23
|
+
TransferSpeedColumn,
|
|
24
|
+
TimeRemainingColumn,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
from .fastget import (
|
|
28
|
+
FastGetSession,
|
|
29
|
+
ProgressCallback,
|
|
30
|
+
FastGetError,
|
|
31
|
+
DEFAULT_THREADS,
|
|
32
|
+
VERSION,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
class RichProgressCallback(ProgressCallback):
|
|
36
|
+
def __init__(self, console: "Console", progress: "Progress"):
|
|
37
|
+
self.console = console
|
|
38
|
+
self.progress = progress
|
|
39
|
+
self.overall_task: Optional[int] = None
|
|
40
|
+
self.worker_tasks: List[int] = []
|
|
41
|
+
self.merge_task: Optional[int] = None
|
|
42
|
+
|
|
43
|
+
async def on_start(self, total_size: int, threads: int, http_version: str, final_url: str, verify_was_enabled: bool) -> None:
|
|
44
|
+
self.console.print(Panel(
|
|
45
|
+
f"[bold]URL[/bold]: {final_url}\n"
|
|
46
|
+
f"[bold]File Size[/bold]: {decimal(total_size)}\n"
|
|
47
|
+
f"[bold]Threads[/bold]: {threads}\n"
|
|
48
|
+
f"[bold]HTTP Version[/bold]: {http_version}\n"
|
|
49
|
+
f"[bold]SSL/TLS Verify[/bold]: {'Enabled' if verify_was_enabled else '[yellow]Disabled[/yellow]'}",
|
|
50
|
+
title="Download Information",
|
|
51
|
+
border_style="green",
|
|
52
|
+
expand=False
|
|
53
|
+
))
|
|
54
|
+
self.overall_task = self.progress.add_task("[bold green]Download", total=total_size)
|
|
55
|
+
if threads > 1:
|
|
56
|
+
part_size = total_size // threads
|
|
57
|
+
for i in range(threads):
|
|
58
|
+
task_total = total_size - (part_size * i) if i == threads - 1 else part_size
|
|
59
|
+
self.worker_tasks.append(self.progress.add_task(f"Worker {i+1}", total=task_total))
|
|
60
|
+
|
|
61
|
+
async def on_update(self, worker_id: int, loaded: int) -> None:
|
|
62
|
+
if self.overall_task is not None:
|
|
63
|
+
self.progress.update(self.overall_task, advance=loaded)
|
|
64
|
+
if self.worker_tasks and worker_id < len(self.worker_tasks):
|
|
65
|
+
self.progress.update(self.worker_tasks[worker_id], advance=loaded)
|
|
66
|
+
|
|
67
|
+
async def on_merge_start(self, total_size: int) -> None:
|
|
68
|
+
self.merge_task = self.progress.add_task("[bold cyan]Merge", total=total_size)
|
|
69
|
+
|
|
70
|
+
async def on_merge_update(self, loaded: int) -> None:
|
|
71
|
+
if self.merge_task is not None:
|
|
72
|
+
self.progress.update(self.merge_task, advance=loaded)
|
|
73
|
+
|
|
74
|
+
async def on_slowdown(self, msg: str) -> None:
|
|
75
|
+
self.console.print(f"[yellow]W[/yellow] {msg}")
|
|
76
|
+
|
|
77
|
+
async def on_error(self, msg: str) -> None:
|
|
78
|
+
if not self.progress.finished:
|
|
79
|
+
self.progress.stop()
|
|
80
|
+
self.console.print(f"[bold red]E[/bold red] {msg}")
|
|
81
|
+
|
|
82
|
+
class SilentProgressCallback(ProgressCallback):
|
|
83
|
+
def __init__(self, console: "Console"):
|
|
84
|
+
self.console = console
|
|
85
|
+
|
|
86
|
+
async def on_error(self, msg: str) -> None:
|
|
87
|
+
self.console.print(f"[bold red]E[/bold red] {msg}")
|
|
88
|
+
|
|
89
|
+
async def main():
|
|
90
|
+
parser = argparse.ArgumentParser(prog="fastget", description=f"High-speed File Downloading Tool", formatter_class=argparse.RawTextHelpFormatter)
|
|
91
|
+
parser.add_argument("url", help="URL to download from.")
|
|
92
|
+
parser.add_argument("-o", "--output", help="Path to save the file. If not specified, it's inferred from the URL.")
|
|
93
|
+
parser.add_argument("-t", "--threads", type=int, default=DEFAULT_THREADS, help=f"Number of parallel connections. (default: {DEFAULT_THREADS})")
|
|
94
|
+
parser.add_argument( "-X", "--request", default="GET", choices=["GET", "POST"], help="HTTP method to use. (default: GET)")
|
|
95
|
+
parser.add_argument( "-H", "--header", action="append", help="Custom header to send with the request (e.g., 'User-Agent: my-app/1.0').\nCan be specified multiple times.")
|
|
96
|
+
parser.add_argument("-d", "--data", help="Data to send in a POST request.")
|
|
97
|
+
parser.add_argument("--no-verify", action="store_false", dest="verify", help="Disable SSL/TLS certificate verification.")
|
|
98
|
+
parser.add_argument("--no-info", action="store_true", help="Silent mode. Suppress progress bar and other info.\nErrors are still printed to stderr.")
|
|
99
|
+
parser.add_argument("--no-http1", action="store_false", dest="http1", help="Disable HTTP/1.x and force HTTP/2.")
|
|
100
|
+
parser.add_argument("--no-http2", action="store_false", dest="http2", help="Disable HTTP/2 and force HTTP/1.x.")
|
|
101
|
+
parser.add_argument("-v", "--version", action="version", version=f"%(prog)s {VERSION}")
|
|
102
|
+
args = parser.parse_args()
|
|
103
|
+
|
|
104
|
+
if args.no_info:
|
|
105
|
+
console = Console(stderr=True, quiet=True)
|
|
106
|
+
callback = SilentProgressCallback(console)
|
|
107
|
+
progress = None
|
|
108
|
+
else:
|
|
109
|
+
console = Console()
|
|
110
|
+
progress = Progress(
|
|
111
|
+
TextColumn("[progress.description]{task.description}"),
|
|
112
|
+
BarColumn(),
|
|
113
|
+
DownloadColumn(),
|
|
114
|
+
TransferSpeedColumn(),
|
|
115
|
+
TextColumn("ETA:"),
|
|
116
|
+
TimeRemainingColumn(),
|
|
117
|
+
console=console,
|
|
118
|
+
transient=True
|
|
119
|
+
)
|
|
120
|
+
callback = RichProgressCallback(console, progress)
|
|
121
|
+
|
|
122
|
+
output_path = args.output
|
|
123
|
+
if not output_path:
|
|
124
|
+
try:
|
|
125
|
+
parsed_url = urlparse(args.url)
|
|
126
|
+
filename = os.path.basename(unquote(parsed_url.path))
|
|
127
|
+
if not filename:
|
|
128
|
+
console.print("[bold red]E[/bold red] Cannot determine output filename from URL. Please specify it with the -o/--output option.")
|
|
129
|
+
sys.exit(1)
|
|
130
|
+
output_path = filename
|
|
131
|
+
except Exception as e:
|
|
132
|
+
console.print(f"[bold red]E[/bold red] Invalid URL provided: {e}")
|
|
133
|
+
sys.exit(1)
|
|
134
|
+
|
|
135
|
+
headers: Dict[str, str] = {}
|
|
136
|
+
if args.header:
|
|
137
|
+
for h in args.header:
|
|
138
|
+
if ":" in h:
|
|
139
|
+
key, value = h.split(":", 1)
|
|
140
|
+
headers[key.strip()] = value.strip()
|
|
141
|
+
elif "=" in h:
|
|
142
|
+
key, value = h.split("=", 1)
|
|
143
|
+
headers[key.strip()] = value.strip()
|
|
144
|
+
else:
|
|
145
|
+
console.print(f"[yellow]W[/yellow] Ignoring malformed header: {h}")
|
|
146
|
+
|
|
147
|
+
session = FastGetSession(
|
|
148
|
+
max_threads=args.threads,
|
|
149
|
+
http1=args.http1,
|
|
150
|
+
http2=args.http2,
|
|
151
|
+
verify=args.verify,
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
try:
|
|
155
|
+
downloader = session.process(
|
|
156
|
+
method=args.request.upper(),
|
|
157
|
+
url=args.url,
|
|
158
|
+
output=output_path,
|
|
159
|
+
data=args.data,
|
|
160
|
+
headers=headers,
|
|
161
|
+
callback=callback,
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
if progress:
|
|
165
|
+
with progress:
|
|
166
|
+
result = await downloader
|
|
167
|
+
else:
|
|
168
|
+
result = await downloader
|
|
169
|
+
|
|
170
|
+
if not args.no_info:
|
|
171
|
+
console.print(f"[bold green]✔ Downloaded[/bold green] Saved to '{result}'")
|
|
172
|
+
|
|
173
|
+
except (FastGetError, Exception) as e:
|
|
174
|
+
if not isinstance(callback, RichProgressCallback) or (isinstance(callback, RichProgressCallback) and callback.progress.finished):
|
|
175
|
+
console.print(f"[bold red]E[/bold red] {e}")
|
|
176
|
+
|
|
177
|
+
if output_path:
|
|
178
|
+
if os.path.exists(output_path):
|
|
179
|
+
try: os.remove(output_path)
|
|
180
|
+
except OSError: pass
|
|
181
|
+
|
|
182
|
+
out_dir = os.path.dirname(output_path) or '.'
|
|
183
|
+
out_base = os.path.basename(output_path)
|
|
184
|
+
for i in range(args.threads):
|
|
185
|
+
part_path = os.path.join(out_dir, f"{out_base}.part{i}")
|
|
186
|
+
if os.path.exists(part_path):
|
|
187
|
+
try: os.remove(part_path)
|
|
188
|
+
except OSError: pass
|
|
189
|
+
sys.exit(1)
|
|
190
|
+
|
|
191
|
+
def run():
|
|
192
|
+
try:
|
|
193
|
+
asyncio.run(main())
|
|
194
|
+
except KeyboardInterrupt:
|
|
195
|
+
Console().print("\n[yellow]W[/yellow] Aborted.")
|
|
196
|
+
sys.exit(130)
|
|
197
|
+
|
|
198
|
+
if __name__ == "__main__":
|
|
199
|
+
run()
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
# ╭──────────────────────────────────────╮
|
|
2
|
+
# │ fastget.py on nercone-fastget │
|
|
3
|
+
# │ Nercone <nercone@diamondgotcat.net> │
|
|
4
|
+
# │ Made by Nercone / MIT License │
|
|
5
|
+
# │ Copyright (c) 2025 DiamondGotCat │
|
|
6
|
+
# ╰──────────────────────────────────────╯
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import asyncio
|
|
10
|
+
import httpx
|
|
11
|
+
from importlib.metadata import version
|
|
12
|
+
from urllib.parse import urlparse, unquote
|
|
13
|
+
from typing import Union, Optional, Dict, Any, TypeVar, Coroutine, Awaitable
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
VERSION = version("nercone-fastget")
|
|
17
|
+
except Exception:
|
|
18
|
+
VERSION = "0.0.0"
|
|
19
|
+
|
|
20
|
+
DEFAULT_CHUNK_SIZE = 1024 * 64
|
|
21
|
+
DEFAULT_TIMEOUT = 30.0
|
|
22
|
+
DEFAULT_RETRIES = 3
|
|
23
|
+
DEFAULT_THREADS = 8
|
|
24
|
+
|
|
25
|
+
T = TypeVar("T")
|
|
26
|
+
|
|
27
|
+
class FastGetError(Exception):
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
class ProgressCallback:
|
|
31
|
+
async def on_start(self, total_size: int, threads: int, http_version: str, final_url: str, verify_was_enabled: bool) -> None:
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
async def on_update(self, worker_id: int, loaded: int) -> None:
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
async def on_complete(self) -> None:
|
|
38
|
+
pass
|
|
39
|
+
|
|
40
|
+
async def on_merge_start(self, total_size: int) -> None:
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
async def on_merge_update(self, loaded: int) -> None:
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
async def on_merge_complete(self) -> None:
|
|
47
|
+
pass
|
|
48
|
+
|
|
49
|
+
async def on_slowdown(self, msg: str) -> None:
|
|
50
|
+
pass
|
|
51
|
+
|
|
52
|
+
async def on_error(self, msg: str) -> None:
|
|
53
|
+
pass
|
|
54
|
+
|
|
55
|
+
class FastGetResponse:
|
|
56
|
+
def __init__(self, original: httpx.Response, content: bytes):
|
|
57
|
+
self._r = original
|
|
58
|
+
self.content = content
|
|
59
|
+
self.url = str(original.url)
|
|
60
|
+
self.status_code = original.status_code
|
|
61
|
+
self.headers = original.headers
|
|
62
|
+
self.http_version = original.http_version
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def text(self) -> str:
|
|
66
|
+
return self._r.text
|
|
67
|
+
|
|
68
|
+
def json(self, **kwargs) -> Any:
|
|
69
|
+
return self._r.json(**kwargs)
|
|
70
|
+
|
|
71
|
+
class FastGetSession:
|
|
72
|
+
def __init__(self, max_threads: int = DEFAULT_THREADS, http1: bool = True, http2: bool = True, verify: bool = True, follow_redirects: bool = True):
|
|
73
|
+
self.max_threads = max_threads
|
|
74
|
+
self.client_args: Dict[str, Any] = {
|
|
75
|
+
"http1": http1,
|
|
76
|
+
"http2": http2,
|
|
77
|
+
"verify": verify,
|
|
78
|
+
"follow_redirects": follow_redirects,
|
|
79
|
+
"timeout": DEFAULT_TIMEOUT
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async def _get_info(self, client: httpx.AsyncClient, method: str, url: str, headers: dict[str,str]) -> tuple[int, bool, bool, Optional[httpx.Response]]:
|
|
83
|
+
headers["User-Agent"] = f'FastGet/{VERSION} (Getting Informations; https://github.com/DiamondGotCat/nercone-fastget/)'
|
|
84
|
+
|
|
85
|
+
if method.upper() != "GET":
|
|
86
|
+
return 0, False, False, None
|
|
87
|
+
|
|
88
|
+
try:
|
|
89
|
+
head_resp = await client.head(url=url, headers=headers)
|
|
90
|
+
|
|
91
|
+
if head_resp.status_code < 400:
|
|
92
|
+
resp = head_resp
|
|
93
|
+
else:
|
|
94
|
+
request = client.build_request(method=method, url=url, headers=headers)
|
|
95
|
+
resp = await client.send(request, stream=True)
|
|
96
|
+
await resp.aclose()
|
|
97
|
+
|
|
98
|
+
size = int(resp.headers.get("content-length", 0))
|
|
99
|
+
accept_ranges = resp.headers.get("accept-ranges", "").lower() == "bytes"
|
|
100
|
+
reject_fg = resp.headers.get("rejectfastget", "").lower() in ["true", "1", "yes"]
|
|
101
|
+
|
|
102
|
+
return size, accept_ranges, reject_fg, resp
|
|
103
|
+
|
|
104
|
+
except Exception:
|
|
105
|
+
return 0, False, True, None
|
|
106
|
+
|
|
107
|
+
async def _download_worker(self, client: httpx.AsyncClient, method: str, url: str, start: int, end: int, worker_id: int, total_threads: int, part_path: str, callback: ProgressCallback, headers: dict[str,str]) -> None:
|
|
108
|
+
headers["Range"] = f"bytes={start}-{end}"
|
|
109
|
+
headers["User-Agent"] = f'FastGet/{VERSION} (Downloading with {total_threads} Thread(s), Connection No. {worker_id}; +https://github.com/DiamondGotCat/nercone-fastget/)'
|
|
110
|
+
|
|
111
|
+
for attempt in range(DEFAULT_RETRIES):
|
|
112
|
+
try:
|
|
113
|
+
async with client.stream(method=method, url=url, headers=headers) as response:
|
|
114
|
+
response.raise_for_status()
|
|
115
|
+
|
|
116
|
+
with open(part_path, "wb") as f:
|
|
117
|
+
async for chunk in response.aiter_bytes(chunk_size=DEFAULT_CHUNK_SIZE):
|
|
118
|
+
if not chunk:
|
|
119
|
+
break
|
|
120
|
+
|
|
121
|
+
f.write(chunk)
|
|
122
|
+
await callback.on_update(worker_id, len(chunk))
|
|
123
|
+
return
|
|
124
|
+
|
|
125
|
+
except (httpx.RequestError, httpx.HTTPStatusError) as e:
|
|
126
|
+
if attempt == DEFAULT_RETRIES - 1:
|
|
127
|
+
await callback.on_error(f"Worker {worker_id} failed: {e}")
|
|
128
|
+
raise
|
|
129
|
+
await asyncio.sleep(1)
|
|
130
|
+
|
|
131
|
+
async def process(self, method: str, url: str, output: Optional[str] = None, data: Any = None, json: Any = None, params: Any = None, headers: Optional[Dict[str, str]] = None, callback: Optional[ProgressCallback] = None) -> Union[str, FastGetResponse]:
|
|
132
|
+
callback = callback or ProgressCallback()
|
|
133
|
+
if headers is None:
|
|
134
|
+
headers = {}
|
|
135
|
+
|
|
136
|
+
async with httpx.AsyncClient(**self.client_args) as client:
|
|
137
|
+
file_size, is_resumable, is_rejected, info_response = await self._get_info(client, method, url, headers)
|
|
138
|
+
|
|
139
|
+
if method.upper() == "GET" and not info_response:
|
|
140
|
+
raise FastGetError(f"Failed to retrieve file information from {url}")
|
|
141
|
+
|
|
142
|
+
use_parallel = True
|
|
143
|
+
if method.upper() != "GET":
|
|
144
|
+
use_parallel = False
|
|
145
|
+
await callback.on_slowdown("Parallel download are currently only supported with the GET method. Using single-threaded download.")
|
|
146
|
+
elif not is_resumable:
|
|
147
|
+
use_parallel = False
|
|
148
|
+
await callback.on_slowdown("Server does not support download range specification. Using single-threaded download.")
|
|
149
|
+
elif is_rejected:
|
|
150
|
+
use_parallel = False
|
|
151
|
+
await callback.on_slowdown("The server rejected Parallel FastGet download. Using single-threaded download.")
|
|
152
|
+
elif not file_size > 0:
|
|
153
|
+
use_parallel = False
|
|
154
|
+
await callback.on_slowdown("The file size reported by the server is invalid. Using single-threaded download.")
|
|
155
|
+
threads = self.max_threads if use_parallel else 1
|
|
156
|
+
|
|
157
|
+
http_version = info_response.http_version if info_response else "HTTP/1.1"
|
|
158
|
+
final_url = str(info_response.url) if info_response else url
|
|
159
|
+
|
|
160
|
+
await callback.on_start(
|
|
161
|
+
total_size=file_size,
|
|
162
|
+
threads=threads,
|
|
163
|
+
http_version=http_version,
|
|
164
|
+
final_url=final_url,
|
|
165
|
+
verify_was_enabled=bool(self.client_args["verify"])
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
if output:
|
|
169
|
+
out_dir = os.path.dirname(output)
|
|
170
|
+
if out_dir:
|
|
171
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
172
|
+
|
|
173
|
+
if use_parallel:
|
|
174
|
+
part_size = file_size // threads
|
|
175
|
+
tasks = []
|
|
176
|
+
part_files = []
|
|
177
|
+
|
|
178
|
+
for i in range(threads):
|
|
179
|
+
start = part_size * i
|
|
180
|
+
end = file_size - 1 if i == threads - 1 else start + part_size - 1
|
|
181
|
+
|
|
182
|
+
part_path = f"{output}.part{i}"
|
|
183
|
+
part_files.append(part_path)
|
|
184
|
+
|
|
185
|
+
tasks.append(self._download_worker(client, method, url, start, end, i, threads, part_path, callback, headers))
|
|
186
|
+
|
|
187
|
+
await asyncio.gather(*tasks)
|
|
188
|
+
|
|
189
|
+
await callback.on_merge_start(file_size)
|
|
190
|
+
with open(output, "wb") as outfile:
|
|
191
|
+
for part_file in part_files:
|
|
192
|
+
if os.path.exists(part_file):
|
|
193
|
+
with open(part_file, "rb") as infile:
|
|
194
|
+
while True:
|
|
195
|
+
chunk = infile.read(DEFAULT_CHUNK_SIZE)
|
|
196
|
+
if not chunk:
|
|
197
|
+
break
|
|
198
|
+
outfile.write(chunk)
|
|
199
|
+
await callback.on_merge_update(len(chunk))
|
|
200
|
+
os.remove(part_file)
|
|
201
|
+
|
|
202
|
+
await callback.on_merge_complete()
|
|
203
|
+
|
|
204
|
+
else:
|
|
205
|
+
headers["User-Agent"] = f'FastGet/{VERSION} (Downloading with Single thread; +https://github.com/DiamondGotCat/nercone-fastget/)'
|
|
206
|
+
async with client.stream(method=method, url=url, data=data, json=json, params=params, headers=headers) as response:
|
|
207
|
+
response.raise_for_status()
|
|
208
|
+
with open(output, "wb") as f:
|
|
209
|
+
async for chunk in response.aiter_bytes(chunk_size=DEFAULT_CHUNK_SIZE):
|
|
210
|
+
f.write(chunk)
|
|
211
|
+
await callback.on_update(0, len(chunk))
|
|
212
|
+
|
|
213
|
+
await callback.on_complete()
|
|
214
|
+
return output
|
|
215
|
+
|
|
216
|
+
else:
|
|
217
|
+
content_buffer = bytearray()
|
|
218
|
+
headers["User-Agent"] = f'FastGet/{VERSION} (Downloading with Single thread; +https://github.com/DiamondGotCat/nercone-fastget/)'
|
|
219
|
+
|
|
220
|
+
async with client.stream(method=method, url=url, data=data, json=json, params=params, headers=headers) as response:
|
|
221
|
+
response.raise_for_status()
|
|
222
|
+
async for chunk in response.aiter_bytes(chunk_size=DEFAULT_CHUNK_SIZE):
|
|
223
|
+
content_buffer.extend(chunk)
|
|
224
|
+
await callback.on_update(0, len(chunk))
|
|
225
|
+
|
|
226
|
+
await callback.on_complete()
|
|
227
|
+
content = bytes(content_buffer)
|
|
228
|
+
response._content = content # type: ignore[attr-defined]
|
|
229
|
+
return FastGetResponse(response, content)
|
|
230
|
+
|
|
231
|
+
def run_sync(coro: Awaitable[T]) -> T:
|
|
232
|
+
try:
|
|
233
|
+
loop = asyncio.get_event_loop()
|
|
234
|
+
except RuntimeError:
|
|
235
|
+
loop = asyncio.new_event_loop()
|
|
236
|
+
asyncio.set_event_loop(loop)
|
|
237
|
+
return loop.run_until_complete(coro)
|
|
238
|
+
|
|
239
|
+
def download(url: str, output: str, **kwargs) -> str | FastGetResponse:
|
|
240
|
+
session = FastGetSession(
|
|
241
|
+
max_threads=kwargs.pop("threads", DEFAULT_THREADS),
|
|
242
|
+
http1=not kwargs.pop("no_http1", False),
|
|
243
|
+
http2=not kwargs.pop("no_http2", False)
|
|
244
|
+
)
|
|
245
|
+
return run_sync(session.process("GET", url, output=output, **kwargs))
|
|
246
|
+
|
|
247
|
+
def request(method: str, url: str, **kwargs) -> str | FastGetResponse:
|
|
248
|
+
session = FastGetSession(
|
|
249
|
+
max_threads=kwargs.pop("threads", DEFAULT_THREADS),
|
|
250
|
+
http1=not kwargs.pop("no_http1", False),
|
|
251
|
+
http2=not kwargs.pop("no_http2", False)
|
|
252
|
+
)
|
|
253
|
+
return run_sync(session.process(method, url, output=None, **kwargs))
|
|
254
|
+
|
|
255
|
+
def get(url: str, **kwargs) -> str | FastGetResponse:
|
|
256
|
+
return request("GET", url, **kwargs)
|
|
257
|
+
|
|
258
|
+
def post(url: str, **kwargs) -> str | FastGetResponse:
|
|
259
|
+
return request("POST", url, **kwargs)
|