nercone-fastget 5.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nercone_fastget-5.3.4/PKG-INFO +131 -0
- nercone_fastget-5.3.4/README.md +115 -0
- nercone_fastget-5.3.4/pyproject.toml +27 -0
- nercone_fastget-5.3.4/src/nercone_fastget/__init__.py +0 -0
- nercone_fastget-5.3.4/src/nercone_fastget/__main__.py +189 -0
- nercone_fastget-5.3.4/src/nercone_fastget/fastget.py +267 -0
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: nercone-fastget
|
|
3
|
+
Version: 5.3.4
|
|
4
|
+
Summary: High-speed File Downloading Tool
|
|
5
|
+
Author: Nercone
|
|
6
|
+
Author-email: Nercone <nercone@diamondgotcat.net>
|
|
7
|
+
License: MIT
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Dist: nercone-modern
|
|
12
|
+
Requires-Dist: curl-cffi
|
|
13
|
+
Requires-Dist: uvloop ; sys_platform != 'win32'
|
|
14
|
+
Requires-Python: >=3.8
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
<img width="1920" alt="fastget" src="https://github.com/user-attachments/assets/fb4b6013-f65b-4c34-ae86-848011e7bfcf" />
|
|
19
|
+
|
|
20
|
+
# FastGet
|
|
21
|
+
High-speed File Downloading Tool
|
|
22
|
+
|
|
23
|
+
## How
|
|
24
|
+
It's using Multiple Thread Download.
|
|
25
|
+
|
|
26
|
+
This is need Range-select feature, in Server-side.
|
|
27
|
+
|
|
28
|
+
## Requiments
|
|
29
|
+
- CPython 3.9+
|
|
30
|
+
- `uv` [PyPI↗︎](https://pypi.org/project/uv/) or `pip3` [PyPI↗︎](https://pypi.org/project/pip/)
|
|
31
|
+
- `nercone-modern` [PyPI↗︎](https://pypi.org/project/nercone-modern/)
|
|
32
|
+
- `rich` [PyPI↗︎](https://pypi.org/project/rich/)
|
|
33
|
+
- `requests` [PyPI↗︎](https://pypi.org/project/requests/)
|
|
34
|
+
|
|
35
|
+
## Installation
|
|
36
|
+
|
|
37
|
+
### using uv (recommended)
|
|
38
|
+
```
|
|
39
|
+
uv tool install nercone-fastget
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### using pip3
|
|
43
|
+
|
|
44
|
+
**System Python:**
|
|
45
|
+
```
|
|
46
|
+
pip3 install nercone-fastget --break-system-packages
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
**Venv Python:**
|
|
50
|
+
```
|
|
51
|
+
pip3 install nercone-fastget
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Update
|
|
55
|
+
|
|
56
|
+
### using uv (recommended)
|
|
57
|
+
```
|
|
58
|
+
uv tool install nercone-fastget --upgrade
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### using pip3
|
|
62
|
+
|
|
63
|
+
**System Python:**
|
|
64
|
+
```
|
|
65
|
+
pip3 install nercone-fastget --upgrade --break-system-packages
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
**Venv Python:**
|
|
69
|
+
```
|
|
70
|
+
pip3 install nercone-fastget --upgrade
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Usage
|
|
74
|
+
|
|
75
|
+
### Show helps
|
|
76
|
+
```
|
|
77
|
+
fastget [-h] [--help]
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```
|
|
81
|
+
nercone@demo ~> fastget -h
|
|
82
|
+
usage: FastGet [-h] [-o OUTPUT] [-t THREADS] url
|
|
83
|
+
|
|
84
|
+
High-speed File Downloading Tool
|
|
85
|
+
|
|
86
|
+
positional arguments:
|
|
87
|
+
url
|
|
88
|
+
|
|
89
|
+
options:
|
|
90
|
+
-h, --help show this help message and exit
|
|
91
|
+
-o OUTPUT, --output OUTPUT
|
|
92
|
+
-t THREADS, --threads THREADS
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Download with default number of threads
|
|
96
|
+
```
|
|
97
|
+
fastget <url>
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
nercone@demo ~> fastget https://download.fedoraproject.org/pub/fedora/linux/releases/43/Workstation/x86_64/iso/Fedora-Workstation-Live-43-1.6.x86_64.iso
|
|
102
|
+
Total file size: 2.74GB (2,742,190,080 B)
|
|
103
|
+
(---------------------) DL All - 7% ( 1552/20922) | No Message
|
|
104
|
+
(---------------------) DL #1 - 7% ( 348/5231) | No Message
|
|
105
|
+
(---------------------) DL #2 - 12% ( 637/5231) | No Message
|
|
106
|
+
(---------------------) DL #3 - 6% ( 315/5231) | No Message
|
|
107
|
+
(---------------------) DL #4 - 5% ( 252/5231) | No Message
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
### Download with custom number of threads
|
|
111
|
+
```
|
|
112
|
+
fastget <url> [-t <number of threads>]
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
```
|
|
116
|
+
nercone@demo ~> fastget https://download.fedoraproject.org/pub/fedora/linux/releases/43/Workstation/x86_64/iso/Fedora-Workstation-Live-43-1.6.x86_64.iso -t 8
|
|
117
|
+
Total file size: 2.74GB (2,742,190,080 B)
|
|
118
|
+
(---------------------) DL All - 15% ( 3142/20922) | No Message
|
|
119
|
+
(---------------------) DL #1 - 19% ( 496/2616) | No Message
|
|
120
|
+
(---------------------) DL #2 - 12% ( 306/2616) | No Message
|
|
121
|
+
(---------------------) DL #3 - 21% ( 562/2616) | No Message
|
|
122
|
+
(---------------------) DL #4 - 14% ( 361/2616) | No Message
|
|
123
|
+
(---------------------) DL #5 - 20% ( 533/2616) | No Message
|
|
124
|
+
(---------------------) DL #6 - 9% ( 225/2616) | No Message
|
|
125
|
+
(---------------------) DL #7 - 14% ( 368/2616) | No Message
|
|
126
|
+
(---------------------) DL #8 - 11% ( 292/2616) | No Message
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+

|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
|
|
2
|
+
<img width="1920" alt="fastget" src="https://github.com/user-attachments/assets/fb4b6013-f65b-4c34-ae86-848011e7bfcf" />
|
|
3
|
+
|
|
4
|
+
# FastGet
|
|
5
|
+
High-speed File Downloading Tool
|
|
6
|
+
|
|
7
|
+
## How
|
|
8
|
+
It's using Multiple Thread Download.
|
|
9
|
+
|
|
10
|
+
This is need Range-select feature, in Server-side.
|
|
11
|
+
|
|
12
|
+
## Requiments
|
|
13
|
+
- CPython 3.9+
|
|
14
|
+
- `uv` [PyPI↗︎](https://pypi.org/project/uv/) or `pip3` [PyPI↗︎](https://pypi.org/project/pip/)
|
|
15
|
+
- `nercone-modern` [PyPI↗︎](https://pypi.org/project/nercone-modern/)
|
|
16
|
+
- `rich` [PyPI↗︎](https://pypi.org/project/rich/)
|
|
17
|
+
- `requests` [PyPI↗︎](https://pypi.org/project/requests/)
|
|
18
|
+
|
|
19
|
+
## Installation
|
|
20
|
+
|
|
21
|
+
### using uv (recommended)
|
|
22
|
+
```
|
|
23
|
+
uv tool install nercone-fastget
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
### using pip3
|
|
27
|
+
|
|
28
|
+
**System Python:**
|
|
29
|
+
```
|
|
30
|
+
pip3 install nercone-fastget --break-system-packages
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
**Venv Python:**
|
|
34
|
+
```
|
|
35
|
+
pip3 install nercone-fastget
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Update
|
|
39
|
+
|
|
40
|
+
### using uv (recommended)
|
|
41
|
+
```
|
|
42
|
+
uv tool install nercone-fastget --upgrade
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
### using pip3
|
|
46
|
+
|
|
47
|
+
**System Python:**
|
|
48
|
+
```
|
|
49
|
+
pip3 install nercone-fastget --upgrade --break-system-packages
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
**Venv Python:**
|
|
53
|
+
```
|
|
54
|
+
pip3 install nercone-fastget --upgrade
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Usage
|
|
58
|
+
|
|
59
|
+
### Show helps
|
|
60
|
+
```
|
|
61
|
+
fastget [-h] [--help]
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
nercone@demo ~> fastget -h
|
|
66
|
+
usage: FastGet [-h] [-o OUTPUT] [-t THREADS] url
|
|
67
|
+
|
|
68
|
+
High-speed File Downloading Tool
|
|
69
|
+
|
|
70
|
+
positional arguments:
|
|
71
|
+
url
|
|
72
|
+
|
|
73
|
+
options:
|
|
74
|
+
-h, --help show this help message and exit
|
|
75
|
+
-o OUTPUT, --output OUTPUT
|
|
76
|
+
-t THREADS, --threads THREADS
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Download with default number of threads
|
|
80
|
+
```
|
|
81
|
+
fastget <url>
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
nercone@demo ~> fastget https://download.fedoraproject.org/pub/fedora/linux/releases/43/Workstation/x86_64/iso/Fedora-Workstation-Live-43-1.6.x86_64.iso
|
|
86
|
+
Total file size: 2.74GB (2,742,190,080 B)
|
|
87
|
+
(---------------------) DL All - 7% ( 1552/20922) | No Message
|
|
88
|
+
(---------------------) DL #1 - 7% ( 348/5231) | No Message
|
|
89
|
+
(---------------------) DL #2 - 12% ( 637/5231) | No Message
|
|
90
|
+
(---------------------) DL #3 - 6% ( 315/5231) | No Message
|
|
91
|
+
(---------------------) DL #4 - 5% ( 252/5231) | No Message
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### Download with custom number of threads
|
|
95
|
+
```
|
|
96
|
+
fastget <url> [-t <number of threads>]
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
nercone@demo ~> fastget https://download.fedoraproject.org/pub/fedora/linux/releases/43/Workstation/x86_64/iso/Fedora-Workstation-Live-43-1.6.x86_64.iso -t 8
|
|
101
|
+
Total file size: 2.74GB (2,742,190,080 B)
|
|
102
|
+
(---------------------) DL All - 15% ( 3142/20922) | No Message
|
|
103
|
+
(---------------------) DL #1 - 19% ( 496/2616) | No Message
|
|
104
|
+
(---------------------) DL #2 - 12% ( 306/2616) | No Message
|
|
105
|
+
(---------------------) DL #3 - 21% ( 562/2616) | No Message
|
|
106
|
+
(---------------------) DL #4 - 14% ( 361/2616) | No Message
|
|
107
|
+
(---------------------) DL #5 - 20% ( 533/2616) | No Message
|
|
108
|
+
(---------------------) DL #6 - 9% ( 225/2616) | No Message
|
|
109
|
+
(---------------------) DL #7 - 14% ( 368/2616) | No Message
|
|
110
|
+
(---------------------) DL #8 - 11% ( 292/2616) | No Message
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
|
|
115
|
+

|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.9.5,<0.10.0"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "nercone-fastget"
|
|
7
|
+
version = "5.3.4"
|
|
8
|
+
description = "High-speed File Downloading Tool"
|
|
9
|
+
readme = { file = "README.md", content-type = "text/markdown" }
|
|
10
|
+
authors = [
|
|
11
|
+
{ name = "Nercone", email = "nercone@diamondgotcat.net" }
|
|
12
|
+
]
|
|
13
|
+
license = { text = "MIT" }
|
|
14
|
+
requires-python = ">=3.8"
|
|
15
|
+
dependencies = [
|
|
16
|
+
"nercone-modern",
|
|
17
|
+
"curl_cffi",
|
|
18
|
+
"uvloop; platform_system != 'Windows'"
|
|
19
|
+
]
|
|
20
|
+
classifiers = [
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"License :: OSI Approved :: MIT License",
|
|
23
|
+
"Operating System :: OS Independent"
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
fastget = "nercone_fastget.__main__:main"
|
|
File without changes
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import math
|
|
3
|
+
import argparse
|
|
4
|
+
import asyncio
|
|
5
|
+
import sys
|
|
6
|
+
from . import fastget
|
|
7
|
+
from urllib.parse import urlparse, unquote
|
|
8
|
+
from nercone_modern.logging import ModernLogging
|
|
9
|
+
from nercone_modern.progressbar import ModernProgressBar
|
|
10
|
+
|
|
11
|
+
class CLIProgress(fastget.ProgressCallback):
|
|
12
|
+
def __init__(self, logger: ModernLogging):
|
|
13
|
+
self.logger = logger
|
|
14
|
+
self.all_bar = None
|
|
15
|
+
self.thread_bars = []
|
|
16
|
+
self.chunk_size_display = 1024 * 128
|
|
17
|
+
self.merge_accumulated = 0
|
|
18
|
+
self.merge_bar = None
|
|
19
|
+
|
|
20
|
+
async def on_start(self, total_size: int, threads: int, http_version: str, final_url: str, verify_was_enabled: bool) -> None:
|
|
21
|
+
self.logger.log(f"File size: {total_size:,} bytes")
|
|
22
|
+
parsed_url = urlparse(final_url)
|
|
23
|
+
protocol = "HTTPS" if parsed_url.scheme.lower() == 'https' else "HTTP"
|
|
24
|
+
details = [http_version.upper()]
|
|
25
|
+
if protocol == "HTTPS":
|
|
26
|
+
details.append("TLS")
|
|
27
|
+
details.append("Verified" if verify_was_enabled else "Unverified")
|
|
28
|
+
connection_type = f"{protocol} ({', '.join(details)})"
|
|
29
|
+
self.logger.log(f"Connection Type: {connection_type}")
|
|
30
|
+
self.logger.log(f"Threads: {threads}")
|
|
31
|
+
if total_size > 0:
|
|
32
|
+
total_steps = max(1, math.ceil(total_size / self.chunk_size_display))
|
|
33
|
+
self.all_bar = ModernProgressBar(total=total_steps, process_name="Total", spinner_mode=False)
|
|
34
|
+
self.all_bar.start()
|
|
35
|
+
if threads > 1:
|
|
36
|
+
part_size = total_size // threads
|
|
37
|
+
for i in range(threads):
|
|
38
|
+
p_steps = max(1, math.ceil(part_size / self.chunk_size_display))
|
|
39
|
+
bar = ModernProgressBar(total=p_steps, process_name=f"DL #{i+1}", spinner_mode=False)
|
|
40
|
+
bar.start()
|
|
41
|
+
self.thread_bars.append(bar)
|
|
42
|
+
|
|
43
|
+
async def on_update(self, worker_id: int, loaded: int) -> None:
|
|
44
|
+
if self.thread_bars and worker_id < len(self.thread_bars):
|
|
45
|
+
self.thread_bars[worker_id].update()
|
|
46
|
+
|
|
47
|
+
if self.all_bar:
|
|
48
|
+
self.all_bar.update()
|
|
49
|
+
|
|
50
|
+
async def on_complete(self) -> None:
|
|
51
|
+
if self.all_bar:
|
|
52
|
+
self.all_bar.finish()
|
|
53
|
+
|
|
54
|
+
for b in self.thread_bars:
|
|
55
|
+
b.finish()
|
|
56
|
+
|
|
57
|
+
async def on_merge_start(self, total_size: int) -> None:
|
|
58
|
+
self.merge_accumulated = 0
|
|
59
|
+
if total_size > 0:
|
|
60
|
+
total_steps = max(1, math.ceil(total_size / self.chunk_size_display))
|
|
61
|
+
self.merge_bar = ModernProgressBar(total=total_steps, process_name="Merge", spinner_mode=False)
|
|
62
|
+
self.merge_bar.start()
|
|
63
|
+
|
|
64
|
+
async def on_merge_update(self, loaded: int) -> None:
|
|
65
|
+
if self.merge_bar:
|
|
66
|
+
self.merge_accumulated += loaded
|
|
67
|
+
while self.merge_accumulated >= self.chunk_size_display:
|
|
68
|
+
self.merge_bar.update()
|
|
69
|
+
self.merge_accumulated -= self.chunk_size_display
|
|
70
|
+
|
|
71
|
+
async def on_merge_complete(self) -> None:
|
|
72
|
+
if self.merge_bar:
|
|
73
|
+
self.merge_bar.finish()
|
|
74
|
+
|
|
75
|
+
async def on_error(self, msg: str) -> None:
|
|
76
|
+
if self.logger:
|
|
77
|
+
self.logger.log(msg, "ERROR")
|
|
78
|
+
|
|
79
|
+
async def async_main() -> None:
|
|
80
|
+
parser = argparse.ArgumentParser(prog='fastget', description='Modern High-Performance Downloader')
|
|
81
|
+
parser.add_argument('url', help="Target URL")
|
|
82
|
+
|
|
83
|
+
parser.add_argument('-o', '--output', help="File destination")
|
|
84
|
+
parser.add_argument('-X', '--method', default='GET', help="HTTP method (GET/POST)")
|
|
85
|
+
parser.add_argument('-d', '--data', help="Data for POST method")
|
|
86
|
+
parser.add_argument('-H', '--header', action='append', help="Custom Headers")
|
|
87
|
+
parser.add_argument('-t', '--threads', type=int, default=fastget.DEFAULT_THREADS, help="Number of threads to use for downloading")
|
|
88
|
+
parser.add_argument('-p', '--print', action='store_true', help="Output data directly to stdout without saving to a file")
|
|
89
|
+
|
|
90
|
+
strategy_group = parser.add_mutually_exclusive_group()
|
|
91
|
+
strategy_group.add_argument('-s', '--storage', '--low-memory', action='store_const', const='storage', dest='strategy', help="Avoid using memory as much as possible, and perform tasks such as saving destinations and merging on received data only on the storage device as much as possible. (default)")
|
|
92
|
+
strategy_group.add_argument('-m', '--memory', '--low-storage', action='store_const', const='memory', dest='strategy', help="Utilize memory efficiently to reduce maximum concurrent storage usage.")
|
|
93
|
+
parser.set_defaults(strategy='storage')
|
|
94
|
+
|
|
95
|
+
parser.add_argument('--no-verify', action='store_true', help="In the case of HTTPS, if a secure connection cannot be established, the system will continue to operate normally.")
|
|
96
|
+
parser.add_argument('--no-info', action='store_true', help="Suppresses all displays such as progress bars. If --print is used, only data is output to stdout.")
|
|
97
|
+
|
|
98
|
+
parser.add_argument('--no-http1', action='store_true', help="Do not use HTTP/1 or HTTP/1.1")
|
|
99
|
+
parser.add_argument('--no-http2', action='store_true', help="Do not use HTTP/2")
|
|
100
|
+
parser.add_argument('--no-http3', action='store_true', help="Do not use HTTP/3")
|
|
101
|
+
|
|
102
|
+
args = parser.parse_args()
|
|
103
|
+
|
|
104
|
+
if args.print and args.output:
|
|
105
|
+
parser.error("If -p/--print is specified, -o/--output cannot be specified.")
|
|
106
|
+
|
|
107
|
+
if args.no_info:
|
|
108
|
+
logger = None
|
|
109
|
+
callback = fastget.ProgressCallback()
|
|
110
|
+
else:
|
|
111
|
+
logger = ModernLogging("FastGet")
|
|
112
|
+
callback = CLIProgress(logger)
|
|
113
|
+
|
|
114
|
+
headers = {}
|
|
115
|
+
if args.header:
|
|
116
|
+
for h in args.header:
|
|
117
|
+
if ':' in h:
|
|
118
|
+
k, v = h.split(':', 1)
|
|
119
|
+
headers[k.strip()] = v.strip()
|
|
120
|
+
|
|
121
|
+
method = args.method.upper()
|
|
122
|
+
if args.data:
|
|
123
|
+
method = 'POST'
|
|
124
|
+
|
|
125
|
+
session = fastget.FastGetSession(
|
|
126
|
+
max_threads=args.threads,
|
|
127
|
+
http1=not args.no_http1,
|
|
128
|
+
http2=not args.no_http2,
|
|
129
|
+
http3=not args.no_http3,
|
|
130
|
+
verify=not args.no_verify
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
start_time = 0
|
|
134
|
+
if logger:
|
|
135
|
+
start_time = asyncio.get_running_loop().time()
|
|
136
|
+
|
|
137
|
+
try:
|
|
138
|
+
if args.print:
|
|
139
|
+
result = await session.process(
|
|
140
|
+
method=method,
|
|
141
|
+
url=args.url,
|
|
142
|
+
data=args.data,
|
|
143
|
+
headers=headers,
|
|
144
|
+
callback=callback
|
|
145
|
+
)
|
|
146
|
+
sys.stdout.buffer.write(result.content)
|
|
147
|
+
else:
|
|
148
|
+
output = args.output
|
|
149
|
+
if not output:
|
|
150
|
+
parsed = urlparse(args.url)
|
|
151
|
+
output = unquote(os.path.basename(parsed.path)) or "downloaded_file"
|
|
152
|
+
|
|
153
|
+
path = await session.process(
|
|
154
|
+
method=method,
|
|
155
|
+
url=args.url,
|
|
156
|
+
output=output,
|
|
157
|
+
data=args.data,
|
|
158
|
+
headers=headers,
|
|
159
|
+
callback=callback,
|
|
160
|
+
strategy=args.strategy
|
|
161
|
+
)
|
|
162
|
+
if logger:
|
|
163
|
+
end_time = asyncio.get_running_loop().time()
|
|
164
|
+
duration_ms = (end_time - start_time) * 1000
|
|
165
|
+
logger.log(f"Completed in {duration_ms:.2f}ms")
|
|
166
|
+
logger.log(f"Saved to: {path}")
|
|
167
|
+
|
|
168
|
+
except fastget.FastGetError as e:
|
|
169
|
+
if logger:
|
|
170
|
+
logger.log(str(e), "CRITICAL")
|
|
171
|
+
else:
|
|
172
|
+
print(f"Error: {e}", file=sys.stderr, flush=True)
|
|
173
|
+
except Exception as e:
|
|
174
|
+
if logger:
|
|
175
|
+
logger.log(f"Unexpected error: {e}", "CRITICAL")
|
|
176
|
+
else:
|
|
177
|
+
print(f"Unexpected error: {e}", file=sys.stderr, flush=True)
|
|
178
|
+
|
|
179
|
+
def main() -> None:
|
|
180
|
+
try:
|
|
181
|
+
import uvloop
|
|
182
|
+
uvloop.install()
|
|
183
|
+
except ImportError:
|
|
184
|
+
pass
|
|
185
|
+
|
|
186
|
+
asyncio.run(async_main())
|
|
187
|
+
|
|
188
|
+
if __name__ == "__main__":
|
|
189
|
+
main()
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import asyncio
|
|
3
|
+
from curl_cffi import aio
|
|
4
|
+
from curl_cffi.errors import CurlError
|
|
5
|
+
from curl_cffi.const import HTTPVersion
|
|
6
|
+
from importlib.metadata import version
|
|
7
|
+
from typing import Union, Optional, Dict, Any, TypeVar, Awaitable, List
|
|
8
|
+
|
|
9
|
+
try:
|
|
10
|
+
VERSION = version("nercone-fastget")
|
|
11
|
+
except Exception:
|
|
12
|
+
VERSION = "0.0.0"
|
|
13
|
+
|
|
14
|
+
DEFAULT_CHUNK_SIZE = 1024 * 64
|
|
15
|
+
DEFAULT_TIMEOUT = 30.0
|
|
16
|
+
DEFAULT_RETRIES = 3
|
|
17
|
+
DEFAULT_THREADS = 8
|
|
18
|
+
|
|
19
|
+
T = TypeVar("T")
|
|
20
|
+
|
|
21
|
+
class FastGetError(Exception):
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
class ProgressCallback:
|
|
25
|
+
async def on_start(self, total_size: int, threads: int, http_version: str, final_url: str, verify_was_enabled: bool) -> None: pass
|
|
26
|
+
async def on_update(self, worker_id: int, loaded: int) -> None: pass
|
|
27
|
+
async def on_complete(self) -> None: pass
|
|
28
|
+
async def on_merge_start(self, total_size: int) -> None: pass
|
|
29
|
+
async def on_merge_update(self, loaded: int) -> None: pass
|
|
30
|
+
async def on_merge_complete(self) -> None: pass
|
|
31
|
+
async def on_error(self, msg: str) -> None: pass
|
|
32
|
+
|
|
33
|
+
class FastGetResponse:
|
|
34
|
+
def __init__(self, original: aio.AsyncClient.Response, content: bytes):
|
|
35
|
+
self._r = original
|
|
36
|
+
self.content = content
|
|
37
|
+
self.url = str(original.url)
|
|
38
|
+
self.status_code = original.status_code
|
|
39
|
+
self.headers = original.headers
|
|
40
|
+
self.http_version = original.http_version
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def text(self) -> str:
|
|
44
|
+
return self.content.decode(self._r.encoding or 'utf-8', errors='replace')
|
|
45
|
+
|
|
46
|
+
def json(self, **kwargs) -> Any:
|
|
47
|
+
return self._r.json(**kwargs)
|
|
48
|
+
|
|
49
|
+
class FastGetSession:
|
|
50
|
+
def __init__(self, max_threads: int = DEFAULT_THREADS, http1: bool = True, http2: bool = True, http3: bool = True, verify: bool = True, follow_redirects: bool = True):
|
|
51
|
+
self.max_threads = max_threads
|
|
52
|
+
|
|
53
|
+
self.http_versions_to_try = []
|
|
54
|
+
if http3:
|
|
55
|
+
self.http_versions_to_try.append(HTTPVersion.V3)
|
|
56
|
+
if http2:
|
|
57
|
+
self.http_versions_to_try.append(HTTPVersion.V2)
|
|
58
|
+
if http1:
|
|
59
|
+
self.http_versions_to_try.append(HTTPVersion.V1_1)
|
|
60
|
+
|
|
61
|
+
if not self.http_versions_to_try:
|
|
62
|
+
raise ValueError("At least one of HTTP/1, HTTP/2, or HTTP/3 must be enabled.")
|
|
63
|
+
|
|
64
|
+
self.client_args = {
|
|
65
|
+
"verify": verify,
|
|
66
|
+
"follow_redirects": follow_redirects,
|
|
67
|
+
"timeout": DEFAULT_TIMEOUT,
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async def _get_info(self, client: aio.AsyncClient, method: str, url: str, **kwargs) -> tuple[int, bool, bool, Optional[aio.AsyncClient.Response]]:
|
|
71
|
+
headers = kwargs.get("headers", {}).copy()
|
|
72
|
+
headers["User-Agent"] = f'FastGet/{VERSION} (Info)'
|
|
73
|
+
if method.upper() != "GET":
|
|
74
|
+
return 0, False, False, None
|
|
75
|
+
|
|
76
|
+
try:
|
|
77
|
+
head_resp = await client.head(url, headers=headers)
|
|
78
|
+
head_resp.raise_for_status()
|
|
79
|
+
size = int(head_resp.headers.get("content-length", 0))
|
|
80
|
+
accept_ranges = head_resp.headers.get("accept-ranges", "").lower() == "bytes"
|
|
81
|
+
reject_fg = head_resp.headers.get("rejectfastget", "").lower() in ["true", "1", "yes"]
|
|
82
|
+
return size, accept_ranges, reject_fg, head_resp
|
|
83
|
+
|
|
84
|
+
except CurlError:
|
|
85
|
+
try:
|
|
86
|
+
get_headers = headers.copy()
|
|
87
|
+
get_headers["Range"] = "bytes=0-0"
|
|
88
|
+
get_resp = await client.get(url, headers=get_headers)
|
|
89
|
+
get_resp.raise_for_status()
|
|
90
|
+
size_str = "0"
|
|
91
|
+
if get_resp.status_code == 206 and "content-range" in get_resp.headers:
|
|
92
|
+
content_range = get_resp.headers["content-range"]
|
|
93
|
+
if '/' in content_range:
|
|
94
|
+
size_str = content_range.split('/')[-1]
|
|
95
|
+
else:
|
|
96
|
+
size_str = get_resp.headers.get("content-length", "0")
|
|
97
|
+
|
|
98
|
+
size = int(size_str) if size_str.isdigit() else 0
|
|
99
|
+
accept_ranges = get_resp.headers.get("accept-ranges", "").lower() == "bytes" or get_resp.status_code == 206
|
|
100
|
+
reject_fg = get_resp.headers.get("rejectfastget", "").lower() in ["true", "1", "yes"]
|
|
101
|
+
return size, accept_ranges, reject_fg, get_resp
|
|
102
|
+
except CurlError:
|
|
103
|
+
raise
|
|
104
|
+
|
|
105
|
+
async def _download_worker_to_storage(self, client: aio.AsyncClient, method: str, url: str, start: int, end: int, worker_id: int, total_threads: int, part_path: str, callback: ProgressCallback, **kwargs) -> None:
|
|
106
|
+
headers = kwargs.get("headers", {}).copy()
|
|
107
|
+
headers["Range"] = f"bytes={start}-{end}"
|
|
108
|
+
headers["User-Agent"] = f'FastGet/{VERSION} (Worker {worker_id+1}/{total_threads})'
|
|
109
|
+
kwargs["headers"] = headers
|
|
110
|
+
for attempt in range(DEFAULT_RETRIES):
|
|
111
|
+
try:
|
|
112
|
+
response = await client.request(method, url, **kwargs)
|
|
113
|
+
response.raise_for_status()
|
|
114
|
+
with open(part_path, "wb") as f:
|
|
115
|
+
async for chunk in response.aiter_content(chunk_size=DEFAULT_CHUNK_SIZE):
|
|
116
|
+
f.write(chunk)
|
|
117
|
+
await callback.on_update(worker_id, len(chunk))
|
|
118
|
+
return
|
|
119
|
+
except CurlError as e:
|
|
120
|
+
if attempt == DEFAULT_RETRIES - 1:
|
|
121
|
+
await callback.on_error(f"Worker {worker_id+1} failed: {e}")
|
|
122
|
+
raise
|
|
123
|
+
await asyncio.sleep(1)
|
|
124
|
+
|
|
125
|
+
async def _download_worker_to_memory(self, client: aio.AsyncClient, method: str, url: str, start: int, end: int, worker_id: int, total_threads: int, callback: ProgressCallback, **kwargs) -> bytes:
|
|
126
|
+
headers = kwargs.get("headers", {}).copy()
|
|
127
|
+
headers["Range"] = f"bytes={start}-{end}"
|
|
128
|
+
headers["User-Agent"] = f'FastGet/{VERSION} (Worker {worker_id+1}/{total_threads})'
|
|
129
|
+
kwargs["headers"] = headers
|
|
130
|
+
part_buffer = bytearray()
|
|
131
|
+
for attempt in range(DEFAULT_RETRIES):
|
|
132
|
+
try:
|
|
133
|
+
response = await client.request(method, url, **kwargs)
|
|
134
|
+
response.raise_for_status()
|
|
135
|
+
async for chunk in response.aiter_content(chunk_size=DEFAULT_CHUNK_SIZE):
|
|
136
|
+
part_buffer.extend(chunk)
|
|
137
|
+
await callback.on_update(worker_id, len(chunk))
|
|
138
|
+
return bytes(part_buffer)
|
|
139
|
+
except CurlError as e:
|
|
140
|
+
if attempt == DEFAULT_RETRIES - 1:
|
|
141
|
+
await callback.on_error(f"Worker {worker_id+1} failed: {e}")
|
|
142
|
+
raise
|
|
143
|
+
part_buffer.clear()
|
|
144
|
+
await asyncio.sleep(1)
|
|
145
|
+
return b""
|
|
146
|
+
|
|
147
|
+
async def process(self, method: str, url: str, output: Optional[str] = None, data: Any = None, json: Any = None, params: Any = None, headers: Dict = None, callback: Optional[ProgressCallback] = None, strategy: str = 'storage') -> Union[str, FastGetResponse]:
|
|
148
|
+
callback = callback or ProgressCallback()
|
|
149
|
+
headers = headers or {}
|
|
150
|
+
req_kwargs = {"data": data, "json": json, "params": params, "headers": headers}
|
|
151
|
+
|
|
152
|
+
client = None
|
|
153
|
+
last_error = None
|
|
154
|
+
info_result = None
|
|
155
|
+
|
|
156
|
+
for version in self.http_versions_to_try:
|
|
157
|
+
client_args = self.client_args.copy()
|
|
158
|
+
client_args["http_version"] = version
|
|
159
|
+
temp_client = aio.AsyncClient(**client_args)
|
|
160
|
+
try:
|
|
161
|
+
info_result = await self._get_info(temp_client, method, url, **req_kwargs)
|
|
162
|
+
client = temp_client
|
|
163
|
+
break
|
|
164
|
+
except CurlError as e:
|
|
165
|
+
last_error = e
|
|
166
|
+
await temp_client.close()
|
|
167
|
+
is_last_attempt = (version == self.http_versions_to_try[-1])
|
|
168
|
+
if not is_last_attempt:
|
|
169
|
+
await callback.on_error(f"Request with {version.name} failed. Trying fallback...")
|
|
170
|
+
|
|
171
|
+
if not client:
|
|
172
|
+
await callback.on_error(f"All HTTP versions failed. Last error: {last_error}")
|
|
173
|
+
raise FastGetError(f"Failed to connect to {url} with all tried HTTP versions.") from last_error
|
|
174
|
+
|
|
175
|
+
try:
|
|
176
|
+
file_size, is_resumable, is_rejected, info_response = info_result
|
|
177
|
+
|
|
178
|
+
if method.upper() == "GET" and not info_response:
|
|
179
|
+
raise FastGetError(f"Failed to retrieve file information from {url}")
|
|
180
|
+
|
|
181
|
+
use_parallel = (method.upper() == "GET" and is_resumable and not is_rejected and file_size > 0 and output is not None and self.max_threads > 1)
|
|
182
|
+
threads = self.max_threads if use_parallel else 1
|
|
183
|
+
http_version = info_response.http_version if info_response else "UNKNOWN"
|
|
184
|
+
final_url = str(info_response.url) if info_response else url
|
|
185
|
+
verify_was_enabled = self.client_args.get("verify", True)
|
|
186
|
+
|
|
187
|
+
await callback.on_start(file_size, threads, http_version, final_url, verify_was_enabled)
|
|
188
|
+
if output:
|
|
189
|
+
out_dir = os.path.dirname(output)
|
|
190
|
+
if out_dir: os.makedirs(out_dir, exist_ok=True)
|
|
191
|
+
if use_parallel:
|
|
192
|
+
part_size = file_size // threads
|
|
193
|
+
tasks = []
|
|
194
|
+
if strategy == 'memory':
|
|
195
|
+
for i in range(threads):
|
|
196
|
+
start = i * part_size
|
|
197
|
+
end = file_size - 1 if i == threads - 1 else start + part_size - 1
|
|
198
|
+
tasks.append(self._download_worker_to_memory(client, method, url, start, end, i, threads, callback, **req_kwargs))
|
|
199
|
+
parts_in_memory: List[bytes] = await asyncio.gather(*tasks)
|
|
200
|
+
await callback.on_merge_start(file_size)
|
|
201
|
+
with open(output, "wb") as outfile:
|
|
202
|
+
for part_data in parts_in_memory:
|
|
203
|
+
outfile.write(part_data)
|
|
204
|
+
await callback.on_merge_update(len(part_data))
|
|
205
|
+
await callback.on_merge_complete()
|
|
206
|
+
else:
|
|
207
|
+
part_files = []
|
|
208
|
+
for i in range(threads):
|
|
209
|
+
start = i * part_size
|
|
210
|
+
end = file_size - 1 if i == threads - 1 else start + part_size - 1
|
|
211
|
+
part_path = f"{output}.part{i}"
|
|
212
|
+
part_files.append(part_path)
|
|
213
|
+
tasks.append(self._download_worker_to_storage(client, method, url, start, end, i, threads, part_path, callback, **req_kwargs))
|
|
214
|
+
await asyncio.gather(*tasks)
|
|
215
|
+
await callback.on_merge_start(file_size)
|
|
216
|
+
with open(output, "wb") as outfile:
|
|
217
|
+
for part_file in part_files:
|
|
218
|
+
try:
|
|
219
|
+
with open(part_file, "rb") as infile:
|
|
220
|
+
while chunk := infile.read(DEFAULT_CHUNK_SIZE):
|
|
221
|
+
outfile.write(chunk)
|
|
222
|
+
await callback.on_merge_update(len(chunk))
|
|
223
|
+
os.remove(part_file)
|
|
224
|
+
except FileNotFoundError:
|
|
225
|
+
await callback.on_error(f"Part file {part_file} not found during merge.")
|
|
226
|
+
await callback.on_merge_complete()
|
|
227
|
+
else:
|
|
228
|
+
headers["User-Agent"] = f'FastGet/{VERSION} (Single-Thread)'
|
|
229
|
+
response = await client.request(method, url, **req_kwargs)
|
|
230
|
+
response.raise_for_status()
|
|
231
|
+
with open(output, "wb") as f:
|
|
232
|
+
async for chunk in response.aiter_content(chunk_size=DEFAULT_CHUNK_SIZE):
|
|
233
|
+
f.write(chunk)
|
|
234
|
+
await callback.on_update(0, len(chunk))
|
|
235
|
+
await callback.on_complete()
|
|
236
|
+
return output
|
|
237
|
+
else:
|
|
238
|
+
headers["User-Agent"] = f'FastGet/{VERSION} (Single-Thread/Memory)'
|
|
239
|
+
response = await client.request(method, url, **req_kwargs)
|
|
240
|
+
response.raise_for_status()
|
|
241
|
+
await callback.on_update(0, len(response.content))
|
|
242
|
+
await callback.on_complete()
|
|
243
|
+
return FastGetResponse(response, response.content)
|
|
244
|
+
finally:
|
|
245
|
+
if client:
|
|
246
|
+
await client.close()
|
|
247
|
+
|
|
248
|
+
def run_sync(coro: Awaitable[T]) -> T:
|
|
249
|
+
try: loop = asyncio.get_running_loop()
|
|
250
|
+
except RuntimeError:
|
|
251
|
+
loop = asyncio.new_event_loop()
|
|
252
|
+
asyncio.set_event_loop(loop)
|
|
253
|
+
return loop.run_until_complete(coro)
|
|
254
|
+
|
|
255
|
+
def download(url: str, output: str, **kwargs) -> str:
|
|
256
|
+
session = FastGetSession(max_threads=kwargs.pop("threads", DEFAULT_THREADS), http1=not kwargs.pop("no_http1", False), http2=not kwargs.pop("no_http2", False), http3=not kwargs.pop("no_http3", False))
|
|
257
|
+
return run_sync(session.process("GET", url, output=output, **kwargs))
|
|
258
|
+
|
|
259
|
+
def request(method: str, url: str, **kwargs) -> FastGetResponse:
|
|
260
|
+
session = FastGetSession(max_threads=kwargs.pop("threads", DEFAULT_THREADS), http1=not kwargs.pop("no_http1", False), http2=not kwargs.pop("no_http2", False), http3=not kwargs.pop("no_http3", False))
|
|
261
|
+
return run_sync(session.process(method, url, output=None, **kwargs))
|
|
262
|
+
|
|
263
|
+
def get(url: str, **kwargs) -> FastGetResponse:
|
|
264
|
+
return request("GET", url, **kwargs)
|
|
265
|
+
|
|
266
|
+
def post(url: str, **kwargs) -> FastGetResponse:
|
|
267
|
+
return request("POST", url, **kwargs)
|