nercone-fastget 5.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,131 @@
1
+ Metadata-Version: 2.3
2
+ Name: nercone-fastget
3
+ Version: 5.3.4
4
+ Summary: High-speed File Downloading Tool
5
+ Author: Nercone
6
+ Author-email: Nercone <nercone@diamondgotcat.net>
7
+ License: MIT
8
+ Classifier: Programming Language :: Python :: 3
9
+ Classifier: License :: OSI Approved :: MIT License
10
+ Classifier: Operating System :: OS Independent
11
+ Requires-Dist: nercone-modern
12
+ Requires-Dist: curl-cffi
13
+ Requires-Dist: uvloop ; sys_platform != 'win32'
14
+ Requires-Python: >=3.8
15
+ Description-Content-Type: text/markdown
16
+
17
+
18
+ <img width="1920" alt="fastget" src="https://github.com/user-attachments/assets/fb4b6013-f65b-4c34-ae86-848011e7bfcf" />
19
+
20
+ # FastGet
21
+ High-speed File Downloading Tool
22
+
23
+ ## How
24
+ It's using Multiple Thread Download.
25
+
26
+ This is need Range-select feature, in Server-side.
27
+
28
+ ## Requiments
29
+ - CPython 3.9+
30
+ - `uv` [PyPI↗︎](https://pypi.org/project/uv/) or `pip3` [PyPI↗︎](https://pypi.org/project/pip/)
31
+ - `nercone-modern` [PyPI↗︎](https://pypi.org/project/nercone-modern/)
32
+ - `rich` [PyPI↗︎](https://pypi.org/project/rich/)
33
+ - `requests` [PyPI↗︎](https://pypi.org/project/requests/)
34
+
35
+ ## Installation
36
+
37
+ ### using uv (recommended)
38
+ ```
39
+ uv tool install nercone-fastget
40
+ ```
41
+
42
+ ### using pip3
43
+
44
+ **System Python:**
45
+ ```
46
+ pip3 install nercone-fastget --break-system-packages
47
+ ```
48
+
49
+ **Venv Python:**
50
+ ```
51
+ pip3 install nercone-fastget
52
+ ```
53
+
54
+ ## Update
55
+
56
+ ### using uv (recommended)
57
+ ```
58
+ uv tool install nercone-fastget --upgrade
59
+ ```
60
+
61
+ ### using pip3
62
+
63
+ **System Python:**
64
+ ```
65
+ pip3 install nercone-fastget --upgrade --break-system-packages
66
+ ```
67
+
68
+ **Venv Python:**
69
+ ```
70
+ pip3 install nercone-fastget --upgrade
71
+ ```
72
+
73
+ ## Usage
74
+
75
+ ### Show helps
76
+ ```
77
+ fastget [-h] [--help]
78
+ ```
79
+
80
+ ```
81
+ nercone@demo ~> fastget -h
82
+ usage: FastGet [-h] [-o OUTPUT] [-t THREADS] url
83
+
84
+ High-speed File Downloading Tool
85
+
86
+ positional arguments:
87
+ url
88
+
89
+ options:
90
+ -h, --help show this help message and exit
91
+ -o OUTPUT, --output OUTPUT
92
+ -t THREADS, --threads THREADS
93
+ ```
94
+
95
+ ### Download with default number of threads
96
+ ```
97
+ fastget <url>
98
+ ```
99
+
100
+ ```
101
+ nercone@demo ~> fastget https://download.fedoraproject.org/pub/fedora/linux/releases/43/Workstation/x86_64/iso/Fedora-Workstation-Live-43-1.6.x86_64.iso
102
+ Total file size: 2.74GB (2,742,190,080 B)
103
+ (---------------------) DL All - 7% ( 1552/20922) | No Message
104
+ (---------------------) DL #1 - 7% ( 348/5231) | No Message
105
+ (---------------------) DL #2 - 12% ( 637/5231) | No Message
106
+ (---------------------) DL #3 - 6% ( 315/5231) | No Message
107
+ (---------------------) DL #4 - 5% ( 252/5231) | No Message
108
+ ```
109
+
110
+ ### Download with custom number of threads
111
+ ```
112
+ fastget <url> [-t <number of threads>]
113
+ ```
114
+
115
+ ```
116
+ nercone@demo ~> fastget https://download.fedoraproject.org/pub/fedora/linux/releases/43/Workstation/x86_64/iso/Fedora-Workstation-Live-43-1.6.x86_64.iso -t 8
117
+ Total file size: 2.74GB (2,742,190,080 B)
118
+ (---------------------) DL All - 15% ( 3142/20922) | No Message
119
+ (---------------------) DL #1 - 19% ( 496/2616) | No Message
120
+ (---------------------) DL #2 - 12% ( 306/2616) | No Message
121
+ (---------------------) DL #3 - 21% ( 562/2616) | No Message
122
+ (---------------------) DL #4 - 14% ( 361/2616) | No Message
123
+ (---------------------) DL #5 - 20% ( 533/2616) | No Message
124
+ (---------------------) DL #6 - 9% ( 225/2616) | No Message
125
+ (---------------------) DL #7 - 14% ( 368/2616) | No Message
126
+ (---------------------) DL #8 - 11% ( 292/2616) | No Message
127
+ ```
128
+
129
+ ---
130
+
131
+ ![PyPI - Version](https://img.shields.io/pypi/v/nercone-fastget)
@@ -0,0 +1,115 @@
1
+
2
+ <img width="1920" alt="fastget" src="https://github.com/user-attachments/assets/fb4b6013-f65b-4c34-ae86-848011e7bfcf" />
3
+
4
+ # FastGet
5
+ High-speed File Downloading Tool
6
+
7
+ ## How
8
+ It's using Multiple Thread Download.
9
+
10
+ This is need Range-select feature, in Server-side.
11
+
12
+ ## Requiments
13
+ - CPython 3.9+
14
+ - `uv` [PyPI↗︎](https://pypi.org/project/uv/) or `pip3` [PyPI↗︎](https://pypi.org/project/pip/)
15
+ - `nercone-modern` [PyPI↗︎](https://pypi.org/project/nercone-modern/)
16
+ - `rich` [PyPI↗︎](https://pypi.org/project/rich/)
17
+ - `requests` [PyPI↗︎](https://pypi.org/project/requests/)
18
+
19
+ ## Installation
20
+
21
+ ### using uv (recommended)
22
+ ```
23
+ uv tool install nercone-fastget
24
+ ```
25
+
26
+ ### using pip3
27
+
28
+ **System Python:**
29
+ ```
30
+ pip3 install nercone-fastget --break-system-packages
31
+ ```
32
+
33
+ **Venv Python:**
34
+ ```
35
+ pip3 install nercone-fastget
36
+ ```
37
+
38
+ ## Update
39
+
40
+ ### using uv (recommended)
41
+ ```
42
+ uv tool install nercone-fastget --upgrade
43
+ ```
44
+
45
+ ### using pip3
46
+
47
+ **System Python:**
48
+ ```
49
+ pip3 install nercone-fastget --upgrade --break-system-packages
50
+ ```
51
+
52
+ **Venv Python:**
53
+ ```
54
+ pip3 install nercone-fastget --upgrade
55
+ ```
56
+
57
+ ## Usage
58
+
59
+ ### Show helps
60
+ ```
61
+ fastget [-h] [--help]
62
+ ```
63
+
64
+ ```
65
+ nercone@demo ~> fastget -h
66
+ usage: FastGet [-h] [-o OUTPUT] [-t THREADS] url
67
+
68
+ High-speed File Downloading Tool
69
+
70
+ positional arguments:
71
+ url
72
+
73
+ options:
74
+ -h, --help show this help message and exit
75
+ -o OUTPUT, --output OUTPUT
76
+ -t THREADS, --threads THREADS
77
+ ```
78
+
79
+ ### Download with default number of threads
80
+ ```
81
+ fastget <url>
82
+ ```
83
+
84
+ ```
85
+ nercone@demo ~> fastget https://download.fedoraproject.org/pub/fedora/linux/releases/43/Workstation/x86_64/iso/Fedora-Workstation-Live-43-1.6.x86_64.iso
86
+ Total file size: 2.74GB (2,742,190,080 B)
87
+ (---------------------) DL All - 7% ( 1552/20922) | No Message
88
+ (---------------------) DL #1 - 7% ( 348/5231) | No Message
89
+ (---------------------) DL #2 - 12% ( 637/5231) | No Message
90
+ (---------------------) DL #3 - 6% ( 315/5231) | No Message
91
+ (---------------------) DL #4 - 5% ( 252/5231) | No Message
92
+ ```
93
+
94
+ ### Download with custom number of threads
95
+ ```
96
+ fastget <url> [-t <number of threads>]
97
+ ```
98
+
99
+ ```
100
+ nercone@demo ~> fastget https://download.fedoraproject.org/pub/fedora/linux/releases/43/Workstation/x86_64/iso/Fedora-Workstation-Live-43-1.6.x86_64.iso -t 8
101
+ Total file size: 2.74GB (2,742,190,080 B)
102
+ (---------------------) DL All - 15% ( 3142/20922) | No Message
103
+ (---------------------) DL #1 - 19% ( 496/2616) | No Message
104
+ (---------------------) DL #2 - 12% ( 306/2616) | No Message
105
+ (---------------------) DL #3 - 21% ( 562/2616) | No Message
106
+ (---------------------) DL #4 - 14% ( 361/2616) | No Message
107
+ (---------------------) DL #5 - 20% ( 533/2616) | No Message
108
+ (---------------------) DL #6 - 9% ( 225/2616) | No Message
109
+ (---------------------) DL #7 - 14% ( 368/2616) | No Message
110
+ (---------------------) DL #8 - 11% ( 292/2616) | No Message
111
+ ```
112
+
113
+ ---
114
+
115
+ ![PyPI - Version](https://img.shields.io/pypi/v/nercone-fastget)
@@ -0,0 +1,27 @@
1
+ [build-system]
2
+ requires = ["uv_build>=0.9.5,<0.10.0"]
3
+ build-backend = "uv_build"
4
+
5
+ [project]
6
+ name = "nercone-fastget"
7
+ version = "5.3.4"
8
+ description = "High-speed File Downloading Tool"
9
+ readme = { file = "README.md", content-type = "text/markdown" }
10
+ authors = [
11
+ { name = "Nercone", email = "nercone@diamondgotcat.net" }
12
+ ]
13
+ license = { text = "MIT" }
14
+ requires-python = ">=3.8"
15
+ dependencies = [
16
+ "nercone-modern",
17
+ "curl_cffi",
18
+ "uvloop; platform_system != 'Windows'"
19
+ ]
20
+ classifiers = [
21
+ "Programming Language :: Python :: 3",
22
+ "License :: OSI Approved :: MIT License",
23
+ "Operating System :: OS Independent"
24
+ ]
25
+
26
+ [project.scripts]
27
+ fastget = "nercone_fastget.__main__:main"
File without changes
@@ -0,0 +1,189 @@
1
+ import os
2
+ import math
3
+ import argparse
4
+ import asyncio
5
+ import sys
6
+ from . import fastget
7
+ from urllib.parse import urlparse, unquote
8
+ from nercone_modern.logging import ModernLogging
9
+ from nercone_modern.progressbar import ModernProgressBar
10
+
11
+ class CLIProgress(fastget.ProgressCallback):
12
+ def __init__(self, logger: ModernLogging):
13
+ self.logger = logger
14
+ self.all_bar = None
15
+ self.thread_bars = []
16
+ self.chunk_size_display = 1024 * 128
17
+ self.merge_accumulated = 0
18
+ self.merge_bar = None
19
+
20
+ async def on_start(self, total_size: int, threads: int, http_version: str, final_url: str, verify_was_enabled: bool) -> None:
21
+ self.logger.log(f"File size: {total_size:,} bytes")
22
+ parsed_url = urlparse(final_url)
23
+ protocol = "HTTPS" if parsed_url.scheme.lower() == 'https' else "HTTP"
24
+ details = [http_version.upper()]
25
+ if protocol == "HTTPS":
26
+ details.append("TLS")
27
+ details.append("Verified" if verify_was_enabled else "Unverified")
28
+ connection_type = f"{protocol} ({', '.join(details)})"
29
+ self.logger.log(f"Connection Type: {connection_type}")
30
+ self.logger.log(f"Threads: {threads}")
31
+ if total_size > 0:
32
+ total_steps = max(1, math.ceil(total_size / self.chunk_size_display))
33
+ self.all_bar = ModernProgressBar(total=total_steps, process_name="Total", spinner_mode=False)
34
+ self.all_bar.start()
35
+ if threads > 1:
36
+ part_size = total_size // threads
37
+ for i in range(threads):
38
+ p_steps = max(1, math.ceil(part_size / self.chunk_size_display))
39
+ bar = ModernProgressBar(total=p_steps, process_name=f"DL #{i+1}", spinner_mode=False)
40
+ bar.start()
41
+ self.thread_bars.append(bar)
42
+
43
+ async def on_update(self, worker_id: int, loaded: int) -> None:
44
+ if self.thread_bars and worker_id < len(self.thread_bars):
45
+ self.thread_bars[worker_id].update()
46
+
47
+ if self.all_bar:
48
+ self.all_bar.update()
49
+
50
+ async def on_complete(self) -> None:
51
+ if self.all_bar:
52
+ self.all_bar.finish()
53
+
54
+ for b in self.thread_bars:
55
+ b.finish()
56
+
57
+ async def on_merge_start(self, total_size: int) -> None:
58
+ self.merge_accumulated = 0
59
+ if total_size > 0:
60
+ total_steps = max(1, math.ceil(total_size / self.chunk_size_display))
61
+ self.merge_bar = ModernProgressBar(total=total_steps, process_name="Merge", spinner_mode=False)
62
+ self.merge_bar.start()
63
+
64
+ async def on_merge_update(self, loaded: int) -> None:
65
+ if self.merge_bar:
66
+ self.merge_accumulated += loaded
67
+ while self.merge_accumulated >= self.chunk_size_display:
68
+ self.merge_bar.update()
69
+ self.merge_accumulated -= self.chunk_size_display
70
+
71
+ async def on_merge_complete(self) -> None:
72
+ if self.merge_bar:
73
+ self.merge_bar.finish()
74
+
75
+ async def on_error(self, msg: str) -> None:
76
+ if self.logger:
77
+ self.logger.log(msg, "ERROR")
78
+
79
+ async def async_main() -> None:
80
+ parser = argparse.ArgumentParser(prog='fastget', description='Modern High-Performance Downloader')
81
+ parser.add_argument('url', help="Target URL")
82
+
83
+ parser.add_argument('-o', '--output', help="File destination")
84
+ parser.add_argument('-X', '--method', default='GET', help="HTTP method (GET/POST)")
85
+ parser.add_argument('-d', '--data', help="Data for POST method")
86
+ parser.add_argument('-H', '--header', action='append', help="Custom Headers")
87
+ parser.add_argument('-t', '--threads', type=int, default=fastget.DEFAULT_THREADS, help="Number of threads to use for downloading")
88
+ parser.add_argument('-p', '--print', action='store_true', help="Output data directly to stdout without saving to a file")
89
+
90
+ strategy_group = parser.add_mutually_exclusive_group()
91
+ strategy_group.add_argument('-s', '--storage', '--low-memory', action='store_const', const='storage', dest='strategy', help="Avoid using memory as much as possible, and perform tasks such as saving destinations and merging on received data only on the storage device as much as possible. (default)")
92
+ strategy_group.add_argument('-m', '--memory', '--low-storage', action='store_const', const='memory', dest='strategy', help="Utilize memory efficiently to reduce maximum concurrent storage usage.")
93
+ parser.set_defaults(strategy='storage')
94
+
95
+ parser.add_argument('--no-verify', action='store_true', help="In the case of HTTPS, if a secure connection cannot be established, the system will continue to operate normally.")
96
+ parser.add_argument('--no-info', action='store_true', help="Suppresses all displays such as progress bars. If --print is used, only data is output to stdout.")
97
+
98
+ parser.add_argument('--no-http1', action='store_true', help="Do not use HTTP/1 or HTTP/1.1")
99
+ parser.add_argument('--no-http2', action='store_true', help="Do not use HTTP/2")
100
+ parser.add_argument('--no-http3', action='store_true', help="Do not use HTTP/3")
101
+
102
+ args = parser.parse_args()
103
+
104
+ if args.print and args.output:
105
+ parser.error("If -p/--print is specified, -o/--output cannot be specified.")
106
+
107
+ if args.no_info:
108
+ logger = None
109
+ callback = fastget.ProgressCallback()
110
+ else:
111
+ logger = ModernLogging("FastGet")
112
+ callback = CLIProgress(logger)
113
+
114
+ headers = {}
115
+ if args.header:
116
+ for h in args.header:
117
+ if ':' in h:
118
+ k, v = h.split(':', 1)
119
+ headers[k.strip()] = v.strip()
120
+
121
+ method = args.method.upper()
122
+ if args.data:
123
+ method = 'POST'
124
+
125
+ session = fastget.FastGetSession(
126
+ max_threads=args.threads,
127
+ http1=not args.no_http1,
128
+ http2=not args.no_http2,
129
+ http3=not args.no_http3,
130
+ verify=not args.no_verify
131
+ )
132
+
133
+ start_time = 0
134
+ if logger:
135
+ start_time = asyncio.get_running_loop().time()
136
+
137
+ try:
138
+ if args.print:
139
+ result = await session.process(
140
+ method=method,
141
+ url=args.url,
142
+ data=args.data,
143
+ headers=headers,
144
+ callback=callback
145
+ )
146
+ sys.stdout.buffer.write(result.content)
147
+ else:
148
+ output = args.output
149
+ if not output:
150
+ parsed = urlparse(args.url)
151
+ output = unquote(os.path.basename(parsed.path)) or "downloaded_file"
152
+
153
+ path = await session.process(
154
+ method=method,
155
+ url=args.url,
156
+ output=output,
157
+ data=args.data,
158
+ headers=headers,
159
+ callback=callback,
160
+ strategy=args.strategy
161
+ )
162
+ if logger:
163
+ end_time = asyncio.get_running_loop().time()
164
+ duration_ms = (end_time - start_time) * 1000
165
+ logger.log(f"Completed in {duration_ms:.2f}ms")
166
+ logger.log(f"Saved to: {path}")
167
+
168
+ except fastget.FastGetError as e:
169
+ if logger:
170
+ logger.log(str(e), "CRITICAL")
171
+ else:
172
+ print(f"Error: {e}", file=sys.stderr, flush=True)
173
+ except Exception as e:
174
+ if logger:
175
+ logger.log(f"Unexpected error: {e}", "CRITICAL")
176
+ else:
177
+ print(f"Unexpected error: {e}", file=sys.stderr, flush=True)
178
+
179
+ def main() -> None:
180
+ try:
181
+ import uvloop
182
+ uvloop.install()
183
+ except ImportError:
184
+ pass
185
+
186
+ asyncio.run(async_main())
187
+
188
+ if __name__ == "__main__":
189
+ main()
@@ -0,0 +1,267 @@
1
+ import os
2
+ import asyncio
3
+ from curl_cffi import aio
4
+ from curl_cffi.errors import CurlError
5
+ from curl_cffi.const import HTTPVersion
6
+ from importlib.metadata import version
7
+ from typing import Union, Optional, Dict, Any, TypeVar, Awaitable, List
8
+
9
+ try:
10
+ VERSION = version("nercone-fastget")
11
+ except Exception:
12
+ VERSION = "0.0.0"
13
+
14
+ DEFAULT_CHUNK_SIZE = 1024 * 64
15
+ DEFAULT_TIMEOUT = 30.0
16
+ DEFAULT_RETRIES = 3
17
+ DEFAULT_THREADS = 8
18
+
19
+ T = TypeVar("T")
20
+
21
+ class FastGetError(Exception):
22
+ pass
23
+
24
+ class ProgressCallback:
25
+ async def on_start(self, total_size: int, threads: int, http_version: str, final_url: str, verify_was_enabled: bool) -> None: pass
26
+ async def on_update(self, worker_id: int, loaded: int) -> None: pass
27
+ async def on_complete(self) -> None: pass
28
+ async def on_merge_start(self, total_size: int) -> None: pass
29
+ async def on_merge_update(self, loaded: int) -> None: pass
30
+ async def on_merge_complete(self) -> None: pass
31
+ async def on_error(self, msg: str) -> None: pass
32
+
33
+ class FastGetResponse:
34
+ def __init__(self, original: aio.AsyncClient.Response, content: bytes):
35
+ self._r = original
36
+ self.content = content
37
+ self.url = str(original.url)
38
+ self.status_code = original.status_code
39
+ self.headers = original.headers
40
+ self.http_version = original.http_version
41
+
42
+ @property
43
+ def text(self) -> str:
44
+ return self.content.decode(self._r.encoding or 'utf-8', errors='replace')
45
+
46
+ def json(self, **kwargs) -> Any:
47
+ return self._r.json(**kwargs)
48
+
49
+ class FastGetSession:
50
+ def __init__(self, max_threads: int = DEFAULT_THREADS, http1: bool = True, http2: bool = True, http3: bool = True, verify: bool = True, follow_redirects: bool = True):
51
+ self.max_threads = max_threads
52
+
53
+ self.http_versions_to_try = []
54
+ if http3:
55
+ self.http_versions_to_try.append(HTTPVersion.V3)
56
+ if http2:
57
+ self.http_versions_to_try.append(HTTPVersion.V2)
58
+ if http1:
59
+ self.http_versions_to_try.append(HTTPVersion.V1_1)
60
+
61
+ if not self.http_versions_to_try:
62
+ raise ValueError("At least one of HTTP/1, HTTP/2, or HTTP/3 must be enabled.")
63
+
64
+ self.client_args = {
65
+ "verify": verify,
66
+ "follow_redirects": follow_redirects,
67
+ "timeout": DEFAULT_TIMEOUT,
68
+ }
69
+
70
+ async def _get_info(self, client: aio.AsyncClient, method: str, url: str, **kwargs) -> tuple[int, bool, bool, Optional[aio.AsyncClient.Response]]:
71
+ headers = kwargs.get("headers", {}).copy()
72
+ headers["User-Agent"] = f'FastGet/{VERSION} (Info)'
73
+ if method.upper() != "GET":
74
+ return 0, False, False, None
75
+
76
+ try:
77
+ head_resp = await client.head(url, headers=headers)
78
+ head_resp.raise_for_status()
79
+ size = int(head_resp.headers.get("content-length", 0))
80
+ accept_ranges = head_resp.headers.get("accept-ranges", "").lower() == "bytes"
81
+ reject_fg = head_resp.headers.get("rejectfastget", "").lower() in ["true", "1", "yes"]
82
+ return size, accept_ranges, reject_fg, head_resp
83
+
84
+ except CurlError:
85
+ try:
86
+ get_headers = headers.copy()
87
+ get_headers["Range"] = "bytes=0-0"
88
+ get_resp = await client.get(url, headers=get_headers)
89
+ get_resp.raise_for_status()
90
+ size_str = "0"
91
+ if get_resp.status_code == 206 and "content-range" in get_resp.headers:
92
+ content_range = get_resp.headers["content-range"]
93
+ if '/' in content_range:
94
+ size_str = content_range.split('/')[-1]
95
+ else:
96
+ size_str = get_resp.headers.get("content-length", "0")
97
+
98
+ size = int(size_str) if size_str.isdigit() else 0
99
+ accept_ranges = get_resp.headers.get("accept-ranges", "").lower() == "bytes" or get_resp.status_code == 206
100
+ reject_fg = get_resp.headers.get("rejectfastget", "").lower() in ["true", "1", "yes"]
101
+ return size, accept_ranges, reject_fg, get_resp
102
+ except CurlError:
103
+ raise
104
+
105
+ async def _download_worker_to_storage(self, client: aio.AsyncClient, method: str, url: str, start: int, end: int, worker_id: int, total_threads: int, part_path: str, callback: ProgressCallback, **kwargs) -> None:
106
+ headers = kwargs.get("headers", {}).copy()
107
+ headers["Range"] = f"bytes={start}-{end}"
108
+ headers["User-Agent"] = f'FastGet/{VERSION} (Worker {worker_id+1}/{total_threads})'
109
+ kwargs["headers"] = headers
110
+ for attempt in range(DEFAULT_RETRIES):
111
+ try:
112
+ response = await client.request(method, url, **kwargs)
113
+ response.raise_for_status()
114
+ with open(part_path, "wb") as f:
115
+ async for chunk in response.aiter_content(chunk_size=DEFAULT_CHUNK_SIZE):
116
+ f.write(chunk)
117
+ await callback.on_update(worker_id, len(chunk))
118
+ return
119
+ except CurlError as e:
120
+ if attempt == DEFAULT_RETRIES - 1:
121
+ await callback.on_error(f"Worker {worker_id+1} failed: {e}")
122
+ raise
123
+ await asyncio.sleep(1)
124
+
125
+ async def _download_worker_to_memory(self, client: aio.AsyncClient, method: str, url: str, start: int, end: int, worker_id: int, total_threads: int, callback: ProgressCallback, **kwargs) -> bytes:
126
+ headers = kwargs.get("headers", {}).copy()
127
+ headers["Range"] = f"bytes={start}-{end}"
128
+ headers["User-Agent"] = f'FastGet/{VERSION} (Worker {worker_id+1}/{total_threads})'
129
+ kwargs["headers"] = headers
130
+ part_buffer = bytearray()
131
+ for attempt in range(DEFAULT_RETRIES):
132
+ try:
133
+ response = await client.request(method, url, **kwargs)
134
+ response.raise_for_status()
135
+ async for chunk in response.aiter_content(chunk_size=DEFAULT_CHUNK_SIZE):
136
+ part_buffer.extend(chunk)
137
+ await callback.on_update(worker_id, len(chunk))
138
+ return bytes(part_buffer)
139
+ except CurlError as e:
140
+ if attempt == DEFAULT_RETRIES - 1:
141
+ await callback.on_error(f"Worker {worker_id+1} failed: {e}")
142
+ raise
143
+ part_buffer.clear()
144
+ await asyncio.sleep(1)
145
+ return b""
146
+
147
+ async def process(self, method: str, url: str, output: Optional[str] = None, data: Any = None, json: Any = None, params: Any = None, headers: Dict = None, callback: Optional[ProgressCallback] = None, strategy: str = 'storage') -> Union[str, FastGetResponse]:
148
+ callback = callback or ProgressCallback()
149
+ headers = headers or {}
150
+ req_kwargs = {"data": data, "json": json, "params": params, "headers": headers}
151
+
152
+ client = None
153
+ last_error = None
154
+ info_result = None
155
+
156
+ for version in self.http_versions_to_try:
157
+ client_args = self.client_args.copy()
158
+ client_args["http_version"] = version
159
+ temp_client = aio.AsyncClient(**client_args)
160
+ try:
161
+ info_result = await self._get_info(temp_client, method, url, **req_kwargs)
162
+ client = temp_client
163
+ break
164
+ except CurlError as e:
165
+ last_error = e
166
+ await temp_client.close()
167
+ is_last_attempt = (version == self.http_versions_to_try[-1])
168
+ if not is_last_attempt:
169
+ await callback.on_error(f"Request with {version.name} failed. Trying fallback...")
170
+
171
+ if not client:
172
+ await callback.on_error(f"All HTTP versions failed. Last error: {last_error}")
173
+ raise FastGetError(f"Failed to connect to {url} with all tried HTTP versions.") from last_error
174
+
175
+ try:
176
+ file_size, is_resumable, is_rejected, info_response = info_result
177
+
178
+ if method.upper() == "GET" and not info_response:
179
+ raise FastGetError(f"Failed to retrieve file information from {url}")
180
+
181
+ use_parallel = (method.upper() == "GET" and is_resumable and not is_rejected and file_size > 0 and output is not None and self.max_threads > 1)
182
+ threads = self.max_threads if use_parallel else 1
183
+ http_version = info_response.http_version if info_response else "UNKNOWN"
184
+ final_url = str(info_response.url) if info_response else url
185
+ verify_was_enabled = self.client_args.get("verify", True)
186
+
187
+ await callback.on_start(file_size, threads, http_version, final_url, verify_was_enabled)
188
+ if output:
189
+ out_dir = os.path.dirname(output)
190
+ if out_dir: os.makedirs(out_dir, exist_ok=True)
191
+ if use_parallel:
192
+ part_size = file_size // threads
193
+ tasks = []
194
+ if strategy == 'memory':
195
+ for i in range(threads):
196
+ start = i * part_size
197
+ end = file_size - 1 if i == threads - 1 else start + part_size - 1
198
+ tasks.append(self._download_worker_to_memory(client, method, url, start, end, i, threads, callback, **req_kwargs))
199
+ parts_in_memory: List[bytes] = await asyncio.gather(*tasks)
200
+ await callback.on_merge_start(file_size)
201
+ with open(output, "wb") as outfile:
202
+ for part_data in parts_in_memory:
203
+ outfile.write(part_data)
204
+ await callback.on_merge_update(len(part_data))
205
+ await callback.on_merge_complete()
206
+ else:
207
+ part_files = []
208
+ for i in range(threads):
209
+ start = i * part_size
210
+ end = file_size - 1 if i == threads - 1 else start + part_size - 1
211
+ part_path = f"{output}.part{i}"
212
+ part_files.append(part_path)
213
+ tasks.append(self._download_worker_to_storage(client, method, url, start, end, i, threads, part_path, callback, **req_kwargs))
214
+ await asyncio.gather(*tasks)
215
+ await callback.on_merge_start(file_size)
216
+ with open(output, "wb") as outfile:
217
+ for part_file in part_files:
218
+ try:
219
+ with open(part_file, "rb") as infile:
220
+ while chunk := infile.read(DEFAULT_CHUNK_SIZE):
221
+ outfile.write(chunk)
222
+ await callback.on_merge_update(len(chunk))
223
+ os.remove(part_file)
224
+ except FileNotFoundError:
225
+ await callback.on_error(f"Part file {part_file} not found during merge.")
226
+ await callback.on_merge_complete()
227
+ else:
228
+ headers["User-Agent"] = f'FastGet/{VERSION} (Single-Thread)'
229
+ response = await client.request(method, url, **req_kwargs)
230
+ response.raise_for_status()
231
+ with open(output, "wb") as f:
232
+ async for chunk in response.aiter_content(chunk_size=DEFAULT_CHUNK_SIZE):
233
+ f.write(chunk)
234
+ await callback.on_update(0, len(chunk))
235
+ await callback.on_complete()
236
+ return output
237
+ else:
238
+ headers["User-Agent"] = f'FastGet/{VERSION} (Single-Thread/Memory)'
239
+ response = await client.request(method, url, **req_kwargs)
240
+ response.raise_for_status()
241
+ await callback.on_update(0, len(response.content))
242
+ await callback.on_complete()
243
+ return FastGetResponse(response, response.content)
244
+ finally:
245
+ if client:
246
+ await client.close()
247
+
248
+ def run_sync(coro: Awaitable[T]) -> T:
249
+ try: loop = asyncio.get_running_loop()
250
+ except RuntimeError:
251
+ loop = asyncio.new_event_loop()
252
+ asyncio.set_event_loop(loop)
253
+ return loop.run_until_complete(coro)
254
+
255
+ def download(url: str, output: str, **kwargs) -> str:
256
+ session = FastGetSession(max_threads=kwargs.pop("threads", DEFAULT_THREADS), http1=not kwargs.pop("no_http1", False), http2=not kwargs.pop("no_http2", False), http3=not kwargs.pop("no_http3", False))
257
+ return run_sync(session.process("GET", url, output=output, **kwargs))
258
+
259
+ def request(method: str, url: str, **kwargs) -> FastGetResponse:
260
+ session = FastGetSession(max_threads=kwargs.pop("threads", DEFAULT_THREADS), http1=not kwargs.pop("no_http1", False), http2=not kwargs.pop("no_http2", False), http3=not kwargs.pop("no_http3", False))
261
+ return run_sync(session.process(method, url, output=None, **kwargs))
262
+
263
+ def get(url: str, **kwargs) -> FastGetResponse:
264
+ return request("GET", url, **kwargs)
265
+
266
+ def post(url: str, **kwargs) -> FastGetResponse:
267
+ return request("POST", url, **kwargs)