moxdl 1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mox.py +1633 -0
- moxdl-1.4.dist-info/METADATA +233 -0
- moxdl-1.4.dist-info/RECORD +7 -0
- moxdl-1.4.dist-info/WHEEL +5 -0
- moxdl-1.4.dist-info/entry_points.txt +2 -0
- moxdl-1.4.dist-info/licenses/LICENSE +21 -0
- moxdl-1.4.dist-info/top_level.txt +1 -0
mox.py
ADDED
|
@@ -0,0 +1,1633 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""mox: a fast, resumable, multi-connection download manager for the terminal.
|
|
3
|
+
|
|
4
|
+
Uses only the Python standard library.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import datetime
|
|
11
|
+
import functools
|
|
12
|
+
import hashlib
|
|
13
|
+
import http.client
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import shlex
|
|
18
|
+
import shutil
|
|
19
|
+
import socket
|
|
20
|
+
import ssl
|
|
21
|
+
import subprocess
|
|
22
|
+
import sys
|
|
23
|
+
import threading
|
|
24
|
+
import time
|
|
25
|
+
import unicodedata
|
|
26
|
+
import urllib.error
|
|
27
|
+
import urllib.parse
|
|
28
|
+
import urllib.request
|
|
29
|
+
from collections import deque
|
|
30
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
31
|
+
|
|
32
|
+
VERSION = "1.4"
|
|
33
|
+
CHUNK = 256 * 1024
|
|
34
|
+
MIN_SPLIT = 256 * 1024 # smallest piece when first dividing a file between connections
|
|
35
|
+
MIN_STEAL = 64 * 1024 # smallest piece an idle connection takes over from a busy one
|
|
36
|
+
MIN_STEAL_ETA = 1.0 # seconds: only take over work from a piece that would take longer than this
|
|
37
|
+
IDLE_POLL = 0.2 # seconds an idle connection waits before looking for work again
|
|
38
|
+
MIN_ACTIVE = 2 # a busy server never makes a download drop below this many connections
|
|
39
|
+
DEFAULT_CONNECTIONS = 16
|
|
40
|
+
MAX_CONNECTIONS = 64
|
|
41
|
+
STATE_SUFFIX = ".mox"
|
|
42
|
+
PART_SUFFIX = ".part"
|
|
43
|
+
SAVE_INTERVAL = 1.0
|
|
44
|
+
FRAME_INTERVAL = 0.25
|
|
45
|
+
SPEED_WINDOW = 0.5 # seconds between updates of each piece's measured speed
|
|
46
|
+
MIN_MEASURE = 0.3 # seconds a request must run before its speed is trusted
|
|
47
|
+
STALL_TIME = 5.0 # seconds without a single byte before a request counts as stalled
|
|
48
|
+
SLOW_WINDOW = 1.5 # seconds between checks of a connection's speed
|
|
49
|
+
SLOW_RATIO = 0.3 # slower than this share of the average connection counts as slow
|
|
50
|
+
SLOW_RECONNECTS = 3 # reconnect a slow connection at most this many times per piece
|
|
51
|
+
RETRYABLE_HTTP = {408, 425, 429, 500, 502, 503, 504}
|
|
52
|
+
BUSY_HTTP = {429, 503} # the server wants fewer connections
|
|
53
|
+
EXPIRED_HTTP = {401, 403, 410} # what an expired signed link usually returns
|
|
54
|
+
RETRYABLE_ERRORS = (urllib.error.URLError, OSError, http.client.HTTPException)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class FatalError(Exception):
|
|
58
|
+
"""A failure that retrying will not fix."""
|
|
59
|
+
|
|
60
|
+
def __init__(self, message: str, status: int | None = None):
|
|
61
|
+
super().__init__(message)
|
|
62
|
+
self.status = status
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class ServerBusy(ConnectionError):
|
|
66
|
+
"""A retryable HTTP status (5xx, 429) on a keep-alive connection."""
|
|
67
|
+
|
|
68
|
+
def __init__(self, status: int):
|
|
69
|
+
super().__init__(f"HTTP {status}")
|
|
70
|
+
self.status = status
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class SlowConnection(Exception):
|
|
74
|
+
"""This connection is much slower than its siblings; reconnecting may get a better route."""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class Retire(Exception):
|
|
78
|
+
"""The server limits connections: this worker steps aside and leaves its piece to the others."""
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# ---------------------------------------------------------------- formatting
|
|
82
|
+
|
|
83
|
+
def human_size(n: float | None) -> str:
|
|
84
|
+
if n is None:
|
|
85
|
+
return "?"
|
|
86
|
+
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
87
|
+
if abs(n) < 1024 or unit == "TB":
|
|
88
|
+
return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
|
|
89
|
+
n /= 1024
|
|
90
|
+
return "?"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def human_time(seconds: float | None) -> str:
|
|
94
|
+
if seconds is None:
|
|
95
|
+
return "--:--"
|
|
96
|
+
seconds = int(seconds)
|
|
97
|
+
hours, rest = divmod(seconds, 3600)
|
|
98
|
+
minutes, seconds = divmod(rest, 60)
|
|
99
|
+
return f"{hours}:{minutes:02}:{seconds:02}" if hours else f"{minutes:02}:{seconds:02}"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def parse_rate(text: str) -> int:
|
|
103
|
+
match = re.fullmatch(r"\s*(\d+(?:\.\d+)?)\s*([kmg]?)b?\s*", text, re.I)
|
|
104
|
+
if not match:
|
|
105
|
+
raise argparse.ArgumentTypeError(f"invalid rate {text!r} (examples: 500K, 2M)")
|
|
106
|
+
multiplier = {"": 1, "k": 1024, "m": 1024**2, "g": 1024**3}[match.group(2).lower()]
|
|
107
|
+
return int(float(match.group(1)) * multiplier)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def char_width(ch: str) -> int:
|
|
111
|
+
if unicodedata.combining(ch):
|
|
112
|
+
return 0
|
|
113
|
+
return 2 if unicodedata.east_asian_width(ch) in "WF" else 1
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def clip(text: str, width: int) -> str:
|
|
117
|
+
"""Cut or pad text to exactly `width` terminal columns."""
|
|
118
|
+
out, used = [], 0
|
|
119
|
+
for ch in text:
|
|
120
|
+
w = char_width(ch)
|
|
121
|
+
if used + w > width:
|
|
122
|
+
if width >= 1 and out:
|
|
123
|
+
while used > width - 1:
|
|
124
|
+
used -= char_width(out.pop())
|
|
125
|
+
out.append("…")
|
|
126
|
+
used += 1
|
|
127
|
+
break
|
|
128
|
+
out.append(ch)
|
|
129
|
+
used += w
|
|
130
|
+
return "".join(out) + " " * (width - used)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
# ---------------------------------------------------------------- HTTP
|
|
134
|
+
|
|
135
|
+
def build_headers(extra, start=None, end=None) -> dict:
|
|
136
|
+
headers = {"User-Agent": f"mox/{VERSION}", "Accept-Encoding": "identity", **extra}
|
|
137
|
+
if start is not None:
|
|
138
|
+
headers["Range"] = f"bytes={start}-" + ("" if end is None else str(end))
|
|
139
|
+
return headers
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class Proxy:
|
|
143
|
+
"""A proxy given with --proxy: socks5://, socks5h:// or http://, optionally with user:password@."""
|
|
144
|
+
|
|
145
|
+
SCHEMES = ("socks5", "socks5h", "http")
|
|
146
|
+
|
|
147
|
+
def __init__(self, url: str):
|
|
148
|
+
parts = urllib.parse.urlsplit(url)
|
|
149
|
+
if parts.scheme not in self.SCHEMES or not parts.hostname:
|
|
150
|
+
raise ValueError(f"unsupported proxy {url!r}; use socks5://host:port or http://host:port")
|
|
151
|
+
self.url = url
|
|
152
|
+
self.socks = parts.scheme.startswith("socks")
|
|
153
|
+
self.host = parts.hostname
|
|
154
|
+
self.port = parts.port or (1080 if self.socks else 8080)
|
|
155
|
+
self.username = urllib.parse.unquote(parts.username) if parts.username else None
|
|
156
|
+
self.password = urllib.parse.unquote(parts.password) if parts.password else ""
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
_proxy: Proxy | None = None
|
|
160
|
+
_opener = urllib.request.build_opener() # honours http_proxy / https_proxy from the environment
|
|
161
|
+
|
|
162
|
+
SOCKS_ERRORS = {
|
|
163
|
+
1: "general failure", 2: "connection not allowed by ruleset", 3: "network unreachable",
|
|
164
|
+
4: "host unreachable", 5: "connection refused", 6: "TTL expired",
|
|
165
|
+
7: "command not supported", 8: "address type not supported",
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _recv_exact(sock: socket.socket, n: int) -> bytes:
|
|
170
|
+
data = b""
|
|
171
|
+
while len(data) < n:
|
|
172
|
+
chunk = sock.recv(n - len(data))
|
|
173
|
+
if not chunk:
|
|
174
|
+
raise ConnectionError("SOCKS proxy closed the connection")
|
|
175
|
+
data += chunk
|
|
176
|
+
return data
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def socks5_connect(proxy: Proxy, host: str, port: int, timeout) -> socket.socket:
|
|
180
|
+
"""Open a TCP connection to host:port through a SOCKS5 proxy (RFC 1928, auth per RFC 1929).
|
|
181
|
+
|
|
182
|
+
The host name is sent to the proxy unresolved, so DNS happens on the proxy's side.
|
|
183
|
+
"""
|
|
184
|
+
timeout = timeout if isinstance(timeout, (int, float)) else None
|
|
185
|
+
sock = socket.create_connection((proxy.host, proxy.port), timeout)
|
|
186
|
+
try:
|
|
187
|
+
methods = b"\x00\x02" if proxy.username else b"\x00"
|
|
188
|
+
sock.sendall(b"\x05" + bytes([len(methods)]) + methods)
|
|
189
|
+
version, method = _recv_exact(sock, 2)
|
|
190
|
+
if version != 5 or method == 0xFF:
|
|
191
|
+
raise FatalError("SOCKS proxy refused every login method we offered")
|
|
192
|
+
if method == 2:
|
|
193
|
+
user, password = proxy.username.encode(), proxy.password.encode()
|
|
194
|
+
sock.sendall(b"\x01" + bytes([len(user)]) + user + bytes([len(password)]) + password)
|
|
195
|
+
if _recv_exact(sock, 2)[1] != 0:
|
|
196
|
+
raise FatalError("SOCKS proxy rejected the user name or password")
|
|
197
|
+
name = host.encode("idna")
|
|
198
|
+
sock.sendall(b"\x05\x01\x00\x03" + bytes([len(name)]) + name + port.to_bytes(2, "big"))
|
|
199
|
+
_, status, _, address_type = _recv_exact(sock, 4)
|
|
200
|
+
if status != 0:
|
|
201
|
+
raise ConnectionError(f"SOCKS proxy: {SOCKS_ERRORS.get(status, f'error {status}')}")
|
|
202
|
+
_recv_exact(sock, {1: 4, 4: 16}.get(address_type) or _recv_exact(sock, 1)[0])
|
|
203
|
+
_recv_exact(sock, 2) # bound port
|
|
204
|
+
return sock
|
|
205
|
+
except BaseException:
|
|
206
|
+
sock.close()
|
|
207
|
+
raise
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
class _SocksMixin:
|
|
211
|
+
def __init__(self, *args, proxy: Proxy, **kwargs):
|
|
212
|
+
super().__init__(*args, **kwargs)
|
|
213
|
+
self._create_connection = lambda address, timeout=None, source_address=None: \
|
|
214
|
+
socks5_connect(proxy, address[0], address[1], timeout)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class SocksHTTPConnection(_SocksMixin, http.client.HTTPConnection):
|
|
218
|
+
pass
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
class SocksHTTPSConnection(_SocksMixin, http.client.HTTPSConnection):
|
|
222
|
+
pass
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
class _SocksHTTPHandler(urllib.request.HTTPHandler):
|
|
226
|
+
def __init__(self, proxy: Proxy):
|
|
227
|
+
super().__init__()
|
|
228
|
+
self.proxy = proxy
|
|
229
|
+
|
|
230
|
+
def http_open(self, req):
|
|
231
|
+
return self.do_open(functools.partial(SocksHTTPConnection, proxy=self.proxy), req)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class _SocksHTTPSHandler(urllib.request.HTTPSHandler):
|
|
235
|
+
def __init__(self, proxy: Proxy):
|
|
236
|
+
super().__init__()
|
|
237
|
+
self.proxy = proxy
|
|
238
|
+
|
|
239
|
+
def https_open(self, req):
|
|
240
|
+
return self.do_open(functools.partial(SocksHTTPSConnection, proxy=self.proxy), req, context=self._context)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def configure_proxy(url: str | None) -> None:
|
|
244
|
+
"""Route every request through --proxy. Without it, http_proxy / https_proxy still apply."""
|
|
245
|
+
global _proxy, _opener
|
|
246
|
+
if not url:
|
|
247
|
+
_proxy, _opener = None, urllib.request.build_opener()
|
|
248
|
+
return
|
|
249
|
+
_proxy = Proxy(url)
|
|
250
|
+
if _proxy.socks:
|
|
251
|
+
_opener = urllib.request.build_opener(urllib.request.ProxyHandler({}),
|
|
252
|
+
_SocksHTTPHandler(_proxy), _SocksHTTPSHandler(_proxy))
|
|
253
|
+
else:
|
|
254
|
+
_opener = urllib.request.build_opener(urllib.request.ProxyHandler({"http": url, "https": url}))
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def request(url, headers, timeout, start=None, end=None):
|
|
258
|
+
req = urllib.request.Request(url, headers=build_headers(headers, start, end))
|
|
259
|
+
try:
|
|
260
|
+
return _opener.open(req, timeout=timeout)
|
|
261
|
+
except urllib.error.HTTPError as e:
|
|
262
|
+
if e.code in RETRYABLE_HTTP:
|
|
263
|
+
raise
|
|
264
|
+
e.close() # the error carries the open response; we are done with it
|
|
265
|
+
raise FatalError(f"HTTP {e.code} {e.reason}", e.code) from None
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def keep_alive_possible(url: str) -> bool:
|
|
269
|
+
"""Our own persistent connections work directly or through SOCKS, but not through an HTTP proxy."""
|
|
270
|
+
if _proxy:
|
|
271
|
+
return _proxy.socks
|
|
272
|
+
parts = urllib.parse.urlsplit(url)
|
|
273
|
+
return not (parts.scheme in urllib.request.getproxies()
|
|
274
|
+
and not urllib.request.proxy_bypass(parts.hostname or ""))
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
class KeepAlive:
|
|
278
|
+
"""One worker's persistent connection, so each new piece skips the TCP and TLS handshakes."""
|
|
279
|
+
|
|
280
|
+
def __init__(self, url: str, timeout: float):
|
|
281
|
+
parts = urllib.parse.urlsplit(url)
|
|
282
|
+
self.https = parts.scheme == "https"
|
|
283
|
+
self.host, self.port = parts.hostname, parts.port
|
|
284
|
+
self.target = (parts.path or "/") + (f"?{parts.query}" if parts.query else "")
|
|
285
|
+
self.timeout = timeout
|
|
286
|
+
self.conn: http.client.HTTPConnection | None = None
|
|
287
|
+
|
|
288
|
+
def _connect(self) -> http.client.HTTPConnection:
|
|
289
|
+
extra = {"proxy": _proxy} if _proxy else {}
|
|
290
|
+
if self.https:
|
|
291
|
+
cls = SocksHTTPSConnection if _proxy else http.client.HTTPSConnection
|
|
292
|
+
return cls(self.host, self.port, timeout=self.timeout, context=ssl.create_default_context(), **extra)
|
|
293
|
+
cls = SocksHTTPConnection if _proxy else http.client.HTTPConnection
|
|
294
|
+
return cls(self.host, self.port, timeout=self.timeout, **extra)
|
|
295
|
+
|
|
296
|
+
def get(self, headers: dict) -> http.client.HTTPResponse:
|
|
297
|
+
reused = self.conn is not None
|
|
298
|
+
while True:
|
|
299
|
+
if self.conn is None:
|
|
300
|
+
self.conn = self._connect()
|
|
301
|
+
try:
|
|
302
|
+
self.conn.request("GET", self.target, headers=headers)
|
|
303
|
+
return self.conn.getresponse()
|
|
304
|
+
except (OSError, http.client.HTTPException):
|
|
305
|
+
self.close()
|
|
306
|
+
if not reused:
|
|
307
|
+
raise
|
|
308
|
+
reused = False # the server dropped the idle connection; one fresh try
|
|
309
|
+
|
|
310
|
+
def close(self) -> None:
|
|
311
|
+
if self.conn is not None:
|
|
312
|
+
self.conn.close()
|
|
313
|
+
self.conn = None
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def safe_name(name: str) -> str:
|
|
317
|
+
name = name.replace("\\", "/").rsplit("/", 1)[-1]
|
|
318
|
+
name = re.sub(r'[\x00-\x1f<>:"|?*]', "_", name).strip(" .")
|
|
319
|
+
return name[:200]
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def filename_from(resp, url: str) -> str:
|
|
323
|
+
disposition = resp.headers.get("Content-Disposition", "")
|
|
324
|
+
name = ""
|
|
325
|
+
encoded = re.search(r"filename\*\s*=\s*([^']*)'[^']*'([^;]+)", disposition, re.I)
|
|
326
|
+
if encoded:
|
|
327
|
+
try:
|
|
328
|
+
name = urllib.parse.unquote(encoded.group(2).strip().strip('"'),
|
|
329
|
+
encoding=encoded.group(1) or "utf-8", errors="replace")
|
|
330
|
+
except LookupError:
|
|
331
|
+
name = urllib.parse.unquote(encoded.group(2).strip().strip('"'))
|
|
332
|
+
else:
|
|
333
|
+
plain = re.search(r'filename\s*=\s*(?:"([^"]+)"|([^;]+))', disposition, re.I)
|
|
334
|
+
if plain:
|
|
335
|
+
name = (plain.group(1) or plain.group(2)).strip()
|
|
336
|
+
if not name:
|
|
337
|
+
name = urllib.parse.unquote(urllib.parse.urlsplit(url).path.rsplit("/", 1)[-1])
|
|
338
|
+
return safe_name(name) or "download"
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def validator_from(resp) -> str | None:
|
|
342
|
+
"""What identifies this version of the file: a strong ETag, or failing that Last-Modified.
|
|
343
|
+
|
|
344
|
+
Weak ETags (W/"...") are useless here, since If-Range never matches them.
|
|
345
|
+
"""
|
|
346
|
+
etag = resp.headers.get("ETag", "").strip()
|
|
347
|
+
if etag.startswith('"'):
|
|
348
|
+
return etag
|
|
349
|
+
return resp.headers.get("Last-Modified", "").strip() or None
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def probe(url, headers, timeout):
|
|
353
|
+
"""Return (final_url, total_size_or_None, supports_ranges, filename, validator_or_None)."""
|
|
354
|
+
try:
|
|
355
|
+
resp = request(url, headers, timeout, start=0)
|
|
356
|
+
except FatalError as e:
|
|
357
|
+
if e.status != 416: # servers answer "bytes=0-" on an empty file with 416
|
|
358
|
+
raise
|
|
359
|
+
resp = request(url, headers, timeout)
|
|
360
|
+
with resp:
|
|
361
|
+
final = resp.geturl()
|
|
362
|
+
name = filename_from(resp, final)
|
|
363
|
+
validator = validator_from(resp)
|
|
364
|
+
if resp.status == 206:
|
|
365
|
+
match = re.match(r"bytes\s+\d+-\d+/(\d+)", resp.headers.get("Content-Range", ""))
|
|
366
|
+
if match:
|
|
367
|
+
return final, int(match.group(1)), True, name, validator
|
|
368
|
+
length = resp.headers.get("Content-Length", "")
|
|
369
|
+
total = int(length) if length.isdigit() and resp.status == 200 else None
|
|
370
|
+
return final, total, False, name, validator
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
if hasattr(os, "pwrite"):
|
|
374
|
+
def write_at(fd: int, data: bytes, pos: int) -> None:
|
|
375
|
+
view = memoryview(data)
|
|
376
|
+
while view:
|
|
377
|
+
written = os.pwrite(fd, view, pos)
|
|
378
|
+
view, pos = view[written:], pos + written
|
|
379
|
+
else: # Windows
|
|
380
|
+
_write_lock = threading.Lock()
|
|
381
|
+
|
|
382
|
+
def write_at(fd: int, data: bytes, pos: int) -> None:
|
|
383
|
+
with _write_lock:
|
|
384
|
+
os.lseek(fd, pos, os.SEEK_SET)
|
|
385
|
+
os.write(fd, data)
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
_claimed_paths: set[str] = set() # paths taken by downloads in this run
|
|
389
|
+
_claim_lock = threading.Lock()
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
class RateLimiter:
|
|
393
|
+
"""Token bucket shared by every connection of every download."""
|
|
394
|
+
|
|
395
|
+
def __init__(self, rate: int | None):
|
|
396
|
+
self.rate = rate
|
|
397
|
+
self.capacity = max((rate or 0) // 4, CHUNK) # small bucket: no big burst at start
|
|
398
|
+
self.tokens = float(self.capacity)
|
|
399
|
+
self.last = time.monotonic()
|
|
400
|
+
self.lock = threading.Lock()
|
|
401
|
+
|
|
402
|
+
def consume(self, n: int) -> None:
|
|
403
|
+
if not self.rate:
|
|
404
|
+
return
|
|
405
|
+
while True:
|
|
406
|
+
with self.lock:
|
|
407
|
+
now = time.monotonic()
|
|
408
|
+
self.tokens = min(self.capacity, self.tokens + (now - self.last) * self.rate)
|
|
409
|
+
self.last = now
|
|
410
|
+
if self.tokens >= n:
|
|
411
|
+
self.tokens -= n
|
|
412
|
+
return
|
|
413
|
+
wait = (n - self.tokens) / self.rate
|
|
414
|
+
time.sleep(wait)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
# ---------------------------------------------------------------- download
|
|
418
|
+
|
|
419
|
+
class Download:
|
|
420
|
+
def __init__(self, url: str, opts, name: str | None = None, sha256: str | None = None):
|
|
421
|
+
self.url = self.final_url = url
|
|
422
|
+
self.opts = opts
|
|
423
|
+
self.custom_name = name
|
|
424
|
+
self.name = name or safe_name(urllib.parse.unquote(urllib.parse.urlsplit(url).path)) or url
|
|
425
|
+
self.sha256 = sha256.lower() if sha256 else None
|
|
426
|
+
self.status = "waiting"
|
|
427
|
+
self.message = ""
|
|
428
|
+
self.error_status: int | None = None
|
|
429
|
+
self.total: int | None = None
|
|
430
|
+
self.done = 0
|
|
431
|
+
self.ranges = False
|
|
432
|
+
self.validator: str | None = None
|
|
433
|
+
self.active = 0
|
|
434
|
+
self.path: str | None = None
|
|
435
|
+
self.segments: list[dict] = []
|
|
436
|
+
self.lock = threading.Lock()
|
|
437
|
+
self.failed = threading.Event()
|
|
438
|
+
self.samples: deque = deque()
|
|
439
|
+
self.retired = 0
|
|
440
|
+
self.work_changed = threading.Condition(self.lock) # signalled whenever a piece finishes
|
|
441
|
+
|
|
442
|
+
# -- public
|
|
443
|
+
|
|
444
|
+
def restart(self) -> Download:
|
|
445
|
+
"""A fresh copy that picks up from the saved state file, for the next scheduled session."""
|
|
446
|
+
return Download(self.url, self.opts, self.custom_name, self.sha256)
|
|
447
|
+
|
|
448
|
+
def run(self, stop: threading.Event, limiter: RateLimiter) -> None:
|
|
449
|
+
if stop.is_set():
|
|
450
|
+
return
|
|
451
|
+
try:
|
|
452
|
+
self._run(stop, limiter)
|
|
453
|
+
except FatalError as e:
|
|
454
|
+
self.status, self.message, self.error_status = "error", str(e), e.status
|
|
455
|
+
except Exception as e: # keep one broken download from killing the rest
|
|
456
|
+
self.status, self.message = "error", f"{type(e).__name__}: {e}"
|
|
457
|
+
|
|
458
|
+
def pieces(self) -> list[tuple[int, int]]:
|
|
459
|
+
"""(start, bytes done) for every piece, for drawing the map."""
|
|
460
|
+
with self.lock:
|
|
461
|
+
return [(s["start"], s["done"]) for s in self.segments]
|
|
462
|
+
|
|
463
|
+
def speed(self) -> float:
|
|
464
|
+
now = time.monotonic()
|
|
465
|
+
self.samples.append((now, self.done))
|
|
466
|
+
while len(self.samples) > 2 and now - self.samples[0][0] > 4:
|
|
467
|
+
self.samples.popleft()
|
|
468
|
+
then, done_then = self.samples[0]
|
|
469
|
+
return (self.done - done_then) / (now - then) if now - then > 0.5 else 0.0
|
|
470
|
+
|
|
471
|
+
# -- orchestration
|
|
472
|
+
|
|
473
|
+
def _run(self, stop, limiter):
|
|
474
|
+
self.status = "connecting"
|
|
475
|
+
info = self._probe(stop)
|
|
476
|
+
if info is None:
|
|
477
|
+
self.status = "paused"
|
|
478
|
+
return
|
|
479
|
+
self.final_url, self.total, self.ranges, server_name, self.validator = info
|
|
480
|
+
if not self.custom_name:
|
|
481
|
+
self.name = server_name
|
|
482
|
+
self.path = self._choose_path()
|
|
483
|
+
self.name = os.path.basename(self.path)
|
|
484
|
+
if self.status == "done":
|
|
485
|
+
return
|
|
486
|
+
|
|
487
|
+
state_path, part_path = self.path + STATE_SUFFIX, self.path + PART_SUFFIX
|
|
488
|
+
resumed = self._load_state(state_path, part_path)
|
|
489
|
+
self.segments = resumed or self._plan()
|
|
490
|
+
self.done = sum(s["done"] for s in self.segments)
|
|
491
|
+
if resumed:
|
|
492
|
+
self.message = f"resumed at {human_size(self.done)}"
|
|
493
|
+
|
|
494
|
+
with open(part_path, "r+b" if resumed else "wb", buffering=0) as f:
|
|
495
|
+
if not resumed and self.total:
|
|
496
|
+
f.truncate(self.total)
|
|
497
|
+
self.status = "downloading"
|
|
498
|
+
connections = self.opts.connections if self.ranges else 1
|
|
499
|
+
workers = [threading.Thread(target=self._worker, args=(f.fileno(), stop, limiter), daemon=True)
|
|
500
|
+
for _ in range(connections)]
|
|
501
|
+
for w in workers:
|
|
502
|
+
w.start()
|
|
503
|
+
while alive := [w for w in workers if w.is_alive()]:
|
|
504
|
+
alive[0].join(SAVE_INTERVAL)
|
|
505
|
+
self._save_state(state_path)
|
|
506
|
+
if self.total is None and not self.failed.is_set() and not stop.is_set():
|
|
507
|
+
f.truncate(self.done)
|
|
508
|
+
|
|
509
|
+
if self.failed.is_set():
|
|
510
|
+
self._save_state(state_path)
|
|
511
|
+
self.status = "error"
|
|
512
|
+
return
|
|
513
|
+
if stop.is_set():
|
|
514
|
+
self._save_state(state_path)
|
|
515
|
+
self.status = "paused"
|
|
516
|
+
return
|
|
517
|
+
if self.total is not None and self.done != self.total:
|
|
518
|
+
raise FatalError(f"incomplete: got {self.done} of {self.total} bytes")
|
|
519
|
+
|
|
520
|
+
if self.sha256:
|
|
521
|
+
self.status = "verifying"
|
|
522
|
+
digest = sha256_file(part_path)
|
|
523
|
+
if digest != self.sha256:
|
|
524
|
+
self._remove(state_path)
|
|
525
|
+
raise FatalError(f"SHA-256 mismatch (got {digest[:16]}…), kept {os.path.basename(part_path)}")
|
|
526
|
+
self.message = "SHA-256 verified"
|
|
527
|
+
os.replace(part_path, self.path)
|
|
528
|
+
self._remove(state_path)
|
|
529
|
+
self.status = "done"
|
|
530
|
+
|
|
531
|
+
def _probe(self, stop):
|
|
532
|
+
attempts = 0
|
|
533
|
+
while not stop.is_set():
|
|
534
|
+
try:
|
|
535
|
+
return probe(self.url, self.opts.headers, self.opts.timeout)
|
|
536
|
+
except RETRYABLE_ERRORS as e:
|
|
537
|
+
attempts += 1
|
|
538
|
+
if attempts > self.opts.retries:
|
|
539
|
+
raise FatalError(f"cannot connect: {reason(e)}") from None
|
|
540
|
+
self.message = f"retry {attempts}: {reason(e)}"
|
|
541
|
+
stop.wait(min(2**attempts, 30))
|
|
542
|
+
return None
|
|
543
|
+
|
|
544
|
+
def _choose_path(self) -> str:
|
|
545
|
+
os.makedirs(self.opts.dir, exist_ok=True)
|
|
546
|
+
path = os.path.join(self.opts.dir, self.name)
|
|
547
|
+
base, ext = os.path.splitext(path)
|
|
548
|
+
counter = 1
|
|
549
|
+
with _claim_lock:
|
|
550
|
+
while True:
|
|
551
|
+
if os.path.abspath(path) not in _claimed_paths:
|
|
552
|
+
if os.path.exists(path + STATE_SUFFIX) or not os.path.exists(path):
|
|
553
|
+
break
|
|
554
|
+
if self.total is not None and os.path.isfile(path) and os.path.getsize(path) == self.total:
|
|
555
|
+
self.status, self.message, self.done = "done", "already downloaded", self.total
|
|
556
|
+
break
|
|
557
|
+
path = f"{base} ({counter}){ext}"
|
|
558
|
+
counter += 1
|
|
559
|
+
_claimed_paths.add(os.path.abspath(path))
|
|
560
|
+
return path
|
|
561
|
+
|
|
562
|
+
def _plan(self) -> list[dict]:
|
|
563
|
+
if self.total == 0:
|
|
564
|
+
return []
|
|
565
|
+
if not self.ranges or self.total is None:
|
|
566
|
+
end = None if self.total is None else self.total - 1
|
|
567
|
+
return [{"start": 0, "end": end, "done": 0}]
|
|
568
|
+
count = max(1, min(self.opts.connections, self.total // MIN_SPLIT))
|
|
569
|
+
size = self.total // count
|
|
570
|
+
return [{"start": i * size,
|
|
571
|
+
"end": self.total - 1 if i == count - 1 else (i + 1) * size - 1,
|
|
572
|
+
"done": 0} for i in range(count)]
|
|
573
|
+
|
|
574
|
+
# -- state file (lets an interrupted download resume)
|
|
575
|
+
|
|
576
|
+
def _load_state(self, state_path, part_path) -> list[dict] | None:
|
|
577
|
+
if not (self.ranges and os.path.exists(state_path) and os.path.exists(part_path)):
|
|
578
|
+
return None
|
|
579
|
+
try:
|
|
580
|
+
with open(state_path, encoding="utf-8") as f:
|
|
581
|
+
state = json.load(f)
|
|
582
|
+
if state.get("total") != self.total or os.path.getsize(part_path) != self.total:
|
|
583
|
+
return None
|
|
584
|
+
# Same size is not proof of the same file: a new release can keep the size.
|
|
585
|
+
# Without a validator on either side, the size check is all there is.
|
|
586
|
+
saved = state.get("validator")
|
|
587
|
+
if saved and self.validator and saved != self.validator:
|
|
588
|
+
self.message = "changed on the server, started over"
|
|
589
|
+
return None
|
|
590
|
+
return [{"start": int(s["start"]), "end": int(s["end"]), "done": int(s["done"])}
|
|
591
|
+
for s in state["segments"]]
|
|
592
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
593
|
+
return None
|
|
594
|
+
|
|
595
|
+
def _save_state(self, state_path) -> None:
|
|
596
|
+
if not self.ranges:
|
|
597
|
+
return
|
|
598
|
+
with self.lock:
|
|
599
|
+
segments = [{k: s[k] for k in ("start", "end", "done")} for s in self.segments]
|
|
600
|
+
state = {"version": 1, "url": self.url, "total": self.total, "validator": self.validator,
|
|
601
|
+
"segments": segments}
|
|
602
|
+
tmp = state_path + ".tmp"
|
|
603
|
+
with open(tmp, "w", encoding="utf-8") as f:
|
|
604
|
+
json.dump(state, f)
|
|
605
|
+
os.replace(tmp, state_path)
|
|
606
|
+
|
|
607
|
+
@staticmethod
|
|
608
|
+
def _remove(path) -> None:
|
|
609
|
+
try:
|
|
610
|
+
os.remove(path)
|
|
611
|
+
except FileNotFoundError:
|
|
612
|
+
pass
|
|
613
|
+
|
|
614
|
+
# -- connections
|
|
615
|
+
|
|
616
|
+
def _halted(self, stop) -> bool:
|
|
617
|
+
return stop.is_set() or self.failed.is_set()
|
|
618
|
+
|
|
619
|
+
@staticmethod
|
|
620
|
+
def _complete(seg) -> bool:
|
|
621
|
+
if seg["end"] is None:
|
|
622
|
+
return seg.get("eof", False)
|
|
623
|
+
return seg["start"] + seg["done"] > seg["end"]
|
|
624
|
+
|
|
625
|
+
@staticmethod
|
|
626
|
+
def _speed_of(seg, now: float) -> float:
|
|
627
|
+
"""Bytes/s of a piece: its smoothed rate, or failing that its rate since its request began."""
|
|
628
|
+
if seg.get("speed"):
|
|
629
|
+
return seg["speed"]
|
|
630
|
+
elapsed = now - seg.get("t0", now)
|
|
631
|
+
if elapsed < MIN_MEASURE:
|
|
632
|
+
return 0.0
|
|
633
|
+
received = seg["done"] - seg["b0"]
|
|
634
|
+
if received:
|
|
635
|
+
return received / elapsed
|
|
636
|
+
# Nothing yet. Only call it stalled after a long while: on a high-latency link a
|
|
637
|
+
# fresh request can easily take a second to deliver its first byte.
|
|
638
|
+
return 1.0 if elapsed >= STALL_TIME else 0.0
|
|
639
|
+
|
|
640
|
+
@staticmethod
|
|
641
|
+
def _free_from(seg) -> int:
|
|
642
|
+
"""First byte nobody has claimed: past both the written data and any chunk being written."""
|
|
643
|
+
return max(seg["start"] + seg["done"], seg.get("reserved", 0))
|
|
644
|
+
|
|
645
|
+
def _next_segment(self, my_speed: float = 0.0) -> dict | None:
|
|
646
|
+
"""Claim unfinished work, or take over part of the piece that will finish last.
|
|
647
|
+
|
|
648
|
+
`my_speed` is how fast the asking connection ran its previous piece.
|
|
649
|
+
"""
|
|
650
|
+
with self.lock:
|
|
651
|
+
for seg in self.segments:
|
|
652
|
+
if not seg.get("busy") and not self._complete(seg):
|
|
653
|
+
seg["busy"] = True
|
|
654
|
+
return seg
|
|
655
|
+
if not self.ranges:
|
|
656
|
+
return None
|
|
657
|
+
# Only pieces that have run long enough to have a measured speed can be split:
|
|
658
|
+
# guessing the speed of a fresh piece lets idle connections pass it back and
|
|
659
|
+
# forth forever without anyone downloading it.
|
|
660
|
+
now = time.monotonic()
|
|
661
|
+
speeds = {id(s): self._speed_of(s, now) for s in self.segments if s.get("busy")}
|
|
662
|
+
measured = [s for s in self.segments if speeds.get(id(s))]
|
|
663
|
+
if not measured:
|
|
664
|
+
return None
|
|
665
|
+
thief = my_speed or sum(speeds[id(s)] for s in measured) / len(measured)
|
|
666
|
+
|
|
667
|
+
def eta(seg) -> float:
|
|
668
|
+
return (seg["end"] - self._free_from(seg) + 1) / speeds[id(seg)]
|
|
669
|
+
|
|
670
|
+
victim = max(measured, key=eta)
|
|
671
|
+
if eta(victim) < MIN_STEAL_ETA: # finishing soon anyway; a new request would not pay off
|
|
672
|
+
return None
|
|
673
|
+
free = self._free_from(victim)
|
|
674
|
+
left = victim["end"] - free + 1
|
|
675
|
+
# Split so both sides finish together: a connection half as fast keeps a third.
|
|
676
|
+
theirs = speeds[id(victim)]
|
|
677
|
+
keep = int(left * theirs / (theirs + thief))
|
|
678
|
+
mid = free + keep
|
|
679
|
+
if victim["end"] - mid + 1 < MIN_STEAL:
|
|
680
|
+
return None
|
|
681
|
+
stolen = {"start": mid, "end": victim["end"], "done": 0, "busy": True}
|
|
682
|
+
victim["end"] = mid - 1
|
|
683
|
+
self.segments.append(stolen)
|
|
684
|
+
return stolen
|
|
685
|
+
|
|
686
|
+
def _worker(self, fd, stop, limiter) -> None:
|
|
687
|
+
with self.lock:
|
|
688
|
+
self.active += 1
|
|
689
|
+
link = KeepAlive(self.final_url, self.opts.timeout) \
|
|
690
|
+
if self.ranges and keep_alive_possible(self.final_url) else None
|
|
691
|
+
my_speed = 0.0
|
|
692
|
+
try:
|
|
693
|
+
while not self._halted(stop):
|
|
694
|
+
seg = self._next_segment(my_speed)
|
|
695
|
+
if seg is None:
|
|
696
|
+
with self.lock:
|
|
697
|
+
if not any(not self._complete(s) for s in self.segments):
|
|
698
|
+
break
|
|
699
|
+
# A busy piece may become worth splitting soon; wake early if one finishes.
|
|
700
|
+
self.work_changed.wait(IDLE_POLL)
|
|
701
|
+
continue
|
|
702
|
+
try:
|
|
703
|
+
self._fetch(seg, fd, stop, limiter, link)
|
|
704
|
+
finally:
|
|
705
|
+
with self.lock:
|
|
706
|
+
seg["busy"] = False
|
|
707
|
+
my_speed = self._speed_of(seg, time.monotonic()) or my_speed
|
|
708
|
+
self.work_changed.notify_all()
|
|
709
|
+
except Retire:
|
|
710
|
+
with self.lock:
|
|
711
|
+
self.retired += 1
|
|
712
|
+
self.message = f"server limits connections, using {self.active - 1}"
|
|
713
|
+
except FatalError as e:
|
|
714
|
+
self.message, self.error_status = str(e), e.status
|
|
715
|
+
self.failed.set()
|
|
716
|
+
except Exception as e:
|
|
717
|
+
self.message = f"{type(e).__name__}: {e}"
|
|
718
|
+
self.failed.set()
|
|
719
|
+
finally:
|
|
720
|
+
if link:
|
|
721
|
+
link.close()
|
|
722
|
+
with self.lock:
|
|
723
|
+
self.active -= 1
|
|
724
|
+
|
|
725
|
+
def _open_range(self, link: KeepAlive | None, start, end):
|
|
726
|
+
if not self.ranges:
|
|
727
|
+
return request(self.final_url, self.opts.headers, self.opts.timeout)
|
|
728
|
+
# If-Range makes the server send the whole new file instead of a piece of it
|
|
729
|
+
# when the file has changed, so pieces of two versions never get mixed.
|
|
730
|
+
headers = {**self.opts.headers, "If-Range": self.validator} if self.validator else self.opts.headers
|
|
731
|
+
if link is None: # going through a proxy: let urllib handle it
|
|
732
|
+
resp = request(self.final_url, headers, self.opts.timeout, start, end)
|
|
733
|
+
else:
|
|
734
|
+
resp = link.get(build_headers(headers, start, end))
|
|
735
|
+
if resp.status != 206:
|
|
736
|
+
link.close()
|
|
737
|
+
if resp.status in RETRYABLE_HTTP:
|
|
738
|
+
raise ServerBusy(resp.status)
|
|
739
|
+
if resp.status != 200:
|
|
740
|
+
raise FatalError(f"HTTP {resp.status} {resp.reason}", resp.status)
|
|
741
|
+
if resp.status != 206:
|
|
742
|
+
resp.close()
|
|
743
|
+
if self.validator:
|
|
744
|
+
raise FatalError("the file changed on the server; run again to start over")
|
|
745
|
+
raise FatalError("server stopped honoring range requests")
|
|
746
|
+
return resp
|
|
747
|
+
|
|
748
|
+
def _fetch(self, seg, fd, stop, limiter, link) -> None:
|
|
749
|
+
attempts = 0
|
|
750
|
+
slow_reconnects = 0
|
|
751
|
+
while not self._halted(stop):
|
|
752
|
+
with self.lock:
|
|
753
|
+
if self._complete(seg):
|
|
754
|
+
return
|
|
755
|
+
pos, end = seg["start"] + seg["done"], seg["end"]
|
|
756
|
+
seg["t0"], seg["b0"] = time.monotonic(), seg["done"]
|
|
757
|
+
received = 0
|
|
758
|
+
finished = False
|
|
759
|
+
try:
|
|
760
|
+
with self._open_range(link, pos, end) as resp:
|
|
761
|
+
window_start, window_bytes = time.monotonic(), 0
|
|
762
|
+
speed_start, speed_bytes = window_start, 0
|
|
763
|
+
while not self._halted(stop):
|
|
764
|
+
with self.lock:
|
|
765
|
+
end = seg["end"]
|
|
766
|
+
want = CHUNK if end is None else min(CHUNK, end - pos + 1)
|
|
767
|
+
if want <= 0:
|
|
768
|
+
finished = True
|
|
769
|
+
break
|
|
770
|
+
data = resp.read1(want)
|
|
771
|
+
if not data:
|
|
772
|
+
break
|
|
773
|
+
limiter.consume(len(data))
|
|
774
|
+
with self.lock:
|
|
775
|
+
# Another connection may have taken over the tail meanwhile:
|
|
776
|
+
# keep only our share, and reserve it before writing.
|
|
777
|
+
end = seg["end"]
|
|
778
|
+
if end is not None:
|
|
779
|
+
data = data[:max(0, end - pos + 1)]
|
|
780
|
+
seg["reserved"] = pos + len(data)
|
|
781
|
+
if not data:
|
|
782
|
+
finished = True
|
|
783
|
+
break
|
|
784
|
+
write_at(fd, data, pos)
|
|
785
|
+
if not received and self.message.startswith("retry"):
|
|
786
|
+
self.message = ""
|
|
787
|
+
pos += len(data)
|
|
788
|
+
received += len(data)
|
|
789
|
+
window_bytes += len(data)
|
|
790
|
+
speed_bytes += len(data)
|
|
791
|
+
now = time.monotonic()
|
|
792
|
+
with self.lock:
|
|
793
|
+
seg["done"] += len(data)
|
|
794
|
+
self.done += len(data)
|
|
795
|
+
if now - speed_start >= SPEED_WINDOW:
|
|
796
|
+
instant = speed_bytes / (now - speed_start)
|
|
797
|
+
seg["speed"] = 0.5 * seg["speed"] + 0.5 * instant if seg.get("speed") else instant
|
|
798
|
+
speed_start, speed_bytes = now, 0
|
|
799
|
+
if now - window_start >= SLOW_WINDOW:
|
|
800
|
+
if slow_reconnects < SLOW_RECONNECTS and self._is_slow(window_bytes / (now - window_start)):
|
|
801
|
+
raise SlowConnection
|
|
802
|
+
window_start, window_bytes = now, 0
|
|
803
|
+
# A keep-alive connection is reusable only once its response is fully read.
|
|
804
|
+
if link and not resp.isclosed():
|
|
805
|
+
link.close()
|
|
806
|
+
if finished or self._halted(stop):
|
|
807
|
+
return
|
|
808
|
+
if end is None:
|
|
809
|
+
with self.lock:
|
|
810
|
+
seg["eof"] = True
|
|
811
|
+
return
|
|
812
|
+
raise ConnectionError("connection closed early")
|
|
813
|
+
except SlowConnection:
|
|
814
|
+
slow_reconnects += 1
|
|
815
|
+
if link:
|
|
816
|
+
link.close()
|
|
817
|
+
except RETRYABLE_ERRORS as e:
|
|
818
|
+
if link:
|
|
819
|
+
link.close()
|
|
820
|
+
if self._server_busy(e) and self.active > MIN_ACTIVE:
|
|
821
|
+
raise Retire from None
|
|
822
|
+
attempts = 1 if received else attempts + 1
|
|
823
|
+
if attempts > self.opts.retries:
|
|
824
|
+
raise FatalError(f"gave up after {self.opts.retries} retries: {reason(e)}") from None
|
|
825
|
+
if not self.ranges: # no way to continue mid-file; start over
|
|
826
|
+
with self.lock:
|
|
827
|
+
self.done -= seg["done"]
|
|
828
|
+
seg["done"] = 0
|
|
829
|
+
self.message = f"retry {attempts}: {reason(e)}"
|
|
830
|
+
stop.wait(min(2**attempts, 30))
|
|
831
|
+
|
|
832
|
+
def _is_slow(self, my_speed: float) -> bool:
|
|
833
|
+
if self.active < 2 or self.opts.limit: # with a speed cap, slowness is on purpose
|
|
834
|
+
return False
|
|
835
|
+
with self.lock:
|
|
836
|
+
measured = [s["speed"] for s in self.segments if s.get("busy") and s.get("speed")]
|
|
837
|
+
if len(measured) < 2:
|
|
838
|
+
return False
|
|
839
|
+
fastest_half = sorted(measured)[len(measured) // 2:]
|
|
840
|
+
typical = sum(fastest_half) / len(fastest_half)
|
|
841
|
+
return my_speed < SLOW_RATIO * typical
|
|
842
|
+
|
|
843
|
+
@staticmethod
|
|
844
|
+
def _server_busy(e: BaseException) -> bool:
|
|
845
|
+
status = getattr(e, "status", None) if isinstance(e, ServerBusy) else getattr(e, "code", None)
|
|
846
|
+
return status in BUSY_HTTP
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def reason(e: BaseException) -> str:
|
|
850
|
+
if isinstance(e, urllib.error.HTTPError):
|
|
851
|
+
return f"HTTP {e.code}"
|
|
852
|
+
if isinstance(e, urllib.error.URLError):
|
|
853
|
+
return str(e.reason)
|
|
854
|
+
return str(e) or type(e).__name__
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def sha256_file(path: str) -> str:
|
|
858
|
+
digest = hashlib.sha256()
|
|
859
|
+
with open(path, "rb") as f:
|
|
860
|
+
while block := f.read(1024 * 1024):
|
|
861
|
+
digest.update(block)
|
|
862
|
+
return digest.hexdigest()
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
# ---------------------------------------------------------------- display
|
|
866
|
+
|
|
867
|
+
STYLE = { # status: (icon, ANSI color)
|
|
868
|
+
"waiting": ("·", "2"),
|
|
869
|
+
"connecting": ("…", "33"),
|
|
870
|
+
"downloading": ("↓", "36"),
|
|
871
|
+
"verifying": ("#", "35"),
|
|
872
|
+
"done": ("✔", "32"),
|
|
873
|
+
"error": ("✖", "31"),
|
|
874
|
+
"paused": ("‖", "33"),
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
BAR = "━" # every cell of the progress bar; its color says how much of that cell is done
|
|
879
|
+
# By shade: nothing, some, most, all of the cell's bytes on disk. Marking the cells connections
|
|
880
|
+
# were writing to made the bar solid white once there were more connections than cells.
|
|
881
|
+
BAR_COLORS = ("2;37", "2;36", "36", "96")
|
|
882
|
+
BAR_MIN, BAR_GOOD, BAR_MAX = 10, 20, 40 # past BAR_MAX a bar only spreads the same information thinner
|
|
883
|
+
NAME_MIN = 16 # narrower than this and rows drop columns to keep the name readable
|
|
884
|
+
NAME_WANT = 40 # names get this much before the bar grows past BAR_MIN
|
|
885
|
+
FIELDS = [(6, ""), (10, ""), (7, ""), (19, "2"), (3, "2")] # width and style: %, speed, ETA, size, connections
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def fields_width(count: int) -> int:
|
|
889
|
+
return sum(w + 2 for w, _ in FIELDS[:count])
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def coverage_map(pieces: list[tuple[int, int]], total: int, width: int) -> list[int]:
|
|
893
|
+
"""Split the file into `width` cells and shade each by how much of it is on disk: 0 for
|
|
894
|
+
nothing, 1 for some, 2 for most, 3 for all."""
|
|
895
|
+
if not total or width <= 0:
|
|
896
|
+
return []
|
|
897
|
+
cell = total / width
|
|
898
|
+
filled = [0.0] * width
|
|
899
|
+
for start, done in pieces:
|
|
900
|
+
end = start + done # bytes [start, end) are on disk
|
|
901
|
+
if done > 0:
|
|
902
|
+
for i in range(int(start // cell), min(width - 1, int((end - 1) // cell)) + 1):
|
|
903
|
+
overlap = min(end, (i + 1) * cell) - max(start, i * cell)
|
|
904
|
+
if overlap > 0:
|
|
905
|
+
filled[i] += overlap / cell
|
|
906
|
+
return [3 if f >= 0.999 else 2 if f >= 0.5 else 1 if f > 0 else 0 for f in filled]
|
|
907
|
+
|
|
908
|
+
|
|
909
|
+
class Display:
|
|
910
|
+
def __init__(self, downloads: list[Download], stream=sys.stderr, watching: bool = False):
|
|
911
|
+
self.downloads = downloads
|
|
912
|
+
self.stream = stream
|
|
913
|
+
self.tty = stream.isatty()
|
|
914
|
+
self.watching = watching
|
|
915
|
+
self.drawn: list[int] = [] # screen width of each row of the last frame
|
|
916
|
+
self.reported: dict[int, str] = {}
|
|
917
|
+
|
|
918
|
+
def draw(self) -> None:
|
|
919
|
+
if not self.tty:
|
|
920
|
+
self._log_changes()
|
|
921
|
+
return
|
|
922
|
+
size = shutil.get_terminal_size((100, 24))
|
|
923
|
+
# One column spare: a row that fills the last column leaves some terminals waiting to wrap.
|
|
924
|
+
width = size.columns - 1
|
|
925
|
+
visible = self._visible(max(1, size.lines - 2))
|
|
926
|
+
layout = self._layout(visible, width)
|
|
927
|
+
rows = [self._row(d, layout, width) for d in visible]
|
|
928
|
+
rows.append(self._summary(width))
|
|
929
|
+
# After a resize the terminal rewraps the old frame, so a row drawn at 120 columns now
|
|
930
|
+
# takes two rows at 80. Moving up one row per old row left stale copies of the frame
|
|
931
|
+
# behind every time the window got narrower.
|
|
932
|
+
up = sum(max(1, -(-w // size.columns)) for w in self.drawn)
|
|
933
|
+
out = ["\x1b[?25l"]
|
|
934
|
+
if up:
|
|
935
|
+
out.append(f"\x1b[{up}F")
|
|
936
|
+
out.extend(f"\x1b[2K{row}\n" for row in rows)
|
|
937
|
+
out.append("\x1b[J")
|
|
938
|
+
self.drawn = [screen_width(row) for row in rows]
|
|
939
|
+
self.stream.write("".join(out))
|
|
940
|
+
self.stream.flush()
|
|
941
|
+
|
|
942
|
+
def close(self) -> None:
|
|
943
|
+
if self.tty:
|
|
944
|
+
self.stream.write("\x1b[?25h")
|
|
945
|
+
self.stream.flush()
|
|
946
|
+
|
|
947
|
+
def _visible(self, limit: int) -> list[Download]:
|
|
948
|
+
if len(self.downloads) <= limit:
|
|
949
|
+
return self.downloads
|
|
950
|
+
rank = {"downloading": 0, "connecting": 0, "verifying": 0, "error": 1, "paused": 1, "waiting": 2, "done": 3}
|
|
951
|
+
chosen = sorted(range(len(self.downloads)), key=lambda i: (rank[self.downloads[i].status], i))[:limit]
|
|
952
|
+
return [self.downloads[i] for i in sorted(chosen)]
|
|
953
|
+
|
|
954
|
+
@staticmethod
|
|
955
|
+
def _layout(visible: list[Download], width: int) -> tuple[int, int, int]:
|
|
956
|
+
"""Column widths shared by every row, so bars and numbers line up: (name, bar, fields
|
|
957
|
+
kept). Fields are dropped from the right, least useful first, until the name has room."""
|
|
958
|
+
longest = max([screen_width(d.name) for d in visible] + [NAME_MIN])
|
|
959
|
+
count = len(FIELDS)
|
|
960
|
+
# Size and connection count matter less than a bar long enough to read as a map.
|
|
961
|
+
while count > 3 and width - 2 - (BAR_GOOD + 2) - fields_width(count) < min(longest, NAME_WANT):
|
|
962
|
+
count -= 1
|
|
963
|
+
while count > 2 and width - 2 - (BAR_MIN + 2) - fields_width(count) < NAME_MIN:
|
|
964
|
+
count -= 1
|
|
965
|
+
room = width - 2 - fields_width(count)
|
|
966
|
+
bar = min(BAR_MAX, room - min(longest, NAME_WANT) - 2)
|
|
967
|
+
if bar < BAR_MIN:
|
|
968
|
+
bar = BAR_MIN if room - NAME_MIN - 2 >= BAR_MIN else 0
|
|
969
|
+
name = max(1, min(longest, room - (bar + 2 if bar else 0)))
|
|
970
|
+
return name, bar, count
|
|
971
|
+
|
|
972
|
+
def _row(self, d: Download, layout: tuple[int, int, int], width: int) -> str:
|
|
973
|
+
"""One line per download: name, a bar that doubles as the piece map, then the numbers."""
|
|
974
|
+
name_width, bar_width, count = layout
|
|
975
|
+
icon, color = STYLE[d.status]
|
|
976
|
+
parts = [(f"{icon} ", color), (clip(d.name, name_width), "2" if d.status == "done" else "")]
|
|
977
|
+
if bar_width:
|
|
978
|
+
parts.append((" ", ""))
|
|
979
|
+
parts.extend(self._bar(d, bar_width) if d.status != "done" else [(" " * bar_width, "")])
|
|
980
|
+
if d.status == "downloading" and not d.message.startswith("retry"):
|
|
981
|
+
speed = d.speed()
|
|
982
|
+
left = (d.total - d.done) / speed if d.total and speed > 0 else None
|
|
983
|
+
values = [percent(d), f"{human_size(speed)}/s", human_time(left),
|
|
984
|
+
f"{human_size(d.done)}/{human_size(d.total)}", f"{d.active}×"]
|
|
985
|
+
parts.extend((" " + value.rjust(w), style) for value, (w, style) in zip(values, FIELDS[:count]))
|
|
986
|
+
else:
|
|
987
|
+
if d.status == "done":
|
|
988
|
+
note, style = f"{d.message or 'done'} · {human_size(d.total or d.done)}", "2"
|
|
989
|
+
elif d.status == "downloading": # retrying
|
|
990
|
+
note, style = d.message, "33"
|
|
991
|
+
else:
|
|
992
|
+
note, style = d.message or d.status, "2"
|
|
993
|
+
parts.append((" " + ("" if d.status == "done" else percent(d)).rjust(FIELDS[0][0]), ""))
|
|
994
|
+
parts.append((" " + note, style))
|
|
995
|
+
return fit(parts, width)
|
|
996
|
+
|
|
997
|
+
@staticmethod
|
|
998
|
+
def _bar(d: Download, width: int) -> list[tuple[str, str]]:
|
|
999
|
+
if d.total:
|
|
1000
|
+
cells = coverage_map(d.pieces(), d.total, width)
|
|
1001
|
+
elif d.status == "downloading": # unknown size: a pulse that shows it is alive
|
|
1002
|
+
head = int(time.monotonic() * 8) % width
|
|
1003
|
+
cells = [3 if (i - head) % width < 3 else 0 for i in range(width)]
|
|
1004
|
+
else:
|
|
1005
|
+
cells = [0] * width
|
|
1006
|
+
parts: list[tuple[str, str]] = []
|
|
1007
|
+
for shade in cells:
|
|
1008
|
+
style = BAR_COLORS[shade]
|
|
1009
|
+
if d.status not in ("downloading", "verifying") and shade:
|
|
1010
|
+
style = "2;33" if d.status == "paused" else "2;37"
|
|
1011
|
+
if parts and parts[-1][1] == style:
|
|
1012
|
+
parts[-1] = (parts[-1][0] + BAR, style)
|
|
1013
|
+
else:
|
|
1014
|
+
parts.append((BAR, style))
|
|
1015
|
+
return parts
|
|
1016
|
+
|
|
1017
|
+
def _summary(self, width: int) -> str:
|
|
1018
|
+
counts: dict[str, int] = {}
|
|
1019
|
+
for d in self.downloads:
|
|
1020
|
+
counts[d.status] = counts.get(d.status, 0) + 1
|
|
1021
|
+
speed = sum(d.speed() for d in self.downloads if d.status == "downloading") \
|
|
1022
|
+
if counts.get("downloading") else 0
|
|
1023
|
+
parts = [f"{counts.get('done', 0)}/{len(self.downloads)} done"]
|
|
1024
|
+
if counts.get("error"):
|
|
1025
|
+
parts.append(f"{counts['error']} failed")
|
|
1026
|
+
if speed:
|
|
1027
|
+
parts.append(f"{human_size(speed)}/s total")
|
|
1028
|
+
if self.watching:
|
|
1029
|
+
parts.append("watching the clipboard for links · Ctrl+C to stop")
|
|
1030
|
+
elif any(d.status in ("waiting", "connecting", "downloading") for d in self.downloads):
|
|
1031
|
+
parts.append("Ctrl+C to pause")
|
|
1032
|
+
return fit([(" " + " · ".join(parts), "2")], width)
|
|
1033
|
+
|
|
1034
|
+
def _log_changes(self) -> None:
|
|
1035
|
+
for i, d in enumerate(self.downloads):
|
|
1036
|
+
if d.status in ("done", "error", "paused") and self.reported.get(i) != d.status:
|
|
1037
|
+
self.reported[i] = d.status
|
|
1038
|
+
detail = d.message or f"{human_size(d.done)}/{human_size(d.total)}"
|
|
1039
|
+
print(f"[{d.status}] {d.name}: {detail}", file=self.stream, flush=True)
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
def percent(d: Download) -> str:
|
|
1043
|
+
if d.total:
|
|
1044
|
+
return f"{min(1.0, d.done / d.total) * 100:.1f}%"
|
|
1045
|
+
return "100.0%" if d.status == "done" else ""
|
|
1046
|
+
|
|
1047
|
+
|
|
1048
|
+
def screen_width(text: str) -> int:
|
|
1049
|
+
return sum(char_width(c) for c in re.sub(r"\x1b\[[0-9;?]*[A-Za-z]", "", text))
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def fit(parts: list[tuple[str, str]], width: int) -> str:
|
|
1053
|
+
"""Join colored pieces, cut at `width` screen columns. Cutting by characters let wide
|
|
1054
|
+
characters in a file name push the row past the edge, where it wrapped and broke the redraw."""
|
|
1055
|
+
out, left = [], width
|
|
1056
|
+
for text, color in parts:
|
|
1057
|
+
kept = []
|
|
1058
|
+
for c in text:
|
|
1059
|
+
w = char_width(c)
|
|
1060
|
+
if w > left:
|
|
1061
|
+
left = 0
|
|
1062
|
+
break
|
|
1063
|
+
kept.append(c)
|
|
1064
|
+
left -= w
|
|
1065
|
+
if kept:
|
|
1066
|
+
text = "".join(kept)
|
|
1067
|
+
out.append(f"\x1b[{color}m{text}\x1b[0m" if color else text)
|
|
1068
|
+
if left <= 0:
|
|
1069
|
+
break
|
|
1070
|
+
return "".join(out)
|
|
1071
|
+
|
|
1072
|
+
|
|
1073
|
+
# ---------------------------------------------------------------- CLI
|
|
1074
|
+
|
|
1075
|
+
def add_network_options(parser: argparse.ArgumentParser) -> None:
|
|
1076
|
+
parser.add_argument("-t", "--timeout", type=float, default=30, help="network timeout in seconds (default: 30)")
|
|
1077
|
+
parser.add_argument("-H", "--header", action="append", default=[], metavar="'NAME: VALUE'",
|
|
1078
|
+
help="extra request header, e.g. cookie or referer (repeatable)")
|
|
1079
|
+
parser.add_argument("-x", "--proxy", metavar="URL",
|
|
1080
|
+
help="proxy for every request: socks5://host:port, socks5h://... or http://host:port")
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
def add_download_options(parser: argparse.ArgumentParser) -> None:
|
|
1084
|
+
parser.add_argument("-c", "--connections", type=int, default=DEFAULT_CONNECTIONS,
|
|
1085
|
+
help=f"connections per file, up to {MAX_CONNECTIONS} (default: {DEFAULT_CONNECTIONS})")
|
|
1086
|
+
parser.add_argument("-j", "--jobs", type=int, default=3, help="files downloaded at once (default: 3)")
|
|
1087
|
+
parser.add_argument("-l", "--limit", type=parse_rate, metavar="RATE", help="total speed limit, e.g. 500K or 2M")
|
|
1088
|
+
parser.add_argument("-r", "--retries", type=int, default=10, help="retries per connection (default: 10)")
|
|
1089
|
+
add_network_options(parser)
|
|
1090
|
+
parser.add_argument("--at", type=parse_clock, metavar="HH:MM",
|
|
1091
|
+
help="wait until this time of day before starting")
|
|
1092
|
+
parser.add_argument("--until", type=parse_clock, metavar="HH:MM",
|
|
1093
|
+
help="pause at this time of day; with --at, carry on at the next --at, day after day")
|
|
1094
|
+
parser.add_argument("-n", "--notify", action="store_true",
|
|
1095
|
+
help="when downloads end, send a desktop notification, and a Telegram message "
|
|
1096
|
+
"if MOX_TELEGRAM_TOKEN and MOX_TELEGRAM_CHAT are set")
|
|
1097
|
+
|
|
1098
|
+
|
|
1099
|
+
def check_network_options(parser: argparse.ArgumentParser, args) -> None:
|
|
1100
|
+
if args.proxy:
|
|
1101
|
+
try:
|
|
1102
|
+
Proxy(args.proxy)
|
|
1103
|
+
except ValueError as e:
|
|
1104
|
+
parser.error(str(e))
|
|
1105
|
+
headers = {}
|
|
1106
|
+
for header in args.header:
|
|
1107
|
+
name, sep, value = header.partition(":")
|
|
1108
|
+
if not sep or not name.strip():
|
|
1109
|
+
parser.error(f"bad header {header!r}, expected 'Name: value'")
|
|
1110
|
+
headers[name.strip()] = value.strip()
|
|
1111
|
+
args.headers = headers
|
|
1112
|
+
|
|
1113
|
+
|
|
1114
|
+
def check_download_options(parser: argparse.ArgumentParser, args) -> None:
|
|
1115
|
+
if not 1 <= args.connections <= MAX_CONNECTIONS:
|
|
1116
|
+
parser.error(f"--connections must be between 1 and {MAX_CONNECTIONS}")
|
|
1117
|
+
if args.jobs < 1:
|
|
1118
|
+
parser.error("--jobs must be at least 1")
|
|
1119
|
+
if args.at is not None and args.at == args.until:
|
|
1120
|
+
parser.error("--at and --until must differ")
|
|
1121
|
+
check_network_options(parser, args)
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def parse_args(argv):
|
|
1125
|
+
parser = argparse.ArgumentParser(
|
|
1126
|
+
prog="mox", description="Fast, resumable, multi-connection downloads in the terminal.",
|
|
1127
|
+
epilog="Interrupted downloads resume automatically: run the same command again. "
|
|
1128
|
+
"Other commands: 'mox resume [DIR]' continues every unfinished download in a folder, "
|
|
1129
|
+
"'mox bench URL' measures how much faster many connections are.")
|
|
1130
|
+
parser.add_argument("urls", nargs="*", metavar="URL")
|
|
1131
|
+
parser.add_argument("-i", "--input", metavar="FILE", help="read URLs from FILE, one per line ('-' for stdin)")
|
|
1132
|
+
parser.add_argument("-d", "--dir", default=".", help="directory to save into (default: current)")
|
|
1133
|
+
parser.add_argument("-o", "--output", metavar="NAME", help="file name to save as (single URL only)")
|
|
1134
|
+
parser.add_argument("--sha256", metavar="HEX", help="verify the file against this checksum (single URL only)")
|
|
1135
|
+
parser.add_argument("-w", "--watch", action="store_true",
|
|
1136
|
+
help="keep running and download every file link you copy to the clipboard")
|
|
1137
|
+
add_download_options(parser)
|
|
1138
|
+
parser.add_argument("-V", "--version", action="version", version=f"mox {VERSION}")
|
|
1139
|
+
args = parser.parse_args(argv)
|
|
1140
|
+
|
|
1141
|
+
urls = list(args.urls)
|
|
1142
|
+
if args.input:
|
|
1143
|
+
source = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8")
|
|
1144
|
+
with source:
|
|
1145
|
+
urls += [line.strip() for line in source if line.strip() and not line.lstrip().startswith("#")]
|
|
1146
|
+
urls = [u if "://" in u else "https://" + u for u in dict.fromkeys(urls)]
|
|
1147
|
+
if not urls and not args.watch:
|
|
1148
|
+
parser.error("give at least one URL (or -i FILE, or --watch)")
|
|
1149
|
+
if (len(urls) > 1 or args.watch) and (args.output or args.sha256):
|
|
1150
|
+
parser.error("--output and --sha256 work with a single URL only")
|
|
1151
|
+
if args.watch and (args.at or args.until):
|
|
1152
|
+
parser.error("--watch cannot be combined with --at or --until")
|
|
1153
|
+
check_download_options(parser, args)
|
|
1154
|
+
args.urls = urls
|
|
1155
|
+
return args
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
# ---------------------------------------------------------------- scheduling
|
|
1159
|
+
|
|
1160
|
+
def parse_clock(text: str) -> datetime.time:
|
|
1161
|
+
match = re.fullmatch(r"\s*(\d{1,2}):(\d{2})(?::(\d{2}))?\s*", text)
|
|
1162
|
+
if not match or int(match.group(1)) > 23 or int(match.group(2)) > 59 or int(match.group(3) or 0) > 59:
|
|
1163
|
+
raise argparse.ArgumentTypeError(f"invalid time {text!r} (examples: 02:00, 23:30)")
|
|
1164
|
+
return datetime.time(int(match.group(1)), int(match.group(2)), int(match.group(3) or 0))
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
def show_clock(clock: datetime.time) -> str:
|
|
1168
|
+
return f"{clock:%H:%M:%S}" if clock.second else f"{clock:%H:%M}"
|
|
1169
|
+
|
|
1170
|
+
|
|
1171
|
+
def next_time(clock: datetime.time, now: datetime.datetime) -> datetime.datetime:
|
|
1172
|
+
"""The next moment the wall clock shows `clock`: later today, or else tomorrow."""
|
|
1173
|
+
moment = now.replace(hour=clock.hour, minute=clock.minute, second=clock.second, microsecond=0)
|
|
1174
|
+
return moment if moment > now else moment + datetime.timedelta(days=1)
|
|
1175
|
+
|
|
1176
|
+
|
|
1177
|
+
def in_window(now: datetime.datetime, start: datetime.time, end: datetime.time) -> bool:
|
|
1178
|
+
"""Whether `now` falls between start and end, for windows that may cross midnight."""
|
|
1179
|
+
t = now.time()
|
|
1180
|
+
return start <= t < end if start < end else (t >= start or t < end)
|
|
1181
|
+
|
|
1182
|
+
|
|
1183
|
+
def wait_until(moment: datetime.datetime) -> bool:
|
|
1184
|
+
"""Sleep until `moment`, showing a countdown. False if the user pressed Ctrl+C."""
|
|
1185
|
+
tty = sys.stderr.isatty()
|
|
1186
|
+
if not tty:
|
|
1187
|
+
print(f"Waiting until {moment:%H:%M:%S}", file=sys.stderr, flush=True)
|
|
1188
|
+
try:
|
|
1189
|
+
while (left := (moment - datetime.datetime.now()).total_seconds()) > 0:
|
|
1190
|
+
if tty:
|
|
1191
|
+
sys.stderr.write(f"\r\x1b[2K\x1b[2m… waiting until {moment:%H:%M:%S} "
|
|
1192
|
+
f"({human_time(left)} left) · Ctrl+C to cancel\x1b[0m")
|
|
1193
|
+
sys.stderr.flush()
|
|
1194
|
+
time.sleep(min(1.0, left))
|
|
1195
|
+
except KeyboardInterrupt:
|
|
1196
|
+
return False
|
|
1197
|
+
finally:
|
|
1198
|
+
if tty:
|
|
1199
|
+
sys.stderr.write("\r\x1b[2K")
|
|
1200
|
+
sys.stderr.flush()
|
|
1201
|
+
return True
|
|
1202
|
+
|
|
1203
|
+
|
|
1204
|
+
# ---------------------------------------------------------------- notifications
|
|
1205
|
+
|
|
1206
|
+
def notify(title: str, body: str) -> None:
|
|
1207
|
+
"""Desktop notification where a tool for it exists, plus Telegram when configured.
|
|
1208
|
+
|
|
1209
|
+
A notification that fails only prints a warning: it never changes the exit code.
|
|
1210
|
+
"""
|
|
1211
|
+
token, chat = os.environ.get("MOX_TELEGRAM_TOKEN"), os.environ.get("MOX_TELEGRAM_CHAT")
|
|
1212
|
+
if token and chat:
|
|
1213
|
+
# MOX_TELEGRAM_API points at a self-hosted Bot API server, if you run one.
|
|
1214
|
+
api = os.environ.get("MOX_TELEGRAM_API", "https://api.telegram.org").rstrip("/")
|
|
1215
|
+
data = urllib.parse.urlencode({"chat_id": chat, "text": f"{title}\n{body}"}).encode()
|
|
1216
|
+
try:
|
|
1217
|
+
_opener.open(urllib.request.Request(f"{api}/bot{token}/sendMessage", data=data), timeout=15).close()
|
|
1218
|
+
except (OSError, http.client.HTTPException) as e:
|
|
1219
|
+
print(f"! Telegram notification failed: {reason(e)}", file=sys.stderr)
|
|
1220
|
+
for command in (["notify-send", "--app-name=mox", title, body],
|
|
1221
|
+
["termux-notification", "--title", title, "--content", body],
|
|
1222
|
+
["osascript", "-e", f"display notification {json.dumps(body)} with title {json.dumps(title)}"]):
|
|
1223
|
+
if shutil.which(command[0]):
|
|
1224
|
+
try:
|
|
1225
|
+
subprocess.run(command, capture_output=True, timeout=10)
|
|
1226
|
+
except (OSError, subprocess.SubprocessError):
|
|
1227
|
+
pass
|
|
1228
|
+
break
|
|
1229
|
+
|
|
1230
|
+
|
|
1231
|
+
def notify_summary(downloads: list[Download], outcome: str) -> None:
|
|
1232
|
+
done = [d for d in downloads if d.status == "done"]
|
|
1233
|
+
failed = [d for d in downloads if d.status == "error"]
|
|
1234
|
+
unfinished = [d for d in downloads if d.status in ("paused", "waiting")]
|
|
1235
|
+
if outcome == "deadline" and unfinished:
|
|
1236
|
+
title = f"mox stopped as scheduled, {len(unfinished)} unfinished"
|
|
1237
|
+
elif failed:
|
|
1238
|
+
title = f"mox: {len(done)} done, {len(failed)} failed"
|
|
1239
|
+
else:
|
|
1240
|
+
title = "mox: all downloads finished" if len(done) != 1 else "mox: download finished"
|
|
1241
|
+
names = [d.name for d in done + failed][:5]
|
|
1242
|
+
more = len(done) + len(failed) - len(names)
|
|
1243
|
+
notify(title, "\n".join(names) + (f"\nand {more} more" if more > 0 else ""))
|
|
1244
|
+
|
|
1245
|
+
|
|
1246
|
+
# ---------------------------------------------------------------- clipboard
|
|
1247
|
+
|
|
1248
|
+
URL_PATTERN = re.compile(r"https?://[^\s<>\"'`]+")
|
|
1249
|
+
PAGE_EXTENSIONS = {"", ".html", ".htm", ".php", ".asp", ".aspx", ".jsp", ".cgi"}
|
|
1250
|
+
CLIPBOARD_POLL = 1.0 # seconds between clipboard reads in --watch mode
|
|
1251
|
+
|
|
1252
|
+
|
|
1253
|
+
def clipboard_command() -> list[str] | None:
|
|
1254
|
+
custom = os.environ.get("MOX_CLIPBOARD_COMMAND")
|
|
1255
|
+
if custom:
|
|
1256
|
+
return shlex.split(custom)
|
|
1257
|
+
candidates = [["wl-paste", "--no-newline"]] if os.environ.get("WAYLAND_DISPLAY") else []
|
|
1258
|
+
candidates += [["xclip", "-o", "-selection", "clipboard"], ["xsel", "--clipboard", "--output"],
|
|
1259
|
+
["pbpaste"], ["termux-clipboard-get"],
|
|
1260
|
+
["powershell.exe", "-NoProfile", "-Command", "Get-Clipboard"]]
|
|
1261
|
+
return next((c for c in candidates if shutil.which(c[0])), None)
|
|
1262
|
+
|
|
1263
|
+
|
|
1264
|
+
def read_clipboard(command: list[str]) -> str:
|
|
1265
|
+
try:
|
|
1266
|
+
return subprocess.run(command, capture_output=True, text=True, timeout=5).stdout
|
|
1267
|
+
except (OSError, subprocess.SubprocessError):
|
|
1268
|
+
return ""
|
|
1269
|
+
|
|
1270
|
+
|
|
1271
|
+
def links_in(text: str) -> list[str]:
|
|
1272
|
+
"""File links in a piece of text. Links to web pages are skipped: copying the address of
|
|
1273
|
+
an article should not download its HTML."""
|
|
1274
|
+
links = []
|
|
1275
|
+
for url in URL_PATTERN.findall(text):
|
|
1276
|
+
url = url.rstrip(".,;:)]}>")
|
|
1277
|
+
name = urllib.parse.urlsplit(url).path.rsplit("/", 1)[-1]
|
|
1278
|
+
if os.path.splitext(name)[1].lower() not in PAGE_EXTENSIONS:
|
|
1279
|
+
links.append(url)
|
|
1280
|
+
return links
|
|
1281
|
+
|
|
1282
|
+
|
|
1283
|
+
# ---------------------------------------------------------------- running downloads
|
|
1284
|
+
|
|
1285
|
+
def run_session(downloads: list[Download], args, limiter: RateLimiter, deadline) -> str:
|
|
1286
|
+
"""Run every waiting download. Returns 'finished', 'interrupted' or 'deadline'."""
|
|
1287
|
+
_claimed_paths.clear()
|
|
1288
|
+
stop = threading.Event()
|
|
1289
|
+
display = Display(downloads)
|
|
1290
|
+
pool = ThreadPoolExecutor(max_workers=args.jobs)
|
|
1291
|
+
futures = [pool.submit(d.run, stop, limiter) for d in downloads if d.status == "waiting"]
|
|
1292
|
+
outcome = "finished"
|
|
1293
|
+
try:
|
|
1294
|
+
try:
|
|
1295
|
+
while not all(f.done() for f in futures):
|
|
1296
|
+
if deadline and outcome == "finished" and datetime.datetime.now() >= deadline:
|
|
1297
|
+
outcome = "deadline"
|
|
1298
|
+
stop.set()
|
|
1299
|
+
display.draw()
|
|
1300
|
+
time.sleep(FRAME_INTERVAL)
|
|
1301
|
+
except KeyboardInterrupt:
|
|
1302
|
+
outcome = "interrupted"
|
|
1303
|
+
stop.set()
|
|
1304
|
+
while not all(f.done() for f in futures):
|
|
1305
|
+
display.draw()
|
|
1306
|
+
time.sleep(0.1)
|
|
1307
|
+
pool.shutdown(wait=True)
|
|
1308
|
+
display.draw()
|
|
1309
|
+
finally:
|
|
1310
|
+
display.close()
|
|
1311
|
+
return outcome
|
|
1312
|
+
|
|
1313
|
+
|
|
1314
|
+
def watch_session(downloads: list[Download], args, limiter: RateLimiter, command: list[str]) -> str:
|
|
1315
|
+
"""Download what is given, then every new file link that shows up on the clipboard."""
|
|
1316
|
+
_claimed_paths.clear()
|
|
1317
|
+
seen = {d.url for d in downloads} | set(links_in(read_clipboard(command))) # not what was there before
|
|
1318
|
+
stop = threading.Event()
|
|
1319
|
+
display = Display(downloads, watching=True)
|
|
1320
|
+
pool = ThreadPoolExecutor(max_workers=args.jobs)
|
|
1321
|
+
futures = [pool.submit(d.run, stop, limiter) for d in downloads]
|
|
1322
|
+
announced: set[int] = set()
|
|
1323
|
+
last_poll = 0.0
|
|
1324
|
+
try:
|
|
1325
|
+
try:
|
|
1326
|
+
while True:
|
|
1327
|
+
if time.monotonic() - last_poll >= CLIPBOARD_POLL:
|
|
1328
|
+
last_poll = time.monotonic()
|
|
1329
|
+
for url in links_in(read_clipboard(command)):
|
|
1330
|
+
if url not in seen:
|
|
1331
|
+
seen.add(url)
|
|
1332
|
+
d = Download(url, args)
|
|
1333
|
+
downloads.append(d)
|
|
1334
|
+
futures.append(pool.submit(d.run, stop, limiter))
|
|
1335
|
+
if args.notify:
|
|
1336
|
+
for d in downloads:
|
|
1337
|
+
if d.status in ("done", "error") and id(d) not in announced:
|
|
1338
|
+
announced.add(id(d))
|
|
1339
|
+
notify("mox: download finished" if d.status == "done" else "mox: download failed",
|
|
1340
|
+
d.name if d.status == "done" else f"{d.name}: {d.message}")
|
|
1341
|
+
display.draw()
|
|
1342
|
+
time.sleep(FRAME_INTERVAL)
|
|
1343
|
+
except KeyboardInterrupt:
|
|
1344
|
+
stop.set()
|
|
1345
|
+
while not all(f.done() for f in futures):
|
|
1346
|
+
display.draw()
|
|
1347
|
+
time.sleep(0.1)
|
|
1348
|
+
pool.shutdown(wait=True)
|
|
1349
|
+
display.draw()
|
|
1350
|
+
finally:
|
|
1351
|
+
display.close()
|
|
1352
|
+
return "interrupted" if any(d.status in ("paused", "waiting") for d in downloads) else "finished"
|
|
1353
|
+
|
|
1354
|
+
|
|
1355
|
+
def expired_link_hint(d: Download) -> str:
|
|
1356
|
+
folder = os.path.dirname(d.path) if d.path else d.opts.dir
|
|
1357
|
+
return (" If this link worked before, it has probably expired. Get a fresh link and run\n"
|
|
1358
|
+
f" mox -d {shlex.quote(folder or '.')} -o {shlex.quote(d.name)} 'NEW_LINK'\n"
|
|
1359
|
+
" to continue from where it stopped.")
|
|
1360
|
+
|
|
1361
|
+
|
|
1362
|
+
def report(downloads: list[Download], outcome: str, args) -> int:
|
|
1363
|
+
failed = [d for d in downloads if d.status == "error"]
|
|
1364
|
+
unfinished = [d for d in downloads if d.status in ("paused", "waiting")]
|
|
1365
|
+
for d in failed:
|
|
1366
|
+
print(f"✖ {d.name}: {d.message}", file=sys.stderr)
|
|
1367
|
+
if d.error_status in EXPIRED_HTTP:
|
|
1368
|
+
print(expired_link_hint(d), file=sys.stderr)
|
|
1369
|
+
if unfinished and outcome == "deadline":
|
|
1370
|
+
print(f"‖ Stopped at {show_clock(args.until)} as scheduled with {len(unfinished)} download(s) unfinished. "
|
|
1371
|
+
"Run the same command again to continue.", file=sys.stderr)
|
|
1372
|
+
elif unfinished:
|
|
1373
|
+
print(f"‖ Paused {len(unfinished)} download(s). Run the same command again, "
|
|
1374
|
+
"or 'mox resume' in that folder, to continue.", file=sys.stderr)
|
|
1375
|
+
if outcome == "interrupted":
|
|
1376
|
+
return 130
|
|
1377
|
+
if failed:
|
|
1378
|
+
return 1
|
|
1379
|
+
return 3 if unfinished else 0
|
|
1380
|
+
|
|
1381
|
+
|
|
1382
|
+
def run_downloads(downloads: list[Download], args) -> int:
|
|
1383
|
+
"""Run downloads, on the --at/--until schedule if there is one, and report how it went."""
|
|
1384
|
+
limiter = RateLimiter(args.limit)
|
|
1385
|
+
while True:
|
|
1386
|
+
now = datetime.datetime.now()
|
|
1387
|
+
if args.at and not (args.until and in_window(now, args.at, args.until)):
|
|
1388
|
+
if not wait_until(next_time(args.at, now)):
|
|
1389
|
+
return 130
|
|
1390
|
+
deadline = next_time(args.until, datetime.datetime.now()) if args.until else None
|
|
1391
|
+
outcome = run_session(downloads, args, limiter, deadline)
|
|
1392
|
+
unfinished = [d for d in downloads if d.status in ("paused", "waiting")]
|
|
1393
|
+
if outcome != "deadline" or not unfinished or not args.at:
|
|
1394
|
+
break
|
|
1395
|
+
print(f"‖ Stopped at {show_clock(args.until)} as scheduled. Carrying on at {show_clock(args.at)}.",
|
|
1396
|
+
file=sys.stderr, flush=True)
|
|
1397
|
+
downloads = [d.restart() if d.status in ("paused", "waiting") else d for d in downloads]
|
|
1398
|
+
if args.notify and outcome != "interrupted":
|
|
1399
|
+
notify_summary(downloads, outcome)
|
|
1400
|
+
return report(downloads, outcome, args)
|
|
1401
|
+
|
|
1402
|
+
|
|
1403
|
+
# ---------------------------------------------------------------- mox resume
|
|
1404
|
+
|
|
1405
|
+
def resume_main(argv) -> int:
|
|
1406
|
+
parser = argparse.ArgumentParser(
|
|
1407
|
+
prog="mox resume", description="Continue every unfinished download in a folder.",
|
|
1408
|
+
epilog="mox remembers each unfinished download in a FILE.mox next to it, link included.")
|
|
1409
|
+
parser.add_argument("dir", nargs="?", default=".", help="folder to look in (default: current)")
|
|
1410
|
+
add_download_options(parser)
|
|
1411
|
+
args = parser.parse_args(argv)
|
|
1412
|
+
check_download_options(parser, args)
|
|
1413
|
+
configure_proxy(args.proxy)
|
|
1414
|
+
|
|
1415
|
+
downloads = []
|
|
1416
|
+
for entry in sorted(os.listdir(args.dir)) if os.path.isdir(args.dir) else []:
|
|
1417
|
+
if not entry.endswith(STATE_SUFFIX):
|
|
1418
|
+
continue
|
|
1419
|
+
try:
|
|
1420
|
+
with open(os.path.join(args.dir, entry), encoding="utf-8") as f:
|
|
1421
|
+
url = json.load(f)["url"]
|
|
1422
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
1423
|
+
print(f"! skipping {entry}: not a state file mox can read", file=sys.stderr)
|
|
1424
|
+
continue
|
|
1425
|
+
downloads.append(Download(url, args, name=entry[:-len(STATE_SUFFIX)]))
|
|
1426
|
+
if not downloads:
|
|
1427
|
+
print(f"No unfinished downloads in {args.dir}.", file=sys.stderr)
|
|
1428
|
+
return 0
|
|
1429
|
+
return run_downloads(downloads, args)
|
|
1430
|
+
|
|
1431
|
+
|
|
1432
|
+
# ---------------------------------------------------------------- mox bench
|
|
1433
|
+
|
|
1434
|
+
def bench_round(url: str, total: int | None, connections: int, seconds: float, args) -> tuple[float, bool, list[str]]:
|
|
1435
|
+
"""Download for `seconds` over `connections` connections, keeping nothing.
|
|
1436
|
+
|
|
1437
|
+
Returns (bytes per second, whether the round was too short to trust, why each refused
|
|
1438
|
+
connection was refused).
|
|
1439
|
+
"""
|
|
1440
|
+
received = [0]
|
|
1441
|
+
first_finish: list[tuple[float, int]] = [] # (time, bytes) when the first connection ran out of work
|
|
1442
|
+
refused: list[str] = []
|
|
1443
|
+
lock = threading.Lock()
|
|
1444
|
+
deadline = time.monotonic() + seconds
|
|
1445
|
+
|
|
1446
|
+
def fetch(start: int | None, end: int | None) -> None:
|
|
1447
|
+
link = KeepAlive(url, args.timeout) if start is not None and keep_alive_possible(url) else None
|
|
1448
|
+
try:
|
|
1449
|
+
if link:
|
|
1450
|
+
resp = link.get(build_headers(args.headers, start, end))
|
|
1451
|
+
if resp.status not in (200, 206):
|
|
1452
|
+
raise ServerBusy(resp.status)
|
|
1453
|
+
else:
|
|
1454
|
+
resp = request(url, args.headers, args.timeout, start, end)
|
|
1455
|
+
with resp:
|
|
1456
|
+
while time.monotonic() < deadline:
|
|
1457
|
+
data = resp.read1(CHUNK)
|
|
1458
|
+
if not data:
|
|
1459
|
+
# Only a connection that got through its whole share ends the window.
|
|
1460
|
+
# A refused one (Hetzner answers 429 past two connections) must not:
|
|
1461
|
+
# it would stop the clock at zero bytes.
|
|
1462
|
+
with lock:
|
|
1463
|
+
if not first_finish:
|
|
1464
|
+
first_finish.append((time.monotonic(), received[0]))
|
|
1465
|
+
break
|
|
1466
|
+
with lock:
|
|
1467
|
+
received[0] += len(data)
|
|
1468
|
+
except (RETRYABLE_ERRORS + (FatalError,)) as e:
|
|
1469
|
+
with lock:
|
|
1470
|
+
refused.append(reason(e))
|
|
1471
|
+
finally:
|
|
1472
|
+
if link:
|
|
1473
|
+
link.close()
|
|
1474
|
+
|
|
1475
|
+
if total:
|
|
1476
|
+
bounds = [(total * i // connections, total * (i + 1) // connections - 1) for i in range(connections)]
|
|
1477
|
+
else:
|
|
1478
|
+
bounds = [(None, None)]
|
|
1479
|
+
threads = [threading.Thread(target=fetch, args=b, daemon=True) for b in bounds]
|
|
1480
|
+
started = time.monotonic()
|
|
1481
|
+
for t in threads:
|
|
1482
|
+
t.start()
|
|
1483
|
+
tty = sys.stderr.isatty()
|
|
1484
|
+
label = f"{connections:>3} connection{'s' if connections > 1 else ' '}"
|
|
1485
|
+
while any(t.is_alive() for t in threads):
|
|
1486
|
+
time.sleep(FRAME_INTERVAL)
|
|
1487
|
+
if tty:
|
|
1488
|
+
elapsed = time.monotonic() - started
|
|
1489
|
+
sys.stderr.write(f"\r\x1b[2K{label} {human_size(received[0] / elapsed) + '/s':>11} "
|
|
1490
|
+
f"\x1b[2m{max(0.0, seconds - elapsed):.0f}s left\x1b[0m")
|
|
1491
|
+
sys.stderr.flush()
|
|
1492
|
+
if tty:
|
|
1493
|
+
sys.stderr.write("\r\x1b[2K")
|
|
1494
|
+
# Count only while every connection was busy. Once the fast ones finish their share, the
|
|
1495
|
+
# slow stragglers alone would drag the average down and hide what N connections can do.
|
|
1496
|
+
ended, got = first_finish[0] if first_finish else (time.monotonic(), received[0])
|
|
1497
|
+
window = max(ended - started, 1e-3)
|
|
1498
|
+
return got / window, window < min(1.0, seconds / 2), refused
|
|
1499
|
+
|
|
1500
|
+
|
|
1501
|
+
def bench_main(argv) -> int:
|
|
1502
|
+
parser = argparse.ArgumentParser(
|
|
1503
|
+
prog="mox bench", description="Measure how much faster many connections are than one. "
|
|
1504
|
+
"Nothing is saved to disk.")
|
|
1505
|
+
parser.add_argument("url", metavar="URL")
|
|
1506
|
+
parser.add_argument("-s", "--seconds", type=float, default=5, help="length of each round (default: 5)")
|
|
1507
|
+
parser.add_argument("-c", "--connections", type=int, default=DEFAULT_CONNECTIONS,
|
|
1508
|
+
help=f"most connections to try (default: {DEFAULT_CONNECTIONS})")
|
|
1509
|
+
add_network_options(parser)
|
|
1510
|
+
args = parser.parse_args(argv)
|
|
1511
|
+
if not 1 <= args.connections <= MAX_CONNECTIONS:
|
|
1512
|
+
parser.error(f"--connections must be between 1 and {MAX_CONNECTIONS}")
|
|
1513
|
+
check_network_options(parser, args)
|
|
1514
|
+
configure_proxy(args.proxy)
|
|
1515
|
+
url = args.url if "://" in args.url else "https://" + args.url
|
|
1516
|
+
|
|
1517
|
+
try:
|
|
1518
|
+
final, total, ranges, name, _ = probe(url, args.headers, args.timeout)
|
|
1519
|
+
except (RETRYABLE_ERRORS + (FatalError,)) as e:
|
|
1520
|
+
print(f"✖ {reason(e)}", file=sys.stderr)
|
|
1521
|
+
return 1
|
|
1522
|
+
support = "supports range requests" if ranges else "does not support range requests"
|
|
1523
|
+
print(f"{name} · {human_size(total)} · the server {support}")
|
|
1524
|
+
levels = sorted({1, min(4, args.connections), args.connections}) if ranges else [1]
|
|
1525
|
+
rates: dict[int, float] = {}
|
|
1526
|
+
allowed: int | None = None # how many connections the server accepted, when it refused some
|
|
1527
|
+
rough = False
|
|
1528
|
+
try:
|
|
1529
|
+
for connections in levels:
|
|
1530
|
+
rate, short, refused = bench_round(final, total if ranges else None, connections, args.seconds, args)
|
|
1531
|
+
rates[connections] = rate
|
|
1532
|
+
rough = rough or short
|
|
1533
|
+
gain = f" {rate / rates[1]:4.1f}×" if connections > 1 and rates[1] else ""
|
|
1534
|
+
label = f"{connections:>3} connection{'s' if connections > 1 else ' '}"
|
|
1535
|
+
note = f" \x1b[2m{len(refused)} refused: {refused[0]}\x1b[0m" if refused and sys.stdout.isatty() \
|
|
1536
|
+
else (f" {len(refused)} refused: {refused[0]}" if refused else "")
|
|
1537
|
+
print(f"{label} {human_size(rate) + '/s':>11}{gain}{note}", flush=True)
|
|
1538
|
+
if refused and allowed is None:
|
|
1539
|
+
allowed = connections - len(refused)
|
|
1540
|
+
except KeyboardInterrupt:
|
|
1541
|
+
return 130
|
|
1542
|
+
|
|
1543
|
+
best = max(rates, key=rates.get)
|
|
1544
|
+
gain = rates[best] / rates[1] if rates[1] else 0
|
|
1545
|
+
if not ranges:
|
|
1546
|
+
print("This server sends files in one piece, so extra connections cannot help here.")
|
|
1547
|
+
elif allowed is not None:
|
|
1548
|
+
print(f"This server accepts only about {max(allowed, 1)} connections at once and refuses the rest. "
|
|
1549
|
+
f"mox backs off by itself, but -c {max(allowed, 1)} avoids the refusals.")
|
|
1550
|
+
elif gain >= 1.5:
|
|
1551
|
+
print(f"This server limits each connection. With -c {best}, mox is about {gain:.0f}× faster here "
|
|
1552
|
+
"than a one-connection download such as wget or curl.")
|
|
1553
|
+
else:
|
|
1554
|
+
print("Extra connections barely help here: your line or the server's total is the limit. "
|
|
1555
|
+
"A few connections, -c 4, are plenty.")
|
|
1556
|
+
if rough:
|
|
1557
|
+
print("The file is small enough to finish within a round, so these numbers are rough.")
|
|
1558
|
+
return 0
|
|
1559
|
+
|
|
1560
|
+
|
|
1561
|
+
# ---------------------------------------------------------------- library
|
|
1562
|
+
|
|
1563
|
+
class DownloadError(Exception):
|
|
1564
|
+
"""Raised by download() when a file could not be downloaded."""
|
|
1565
|
+
|
|
1566
|
+
|
|
1567
|
+
def download(url: str, directory: str = ".", *, name: str | None = None,
|
|
1568
|
+
connections: int = DEFAULT_CONNECTIONS, limit: int | str | None = None, retries: int = 10,
|
|
1569
|
+
timeout: float = 30, headers: dict | None = None, sha256: str | None = None,
|
|
1570
|
+
proxy: str | None = None, progress=None) -> str:
|
|
1571
|
+
"""Download one URL over many connections and return the path of the saved file.
|
|
1572
|
+
|
|
1573
|
+
It resumes a matching unfinished download in `directory`, just like the command does.
|
|
1574
|
+
`limit` is bytes per second, or text such as "2M". `progress`, if given, is called about
|
|
1575
|
+
four times a second with (bytes done, total bytes or None). `proxy` applies to the whole
|
|
1576
|
+
process, since all requests share one opener. Raises DownloadError on failure.
|
|
1577
|
+
"""
|
|
1578
|
+
if not 1 <= connections <= MAX_CONNECTIONS:
|
|
1579
|
+
raise ValueError(f"connections must be between 1 and {MAX_CONNECTIONS}")
|
|
1580
|
+
if isinstance(limit, str):
|
|
1581
|
+
try:
|
|
1582
|
+
limit = parse_rate(limit)
|
|
1583
|
+
except argparse.ArgumentTypeError as e:
|
|
1584
|
+
raise ValueError(str(e)) from None
|
|
1585
|
+
configure_proxy(proxy)
|
|
1586
|
+
opts = argparse.Namespace(dir=directory, connections=connections, retries=retries,
|
|
1587
|
+
timeout=timeout, headers=dict(headers or {}), limit=limit)
|
|
1588
|
+
d = Download(url if "://" in url else "https://" + url, opts, name=name, sha256=sha256)
|
|
1589
|
+
stop = threading.Event()
|
|
1590
|
+
worker = threading.Thread(target=d.run, args=(stop, RateLimiter(limit)), daemon=True)
|
|
1591
|
+
worker.start()
|
|
1592
|
+
try:
|
|
1593
|
+
while worker.is_alive():
|
|
1594
|
+
worker.join(FRAME_INTERVAL)
|
|
1595
|
+
if progress:
|
|
1596
|
+
progress(d.done, d.total)
|
|
1597
|
+
except BaseException:
|
|
1598
|
+
stop.set()
|
|
1599
|
+
worker.join()
|
|
1600
|
+
raise
|
|
1601
|
+
finally:
|
|
1602
|
+
if d.path:
|
|
1603
|
+
with _claim_lock:
|
|
1604
|
+
_claimed_paths.discard(os.path.abspath(d.path))
|
|
1605
|
+
if d.status != "done":
|
|
1606
|
+
raise DownloadError(f"{d.name}: {d.message or d.status}")
|
|
1607
|
+
return d.path
|
|
1608
|
+
|
|
1609
|
+
|
|
1610
|
+
# ---------------------------------------------------------------- main
|
|
1611
|
+
|
|
1612
|
+
def main(argv=None) -> int:
|
|
1613
|
+
argv = sys.argv[1:] if argv is None else list(argv)
|
|
1614
|
+
if argv and argv[0] == "resume":
|
|
1615
|
+
return resume_main(argv[1:])
|
|
1616
|
+
if argv and argv[0] == "bench":
|
|
1617
|
+
return bench_main(argv[1:])
|
|
1618
|
+
args = parse_args(argv)
|
|
1619
|
+
configure_proxy(args.proxy)
|
|
1620
|
+
downloads = [Download(url, args, name=args.output, sha256=args.sha256) for url in args.urls]
|
|
1621
|
+
if not args.watch:
|
|
1622
|
+
return run_downloads(downloads, args)
|
|
1623
|
+
command = clipboard_command()
|
|
1624
|
+
if not command:
|
|
1625
|
+
print("✖ --watch needs a clipboard tool: wl-clipboard on Wayland, xclip or xsel on X11, "
|
|
1626
|
+
"termux-api on Termux. Or set MOX_CLIPBOARD_COMMAND.", file=sys.stderr)
|
|
1627
|
+
return 2
|
|
1628
|
+
outcome = watch_session(downloads, args, RateLimiter(args.limit), command)
|
|
1629
|
+
return report(downloads, outcome, args)
|
|
1630
|
+
|
|
1631
|
+
|
|
1632
|
+
if __name__ == "__main__":
|
|
1633
|
+
sys.exit(main())
|