moxdl 1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
mox.py ADDED
@@ -0,0 +1,1633 @@
1
+ #!/usr/bin/env python3
2
+ """mox: a fast, resumable, multi-connection download manager for the terminal.
3
+
4
+ Uses only the Python standard library.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import datetime
11
+ import functools
12
+ import hashlib
13
+ import http.client
14
+ import json
15
+ import os
16
+ import re
17
+ import shlex
18
+ import shutil
19
+ import socket
20
+ import ssl
21
+ import subprocess
22
+ import sys
23
+ import threading
24
+ import time
25
+ import unicodedata
26
+ import urllib.error
27
+ import urllib.parse
28
+ import urllib.request
29
+ from collections import deque
30
+ from concurrent.futures import ThreadPoolExecutor
31
+
32
+ VERSION = "1.4"
33
+ CHUNK = 256 * 1024
34
+ MIN_SPLIT = 256 * 1024 # smallest piece when first dividing a file between connections
35
+ MIN_STEAL = 64 * 1024 # smallest piece an idle connection takes over from a busy one
36
+ MIN_STEAL_ETA = 1.0 # seconds: only take over work from a piece that would take longer than this
37
+ IDLE_POLL = 0.2 # seconds an idle connection waits before looking for work again
38
+ MIN_ACTIVE = 2 # a busy server never makes a download drop below this many connections
39
+ DEFAULT_CONNECTIONS = 16
40
+ MAX_CONNECTIONS = 64
41
+ STATE_SUFFIX = ".mox"
42
+ PART_SUFFIX = ".part"
43
+ SAVE_INTERVAL = 1.0
44
+ FRAME_INTERVAL = 0.25
45
+ SPEED_WINDOW = 0.5 # seconds between updates of each piece's measured speed
46
+ MIN_MEASURE = 0.3 # seconds a request must run before its speed is trusted
47
+ STALL_TIME = 5.0 # seconds without a single byte before a request counts as stalled
48
+ SLOW_WINDOW = 1.5 # seconds between checks of a connection's speed
49
+ SLOW_RATIO = 0.3 # slower than this share of the average connection counts as slow
50
+ SLOW_RECONNECTS = 3 # reconnect a slow connection at most this many times per piece
51
+ RETRYABLE_HTTP = {408, 425, 429, 500, 502, 503, 504}
52
+ BUSY_HTTP = {429, 503} # the server wants fewer connections
53
+ EXPIRED_HTTP = {401, 403, 410} # what an expired signed link usually returns
54
+ RETRYABLE_ERRORS = (urllib.error.URLError, OSError, http.client.HTTPException)
55
+
56
+
57
+ class FatalError(Exception):
58
+ """A failure that retrying will not fix."""
59
+
60
+ def __init__(self, message: str, status: int | None = None):
61
+ super().__init__(message)
62
+ self.status = status
63
+
64
+
65
+ class ServerBusy(ConnectionError):
66
+ """A retryable HTTP status (5xx, 429) on a keep-alive connection."""
67
+
68
+ def __init__(self, status: int):
69
+ super().__init__(f"HTTP {status}")
70
+ self.status = status
71
+
72
+
73
+ class SlowConnection(Exception):
74
+ """This connection is much slower than its siblings; reconnecting may get a better route."""
75
+
76
+
77
+ class Retire(Exception):
78
+ """The server limits connections: this worker steps aside and leaves its piece to the others."""
79
+
80
+
81
+ # ---------------------------------------------------------------- formatting
82
+
83
+ def human_size(n: float | None) -> str:
84
+ if n is None:
85
+ return "?"
86
+ for unit in ("B", "KB", "MB", "GB", "TB"):
87
+ if abs(n) < 1024 or unit == "TB":
88
+ return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
89
+ n /= 1024
90
+ return "?"
91
+
92
+
93
+ def human_time(seconds: float | None) -> str:
94
+ if seconds is None:
95
+ return "--:--"
96
+ seconds = int(seconds)
97
+ hours, rest = divmod(seconds, 3600)
98
+ minutes, seconds = divmod(rest, 60)
99
+ return f"{hours}:{minutes:02}:{seconds:02}" if hours else f"{minutes:02}:{seconds:02}"
100
+
101
+
102
+ def parse_rate(text: str) -> int:
103
+ match = re.fullmatch(r"\s*(\d+(?:\.\d+)?)\s*([kmg]?)b?\s*", text, re.I)
104
+ if not match:
105
+ raise argparse.ArgumentTypeError(f"invalid rate {text!r} (examples: 500K, 2M)")
106
+ multiplier = {"": 1, "k": 1024, "m": 1024**2, "g": 1024**3}[match.group(2).lower()]
107
+ return int(float(match.group(1)) * multiplier)
108
+
109
+
110
+ def char_width(ch: str) -> int:
111
+ if unicodedata.combining(ch):
112
+ return 0
113
+ return 2 if unicodedata.east_asian_width(ch) in "WF" else 1
114
+
115
+
116
+ def clip(text: str, width: int) -> str:
117
+ """Cut or pad text to exactly `width` terminal columns."""
118
+ out, used = [], 0
119
+ for ch in text:
120
+ w = char_width(ch)
121
+ if used + w > width:
122
+ if width >= 1 and out:
123
+ while used > width - 1:
124
+ used -= char_width(out.pop())
125
+ out.append("…")
126
+ used += 1
127
+ break
128
+ out.append(ch)
129
+ used += w
130
+ return "".join(out) + " " * (width - used)
131
+
132
+
133
+ # ---------------------------------------------------------------- HTTP
134
+
135
+ def build_headers(extra, start=None, end=None) -> dict:
136
+ headers = {"User-Agent": f"mox/{VERSION}", "Accept-Encoding": "identity", **extra}
137
+ if start is not None:
138
+ headers["Range"] = f"bytes={start}-" + ("" if end is None else str(end))
139
+ return headers
140
+
141
+
142
+ class Proxy:
143
+ """A proxy given with --proxy: socks5://, socks5h:// or http://, optionally with user:password@."""
144
+
145
+ SCHEMES = ("socks5", "socks5h", "http")
146
+
147
+ def __init__(self, url: str):
148
+ parts = urllib.parse.urlsplit(url)
149
+ if parts.scheme not in self.SCHEMES or not parts.hostname:
150
+ raise ValueError(f"unsupported proxy {url!r}; use socks5://host:port or http://host:port")
151
+ self.url = url
152
+ self.socks = parts.scheme.startswith("socks")
153
+ self.host = parts.hostname
154
+ self.port = parts.port or (1080 if self.socks else 8080)
155
+ self.username = urllib.parse.unquote(parts.username) if parts.username else None
156
+ self.password = urllib.parse.unquote(parts.password) if parts.password else ""
157
+
158
+
159
+ _proxy: Proxy | None = None
160
+ _opener = urllib.request.build_opener() # honours http_proxy / https_proxy from the environment
161
+
162
+ SOCKS_ERRORS = {
163
+ 1: "general failure", 2: "connection not allowed by ruleset", 3: "network unreachable",
164
+ 4: "host unreachable", 5: "connection refused", 6: "TTL expired",
165
+ 7: "command not supported", 8: "address type not supported",
166
+ }
167
+
168
+
169
+ def _recv_exact(sock: socket.socket, n: int) -> bytes:
170
+ data = b""
171
+ while len(data) < n:
172
+ chunk = sock.recv(n - len(data))
173
+ if not chunk:
174
+ raise ConnectionError("SOCKS proxy closed the connection")
175
+ data += chunk
176
+ return data
177
+
178
+
179
+ def socks5_connect(proxy: Proxy, host: str, port: int, timeout) -> socket.socket:
180
+ """Open a TCP connection to host:port through a SOCKS5 proxy (RFC 1928, auth per RFC 1929).
181
+
182
+ The host name is sent to the proxy unresolved, so DNS happens on the proxy's side.
183
+ """
184
+ timeout = timeout if isinstance(timeout, (int, float)) else None
185
+ sock = socket.create_connection((proxy.host, proxy.port), timeout)
186
+ try:
187
+ methods = b"\x00\x02" if proxy.username else b"\x00"
188
+ sock.sendall(b"\x05" + bytes([len(methods)]) + methods)
189
+ version, method = _recv_exact(sock, 2)
190
+ if version != 5 or method == 0xFF:
191
+ raise FatalError("SOCKS proxy refused every login method we offered")
192
+ if method == 2:
193
+ user, password = proxy.username.encode(), proxy.password.encode()
194
+ sock.sendall(b"\x01" + bytes([len(user)]) + user + bytes([len(password)]) + password)
195
+ if _recv_exact(sock, 2)[1] != 0:
196
+ raise FatalError("SOCKS proxy rejected the user name or password")
197
+ name = host.encode("idna")
198
+ sock.sendall(b"\x05\x01\x00\x03" + bytes([len(name)]) + name + port.to_bytes(2, "big"))
199
+ _, status, _, address_type = _recv_exact(sock, 4)
200
+ if status != 0:
201
+ raise ConnectionError(f"SOCKS proxy: {SOCKS_ERRORS.get(status, f'error {status}')}")
202
+ _recv_exact(sock, {1: 4, 4: 16}.get(address_type) or _recv_exact(sock, 1)[0])
203
+ _recv_exact(sock, 2) # bound port
204
+ return sock
205
+ except BaseException:
206
+ sock.close()
207
+ raise
208
+
209
+
210
+ class _SocksMixin:
211
+ def __init__(self, *args, proxy: Proxy, **kwargs):
212
+ super().__init__(*args, **kwargs)
213
+ self._create_connection = lambda address, timeout=None, source_address=None: \
214
+ socks5_connect(proxy, address[0], address[1], timeout)
215
+
216
+
217
+ class SocksHTTPConnection(_SocksMixin, http.client.HTTPConnection):
218
+ pass
219
+
220
+
221
+ class SocksHTTPSConnection(_SocksMixin, http.client.HTTPSConnection):
222
+ pass
223
+
224
+
225
+ class _SocksHTTPHandler(urllib.request.HTTPHandler):
226
+ def __init__(self, proxy: Proxy):
227
+ super().__init__()
228
+ self.proxy = proxy
229
+
230
+ def http_open(self, req):
231
+ return self.do_open(functools.partial(SocksHTTPConnection, proxy=self.proxy), req)
232
+
233
+
234
+ class _SocksHTTPSHandler(urllib.request.HTTPSHandler):
235
+ def __init__(self, proxy: Proxy):
236
+ super().__init__()
237
+ self.proxy = proxy
238
+
239
+ def https_open(self, req):
240
+ return self.do_open(functools.partial(SocksHTTPSConnection, proxy=self.proxy), req, context=self._context)
241
+
242
+
243
+ def configure_proxy(url: str | None) -> None:
244
+ """Route every request through --proxy. Without it, http_proxy / https_proxy still apply."""
245
+ global _proxy, _opener
246
+ if not url:
247
+ _proxy, _opener = None, urllib.request.build_opener()
248
+ return
249
+ _proxy = Proxy(url)
250
+ if _proxy.socks:
251
+ _opener = urllib.request.build_opener(urllib.request.ProxyHandler({}),
252
+ _SocksHTTPHandler(_proxy), _SocksHTTPSHandler(_proxy))
253
+ else:
254
+ _opener = urllib.request.build_opener(urllib.request.ProxyHandler({"http": url, "https": url}))
255
+
256
+
257
+ def request(url, headers, timeout, start=None, end=None):
258
+ req = urllib.request.Request(url, headers=build_headers(headers, start, end))
259
+ try:
260
+ return _opener.open(req, timeout=timeout)
261
+ except urllib.error.HTTPError as e:
262
+ if e.code in RETRYABLE_HTTP:
263
+ raise
264
+ e.close() # the error carries the open response; we are done with it
265
+ raise FatalError(f"HTTP {e.code} {e.reason}", e.code) from None
266
+
267
+
268
+ def keep_alive_possible(url: str) -> bool:
269
+ """Our own persistent connections work directly or through SOCKS, but not through an HTTP proxy."""
270
+ if _proxy:
271
+ return _proxy.socks
272
+ parts = urllib.parse.urlsplit(url)
273
+ return not (parts.scheme in urllib.request.getproxies()
274
+ and not urllib.request.proxy_bypass(parts.hostname or ""))
275
+
276
+
277
+ class KeepAlive:
278
+ """One worker's persistent connection, so each new piece skips the TCP and TLS handshakes."""
279
+
280
+ def __init__(self, url: str, timeout: float):
281
+ parts = urllib.parse.urlsplit(url)
282
+ self.https = parts.scheme == "https"
283
+ self.host, self.port = parts.hostname, parts.port
284
+ self.target = (parts.path or "/") + (f"?{parts.query}" if parts.query else "")
285
+ self.timeout = timeout
286
+ self.conn: http.client.HTTPConnection | None = None
287
+
288
+ def _connect(self) -> http.client.HTTPConnection:
289
+ extra = {"proxy": _proxy} if _proxy else {}
290
+ if self.https:
291
+ cls = SocksHTTPSConnection if _proxy else http.client.HTTPSConnection
292
+ return cls(self.host, self.port, timeout=self.timeout, context=ssl.create_default_context(), **extra)
293
+ cls = SocksHTTPConnection if _proxy else http.client.HTTPConnection
294
+ return cls(self.host, self.port, timeout=self.timeout, **extra)
295
+
296
+ def get(self, headers: dict) -> http.client.HTTPResponse:
297
+ reused = self.conn is not None
298
+ while True:
299
+ if self.conn is None:
300
+ self.conn = self._connect()
301
+ try:
302
+ self.conn.request("GET", self.target, headers=headers)
303
+ return self.conn.getresponse()
304
+ except (OSError, http.client.HTTPException):
305
+ self.close()
306
+ if not reused:
307
+ raise
308
+ reused = False # the server dropped the idle connection; one fresh try
309
+
310
+ def close(self) -> None:
311
+ if self.conn is not None:
312
+ self.conn.close()
313
+ self.conn = None
314
+
315
+
316
+ def safe_name(name: str) -> str:
317
+ name = name.replace("\\", "/").rsplit("/", 1)[-1]
318
+ name = re.sub(r'[\x00-\x1f<>:"|?*]', "_", name).strip(" .")
319
+ return name[:200]
320
+
321
+
322
+ def filename_from(resp, url: str) -> str:
323
+ disposition = resp.headers.get("Content-Disposition", "")
324
+ name = ""
325
+ encoded = re.search(r"filename\*\s*=\s*([^']*)'[^']*'([^;]+)", disposition, re.I)
326
+ if encoded:
327
+ try:
328
+ name = urllib.parse.unquote(encoded.group(2).strip().strip('"'),
329
+ encoding=encoded.group(1) or "utf-8", errors="replace")
330
+ except LookupError:
331
+ name = urllib.parse.unquote(encoded.group(2).strip().strip('"'))
332
+ else:
333
+ plain = re.search(r'filename\s*=\s*(?:"([^"]+)"|([^;]+))', disposition, re.I)
334
+ if plain:
335
+ name = (plain.group(1) or plain.group(2)).strip()
336
+ if not name:
337
+ name = urllib.parse.unquote(urllib.parse.urlsplit(url).path.rsplit("/", 1)[-1])
338
+ return safe_name(name) or "download"
339
+
340
+
341
+ def validator_from(resp) -> str | None:
342
+ """What identifies this version of the file: a strong ETag, or failing that Last-Modified.
343
+
344
+ Weak ETags (W/"...") are useless here, since If-Range never matches them.
345
+ """
346
+ etag = resp.headers.get("ETag", "").strip()
347
+ if etag.startswith('"'):
348
+ return etag
349
+ return resp.headers.get("Last-Modified", "").strip() or None
350
+
351
+
352
+ def probe(url, headers, timeout):
353
+ """Return (final_url, total_size_or_None, supports_ranges, filename, validator_or_None)."""
354
+ try:
355
+ resp = request(url, headers, timeout, start=0)
356
+ except FatalError as e:
357
+ if e.status != 416: # servers answer "bytes=0-" on an empty file with 416
358
+ raise
359
+ resp = request(url, headers, timeout)
360
+ with resp:
361
+ final = resp.geturl()
362
+ name = filename_from(resp, final)
363
+ validator = validator_from(resp)
364
+ if resp.status == 206:
365
+ match = re.match(r"bytes\s+\d+-\d+/(\d+)", resp.headers.get("Content-Range", ""))
366
+ if match:
367
+ return final, int(match.group(1)), True, name, validator
368
+ length = resp.headers.get("Content-Length", "")
369
+ total = int(length) if length.isdigit() and resp.status == 200 else None
370
+ return final, total, False, name, validator
371
+
372
+
373
+ if hasattr(os, "pwrite"):
374
+ def write_at(fd: int, data: bytes, pos: int) -> None:
375
+ view = memoryview(data)
376
+ while view:
377
+ written = os.pwrite(fd, view, pos)
378
+ view, pos = view[written:], pos + written
379
+ else: # Windows
380
+ _write_lock = threading.Lock()
381
+
382
+ def write_at(fd: int, data: bytes, pos: int) -> None:
383
+ with _write_lock:
384
+ os.lseek(fd, pos, os.SEEK_SET)
385
+ os.write(fd, data)
386
+
387
+
388
+ _claimed_paths: set[str] = set() # paths taken by downloads in this run
389
+ _claim_lock = threading.Lock()
390
+
391
+
392
+ class RateLimiter:
393
+ """Token bucket shared by every connection of every download."""
394
+
395
+ def __init__(self, rate: int | None):
396
+ self.rate = rate
397
+ self.capacity = max((rate or 0) // 4, CHUNK) # small bucket: no big burst at start
398
+ self.tokens = float(self.capacity)
399
+ self.last = time.monotonic()
400
+ self.lock = threading.Lock()
401
+
402
+ def consume(self, n: int) -> None:
403
+ if not self.rate:
404
+ return
405
+ while True:
406
+ with self.lock:
407
+ now = time.monotonic()
408
+ self.tokens = min(self.capacity, self.tokens + (now - self.last) * self.rate)
409
+ self.last = now
410
+ if self.tokens >= n:
411
+ self.tokens -= n
412
+ return
413
+ wait = (n - self.tokens) / self.rate
414
+ time.sleep(wait)
415
+
416
+
417
+ # ---------------------------------------------------------------- download
418
+
419
+ class Download:
420
+ def __init__(self, url: str, opts, name: str | None = None, sha256: str | None = None):
421
+ self.url = self.final_url = url
422
+ self.opts = opts
423
+ self.custom_name = name
424
+ self.name = name or safe_name(urllib.parse.unquote(urllib.parse.urlsplit(url).path)) or url
425
+ self.sha256 = sha256.lower() if sha256 else None
426
+ self.status = "waiting"
427
+ self.message = ""
428
+ self.error_status: int | None = None
429
+ self.total: int | None = None
430
+ self.done = 0
431
+ self.ranges = False
432
+ self.validator: str | None = None
433
+ self.active = 0
434
+ self.path: str | None = None
435
+ self.segments: list[dict] = []
436
+ self.lock = threading.Lock()
437
+ self.failed = threading.Event()
438
+ self.samples: deque = deque()
439
+ self.retired = 0
440
+ self.work_changed = threading.Condition(self.lock) # signalled whenever a piece finishes
441
+
442
+ # -- public
443
+
444
+ def restart(self) -> Download:
445
+ """A fresh copy that picks up from the saved state file, for the next scheduled session."""
446
+ return Download(self.url, self.opts, self.custom_name, self.sha256)
447
+
448
+ def run(self, stop: threading.Event, limiter: RateLimiter) -> None:
449
+ if stop.is_set():
450
+ return
451
+ try:
452
+ self._run(stop, limiter)
453
+ except FatalError as e:
454
+ self.status, self.message, self.error_status = "error", str(e), e.status
455
+ except Exception as e: # keep one broken download from killing the rest
456
+ self.status, self.message = "error", f"{type(e).__name__}: {e}"
457
+
458
+ def pieces(self) -> list[tuple[int, int]]:
459
+ """(start, bytes done) for every piece, for drawing the map."""
460
+ with self.lock:
461
+ return [(s["start"], s["done"]) for s in self.segments]
462
+
463
+ def speed(self) -> float:
464
+ now = time.monotonic()
465
+ self.samples.append((now, self.done))
466
+ while len(self.samples) > 2 and now - self.samples[0][0] > 4:
467
+ self.samples.popleft()
468
+ then, done_then = self.samples[0]
469
+ return (self.done - done_then) / (now - then) if now - then > 0.5 else 0.0
470
+
471
+ # -- orchestration
472
+
473
+ def _run(self, stop, limiter):
474
+ self.status = "connecting"
475
+ info = self._probe(stop)
476
+ if info is None:
477
+ self.status = "paused"
478
+ return
479
+ self.final_url, self.total, self.ranges, server_name, self.validator = info
480
+ if not self.custom_name:
481
+ self.name = server_name
482
+ self.path = self._choose_path()
483
+ self.name = os.path.basename(self.path)
484
+ if self.status == "done":
485
+ return
486
+
487
+ state_path, part_path = self.path + STATE_SUFFIX, self.path + PART_SUFFIX
488
+ resumed = self._load_state(state_path, part_path)
489
+ self.segments = resumed or self._plan()
490
+ self.done = sum(s["done"] for s in self.segments)
491
+ if resumed:
492
+ self.message = f"resumed at {human_size(self.done)}"
493
+
494
+ with open(part_path, "r+b" if resumed else "wb", buffering=0) as f:
495
+ if not resumed and self.total:
496
+ f.truncate(self.total)
497
+ self.status = "downloading"
498
+ connections = self.opts.connections if self.ranges else 1
499
+ workers = [threading.Thread(target=self._worker, args=(f.fileno(), stop, limiter), daemon=True)
500
+ for _ in range(connections)]
501
+ for w in workers:
502
+ w.start()
503
+ while alive := [w for w in workers if w.is_alive()]:
504
+ alive[0].join(SAVE_INTERVAL)
505
+ self._save_state(state_path)
506
+ if self.total is None and not self.failed.is_set() and not stop.is_set():
507
+ f.truncate(self.done)
508
+
509
+ if self.failed.is_set():
510
+ self._save_state(state_path)
511
+ self.status = "error"
512
+ return
513
+ if stop.is_set():
514
+ self._save_state(state_path)
515
+ self.status = "paused"
516
+ return
517
+ if self.total is not None and self.done != self.total:
518
+ raise FatalError(f"incomplete: got {self.done} of {self.total} bytes")
519
+
520
+ if self.sha256:
521
+ self.status = "verifying"
522
+ digest = sha256_file(part_path)
523
+ if digest != self.sha256:
524
+ self._remove(state_path)
525
+ raise FatalError(f"SHA-256 mismatch (got {digest[:16]}…), kept {os.path.basename(part_path)}")
526
+ self.message = "SHA-256 verified"
527
+ os.replace(part_path, self.path)
528
+ self._remove(state_path)
529
+ self.status = "done"
530
+
531
+ def _probe(self, stop):
532
+ attempts = 0
533
+ while not stop.is_set():
534
+ try:
535
+ return probe(self.url, self.opts.headers, self.opts.timeout)
536
+ except RETRYABLE_ERRORS as e:
537
+ attempts += 1
538
+ if attempts > self.opts.retries:
539
+ raise FatalError(f"cannot connect: {reason(e)}") from None
540
+ self.message = f"retry {attempts}: {reason(e)}"
541
+ stop.wait(min(2**attempts, 30))
542
+ return None
543
+
544
+ def _choose_path(self) -> str:
545
+ os.makedirs(self.opts.dir, exist_ok=True)
546
+ path = os.path.join(self.opts.dir, self.name)
547
+ base, ext = os.path.splitext(path)
548
+ counter = 1
549
+ with _claim_lock:
550
+ while True:
551
+ if os.path.abspath(path) not in _claimed_paths:
552
+ if os.path.exists(path + STATE_SUFFIX) or not os.path.exists(path):
553
+ break
554
+ if self.total is not None and os.path.isfile(path) and os.path.getsize(path) == self.total:
555
+ self.status, self.message, self.done = "done", "already downloaded", self.total
556
+ break
557
+ path = f"{base} ({counter}){ext}"
558
+ counter += 1
559
+ _claimed_paths.add(os.path.abspath(path))
560
+ return path
561
+
562
+ def _plan(self) -> list[dict]:
563
+ if self.total == 0:
564
+ return []
565
+ if not self.ranges or self.total is None:
566
+ end = None if self.total is None else self.total - 1
567
+ return [{"start": 0, "end": end, "done": 0}]
568
+ count = max(1, min(self.opts.connections, self.total // MIN_SPLIT))
569
+ size = self.total // count
570
+ return [{"start": i * size,
571
+ "end": self.total - 1 if i == count - 1 else (i + 1) * size - 1,
572
+ "done": 0} for i in range(count)]
573
+
574
+ # -- state file (lets an interrupted download resume)
575
+
576
+ def _load_state(self, state_path, part_path) -> list[dict] | None:
577
+ if not (self.ranges and os.path.exists(state_path) and os.path.exists(part_path)):
578
+ return None
579
+ try:
580
+ with open(state_path, encoding="utf-8") as f:
581
+ state = json.load(f)
582
+ if state.get("total") != self.total or os.path.getsize(part_path) != self.total:
583
+ return None
584
+ # Same size is not proof of the same file: a new release can keep the size.
585
+ # Without a validator on either side, the size check is all there is.
586
+ saved = state.get("validator")
587
+ if saved and self.validator and saved != self.validator:
588
+ self.message = "changed on the server, started over"
589
+ return None
590
+ return [{"start": int(s["start"]), "end": int(s["end"]), "done": int(s["done"])}
591
+ for s in state["segments"]]
592
+ except (OSError, ValueError, KeyError, TypeError):
593
+ return None
594
+
595
+ def _save_state(self, state_path) -> None:
596
+ if not self.ranges:
597
+ return
598
+ with self.lock:
599
+ segments = [{k: s[k] for k in ("start", "end", "done")} for s in self.segments]
600
+ state = {"version": 1, "url": self.url, "total": self.total, "validator": self.validator,
601
+ "segments": segments}
602
+ tmp = state_path + ".tmp"
603
+ with open(tmp, "w", encoding="utf-8") as f:
604
+ json.dump(state, f)
605
+ os.replace(tmp, state_path)
606
+
607
+ @staticmethod
608
+ def _remove(path) -> None:
609
+ try:
610
+ os.remove(path)
611
+ except FileNotFoundError:
612
+ pass
613
+
614
+ # -- connections
615
+
616
+ def _halted(self, stop) -> bool:
617
+ return stop.is_set() or self.failed.is_set()
618
+
619
+ @staticmethod
620
+ def _complete(seg) -> bool:
621
+ if seg["end"] is None:
622
+ return seg.get("eof", False)
623
+ return seg["start"] + seg["done"] > seg["end"]
624
+
625
+ @staticmethod
626
+ def _speed_of(seg, now: float) -> float:
627
+ """Bytes/s of a piece: its smoothed rate, or failing that its rate since its request began."""
628
+ if seg.get("speed"):
629
+ return seg["speed"]
630
+ elapsed = now - seg.get("t0", now)
631
+ if elapsed < MIN_MEASURE:
632
+ return 0.0
633
+ received = seg["done"] - seg["b0"]
634
+ if received:
635
+ return received / elapsed
636
+ # Nothing yet. Only call it stalled after a long while: on a high-latency link a
637
+ # fresh request can easily take a second to deliver its first byte.
638
+ return 1.0 if elapsed >= STALL_TIME else 0.0
639
+
640
+ @staticmethod
641
+ def _free_from(seg) -> int:
642
+ """First byte nobody has claimed: past both the written data and any chunk being written."""
643
+ return max(seg["start"] + seg["done"], seg.get("reserved", 0))
644
+
645
+ def _next_segment(self, my_speed: float = 0.0) -> dict | None:
646
+ """Claim unfinished work, or take over part of the piece that will finish last.
647
+
648
+ `my_speed` is how fast the asking connection ran its previous piece.
649
+ """
650
+ with self.lock:
651
+ for seg in self.segments:
652
+ if not seg.get("busy") and not self._complete(seg):
653
+ seg["busy"] = True
654
+ return seg
655
+ if not self.ranges:
656
+ return None
657
+ # Only pieces that have run long enough to have a measured speed can be split:
658
+ # guessing the speed of a fresh piece lets idle connections pass it back and
659
+ # forth forever without anyone downloading it.
660
+ now = time.monotonic()
661
+ speeds = {id(s): self._speed_of(s, now) for s in self.segments if s.get("busy")}
662
+ measured = [s for s in self.segments if speeds.get(id(s))]
663
+ if not measured:
664
+ return None
665
+ thief = my_speed or sum(speeds[id(s)] for s in measured) / len(measured)
666
+
667
+ def eta(seg) -> float:
668
+ return (seg["end"] - self._free_from(seg) + 1) / speeds[id(seg)]
669
+
670
+ victim = max(measured, key=eta)
671
+ if eta(victim) < MIN_STEAL_ETA: # finishing soon anyway; a new request would not pay off
672
+ return None
673
+ free = self._free_from(victim)
674
+ left = victim["end"] - free + 1
675
+ # Split so both sides finish together: a connection half as fast keeps a third.
676
+ theirs = speeds[id(victim)]
677
+ keep = int(left * theirs / (theirs + thief))
678
+ mid = free + keep
679
+ if victim["end"] - mid + 1 < MIN_STEAL:
680
+ return None
681
+ stolen = {"start": mid, "end": victim["end"], "done": 0, "busy": True}
682
+ victim["end"] = mid - 1
683
+ self.segments.append(stolen)
684
+ return stolen
685
+
686
+ def _worker(self, fd, stop, limiter) -> None:
687
+ with self.lock:
688
+ self.active += 1
689
+ link = KeepAlive(self.final_url, self.opts.timeout) \
690
+ if self.ranges and keep_alive_possible(self.final_url) else None
691
+ my_speed = 0.0
692
+ try:
693
+ while not self._halted(stop):
694
+ seg = self._next_segment(my_speed)
695
+ if seg is None:
696
+ with self.lock:
697
+ if not any(not self._complete(s) for s in self.segments):
698
+ break
699
+ # A busy piece may become worth splitting soon; wake early if one finishes.
700
+ self.work_changed.wait(IDLE_POLL)
701
+ continue
702
+ try:
703
+ self._fetch(seg, fd, stop, limiter, link)
704
+ finally:
705
+ with self.lock:
706
+ seg["busy"] = False
707
+ my_speed = self._speed_of(seg, time.monotonic()) or my_speed
708
+ self.work_changed.notify_all()
709
+ except Retire:
710
+ with self.lock:
711
+ self.retired += 1
712
+ self.message = f"server limits connections, using {self.active - 1}"
713
+ except FatalError as e:
714
+ self.message, self.error_status = str(e), e.status
715
+ self.failed.set()
716
+ except Exception as e:
717
+ self.message = f"{type(e).__name__}: {e}"
718
+ self.failed.set()
719
+ finally:
720
+ if link:
721
+ link.close()
722
+ with self.lock:
723
+ self.active -= 1
724
+
725
+ def _open_range(self, link: KeepAlive | None, start, end):
726
+ if not self.ranges:
727
+ return request(self.final_url, self.opts.headers, self.opts.timeout)
728
+ # If-Range makes the server send the whole new file instead of a piece of it
729
+ # when the file has changed, so pieces of two versions never get mixed.
730
+ headers = {**self.opts.headers, "If-Range": self.validator} if self.validator else self.opts.headers
731
+ if link is None: # going through a proxy: let urllib handle it
732
+ resp = request(self.final_url, headers, self.opts.timeout, start, end)
733
+ else:
734
+ resp = link.get(build_headers(headers, start, end))
735
+ if resp.status != 206:
736
+ link.close()
737
+ if resp.status in RETRYABLE_HTTP:
738
+ raise ServerBusy(resp.status)
739
+ if resp.status != 200:
740
+ raise FatalError(f"HTTP {resp.status} {resp.reason}", resp.status)
741
+ if resp.status != 206:
742
+ resp.close()
743
+ if self.validator:
744
+ raise FatalError("the file changed on the server; run again to start over")
745
+ raise FatalError("server stopped honoring range requests")
746
+ return resp
747
+
748
+ def _fetch(self, seg, fd, stop, limiter, link) -> None:
749
+ attempts = 0
750
+ slow_reconnects = 0
751
+ while not self._halted(stop):
752
+ with self.lock:
753
+ if self._complete(seg):
754
+ return
755
+ pos, end = seg["start"] + seg["done"], seg["end"]
756
+ seg["t0"], seg["b0"] = time.monotonic(), seg["done"]
757
+ received = 0
758
+ finished = False
759
+ try:
760
+ with self._open_range(link, pos, end) as resp:
761
+ window_start, window_bytes = time.monotonic(), 0
762
+ speed_start, speed_bytes = window_start, 0
763
+ while not self._halted(stop):
764
+ with self.lock:
765
+ end = seg["end"]
766
+ want = CHUNK if end is None else min(CHUNK, end - pos + 1)
767
+ if want <= 0:
768
+ finished = True
769
+ break
770
+ data = resp.read1(want)
771
+ if not data:
772
+ break
773
+ limiter.consume(len(data))
774
+ with self.lock:
775
+ # Another connection may have taken over the tail meanwhile:
776
+ # keep only our share, and reserve it before writing.
777
+ end = seg["end"]
778
+ if end is not None:
779
+ data = data[:max(0, end - pos + 1)]
780
+ seg["reserved"] = pos + len(data)
781
+ if not data:
782
+ finished = True
783
+ break
784
+ write_at(fd, data, pos)
785
+ if not received and self.message.startswith("retry"):
786
+ self.message = ""
787
+ pos += len(data)
788
+ received += len(data)
789
+ window_bytes += len(data)
790
+ speed_bytes += len(data)
791
+ now = time.monotonic()
792
+ with self.lock:
793
+ seg["done"] += len(data)
794
+ self.done += len(data)
795
+ if now - speed_start >= SPEED_WINDOW:
796
+ instant = speed_bytes / (now - speed_start)
797
+ seg["speed"] = 0.5 * seg["speed"] + 0.5 * instant if seg.get("speed") else instant
798
+ speed_start, speed_bytes = now, 0
799
+ if now - window_start >= SLOW_WINDOW:
800
+ if slow_reconnects < SLOW_RECONNECTS and self._is_slow(window_bytes / (now - window_start)):
801
+ raise SlowConnection
802
+ window_start, window_bytes = now, 0
803
+ # A keep-alive connection is reusable only once its response is fully read.
804
+ if link and not resp.isclosed():
805
+ link.close()
806
+ if finished or self._halted(stop):
807
+ return
808
+ if end is None:
809
+ with self.lock:
810
+ seg["eof"] = True
811
+ return
812
+ raise ConnectionError("connection closed early")
813
+ except SlowConnection:
814
+ slow_reconnects += 1
815
+ if link:
816
+ link.close()
817
+ except RETRYABLE_ERRORS as e:
818
+ if link:
819
+ link.close()
820
+ if self._server_busy(e) and self.active > MIN_ACTIVE:
821
+ raise Retire from None
822
+ attempts = 1 if received else attempts + 1
823
+ if attempts > self.opts.retries:
824
+ raise FatalError(f"gave up after {self.opts.retries} retries: {reason(e)}") from None
825
+ if not self.ranges: # no way to continue mid-file; start over
826
+ with self.lock:
827
+ self.done -= seg["done"]
828
+ seg["done"] = 0
829
+ self.message = f"retry {attempts}: {reason(e)}"
830
+ stop.wait(min(2**attempts, 30))
831
+
832
+ def _is_slow(self, my_speed: float) -> bool:
833
+ if self.active < 2 or self.opts.limit: # with a speed cap, slowness is on purpose
834
+ return False
835
+ with self.lock:
836
+ measured = [s["speed"] for s in self.segments if s.get("busy") and s.get("speed")]
837
+ if len(measured) < 2:
838
+ return False
839
+ fastest_half = sorted(measured)[len(measured) // 2:]
840
+ typical = sum(fastest_half) / len(fastest_half)
841
+ return my_speed < SLOW_RATIO * typical
842
+
843
+ @staticmethod
844
+ def _server_busy(e: BaseException) -> bool:
845
+ status = getattr(e, "status", None) if isinstance(e, ServerBusy) else getattr(e, "code", None)
846
+ return status in BUSY_HTTP
847
+
848
+
849
+ def reason(e: BaseException) -> str:
850
+ if isinstance(e, urllib.error.HTTPError):
851
+ return f"HTTP {e.code}"
852
+ if isinstance(e, urllib.error.URLError):
853
+ return str(e.reason)
854
+ return str(e) or type(e).__name__
855
+
856
+
857
+ def sha256_file(path: str) -> str:
858
+ digest = hashlib.sha256()
859
+ with open(path, "rb") as f:
860
+ while block := f.read(1024 * 1024):
861
+ digest.update(block)
862
+ return digest.hexdigest()
863
+
864
+
865
+ # ---------------------------------------------------------------- display
866
+
867
+ STYLE = { # status: (icon, ANSI color)
868
+ "waiting": ("·", "2"),
869
+ "connecting": ("…", "33"),
870
+ "downloading": ("↓", "36"),
871
+ "verifying": ("#", "35"),
872
+ "done": ("✔", "32"),
873
+ "error": ("✖", "31"),
874
+ "paused": ("‖", "33"),
875
+ }
876
+
877
+
878
+ BAR = "━" # every cell of the progress bar; its color says how much of that cell is done
879
+ # By shade: nothing, some, most, all of the cell's bytes on disk. Marking the cells connections
880
+ # were writing to made the bar solid white once there were more connections than cells.
881
+ BAR_COLORS = ("2;37", "2;36", "36", "96")
882
+ BAR_MIN, BAR_GOOD, BAR_MAX = 10, 20, 40 # past BAR_MAX a bar only spreads the same information thinner
883
+ NAME_MIN = 16 # narrower than this and rows drop columns to keep the name readable
884
+ NAME_WANT = 40 # names get this much before the bar grows past BAR_MIN
885
+ FIELDS = [(6, ""), (10, ""), (7, ""), (19, "2"), (3, "2")] # width and style: %, speed, ETA, size, connections
886
+
887
+
888
+ def fields_width(count: int) -> int:
889
+ return sum(w + 2 for w, _ in FIELDS[:count])
890
+
891
+
892
+ def coverage_map(pieces: list[tuple[int, int]], total: int, width: int) -> list[int]:
893
+ """Split the file into `width` cells and shade each by how much of it is on disk: 0 for
894
+ nothing, 1 for some, 2 for most, 3 for all."""
895
+ if not total or width <= 0:
896
+ return []
897
+ cell = total / width
898
+ filled = [0.0] * width
899
+ for start, done in pieces:
900
+ end = start + done # bytes [start, end) are on disk
901
+ if done > 0:
902
+ for i in range(int(start // cell), min(width - 1, int((end - 1) // cell)) + 1):
903
+ overlap = min(end, (i + 1) * cell) - max(start, i * cell)
904
+ if overlap > 0:
905
+ filled[i] += overlap / cell
906
+ return [3 if f >= 0.999 else 2 if f >= 0.5 else 1 if f > 0 else 0 for f in filled]
907
+
908
+
909
+ class Display:
910
+ def __init__(self, downloads: list[Download], stream=sys.stderr, watching: bool = False):
911
+ self.downloads = downloads
912
+ self.stream = stream
913
+ self.tty = stream.isatty()
914
+ self.watching = watching
915
+ self.drawn: list[int] = [] # screen width of each row of the last frame
916
+ self.reported: dict[int, str] = {}
917
+
918
+ def draw(self) -> None:
919
+ if not self.tty:
920
+ self._log_changes()
921
+ return
922
+ size = shutil.get_terminal_size((100, 24))
923
+ # One column spare: a row that fills the last column leaves some terminals waiting to wrap.
924
+ width = size.columns - 1
925
+ visible = self._visible(max(1, size.lines - 2))
926
+ layout = self._layout(visible, width)
927
+ rows = [self._row(d, layout, width) for d in visible]
928
+ rows.append(self._summary(width))
929
+ # After a resize the terminal rewraps the old frame, so a row drawn at 120 columns now
930
+ # takes two rows at 80. Moving up one row per old row left stale copies of the frame
931
+ # behind every time the window got narrower.
932
+ up = sum(max(1, -(-w // size.columns)) for w in self.drawn)
933
+ out = ["\x1b[?25l"]
934
+ if up:
935
+ out.append(f"\x1b[{up}F")
936
+ out.extend(f"\x1b[2K{row}\n" for row in rows)
937
+ out.append("\x1b[J")
938
+ self.drawn = [screen_width(row) for row in rows]
939
+ self.stream.write("".join(out))
940
+ self.stream.flush()
941
+
942
+ def close(self) -> None:
943
+ if self.tty:
944
+ self.stream.write("\x1b[?25h")
945
+ self.stream.flush()
946
+
947
+ def _visible(self, limit: int) -> list[Download]:
948
+ if len(self.downloads) <= limit:
949
+ return self.downloads
950
+ rank = {"downloading": 0, "connecting": 0, "verifying": 0, "error": 1, "paused": 1, "waiting": 2, "done": 3}
951
+ chosen = sorted(range(len(self.downloads)), key=lambda i: (rank[self.downloads[i].status], i))[:limit]
952
+ return [self.downloads[i] for i in sorted(chosen)]
953
+
954
+ @staticmethod
955
+ def _layout(visible: list[Download], width: int) -> tuple[int, int, int]:
956
+ """Column widths shared by every row, so bars and numbers line up: (name, bar, fields
957
+ kept). Fields are dropped from the right, least useful first, until the name has room."""
958
+ longest = max([screen_width(d.name) for d in visible] + [NAME_MIN])
959
+ count = len(FIELDS)
960
+ # Size and connection count matter less than a bar long enough to read as a map.
961
+ while count > 3 and width - 2 - (BAR_GOOD + 2) - fields_width(count) < min(longest, NAME_WANT):
962
+ count -= 1
963
+ while count > 2 and width - 2 - (BAR_MIN + 2) - fields_width(count) < NAME_MIN:
964
+ count -= 1
965
+ room = width - 2 - fields_width(count)
966
+ bar = min(BAR_MAX, room - min(longest, NAME_WANT) - 2)
967
+ if bar < BAR_MIN:
968
+ bar = BAR_MIN if room - NAME_MIN - 2 >= BAR_MIN else 0
969
+ name = max(1, min(longest, room - (bar + 2 if bar else 0)))
970
+ return name, bar, count
971
+
972
+ def _row(self, d: Download, layout: tuple[int, int, int], width: int) -> str:
973
+ """One line per download: name, a bar that doubles as the piece map, then the numbers."""
974
+ name_width, bar_width, count = layout
975
+ icon, color = STYLE[d.status]
976
+ parts = [(f"{icon} ", color), (clip(d.name, name_width), "2" if d.status == "done" else "")]
977
+ if bar_width:
978
+ parts.append((" ", ""))
979
+ parts.extend(self._bar(d, bar_width) if d.status != "done" else [(" " * bar_width, "")])
980
+ if d.status == "downloading" and not d.message.startswith("retry"):
981
+ speed = d.speed()
982
+ left = (d.total - d.done) / speed if d.total and speed > 0 else None
983
+ values = [percent(d), f"{human_size(speed)}/s", human_time(left),
984
+ f"{human_size(d.done)}/{human_size(d.total)}", f"{d.active}×"]
985
+ parts.extend((" " + value.rjust(w), style) for value, (w, style) in zip(values, FIELDS[:count]))
986
+ else:
987
+ if d.status == "done":
988
+ note, style = f"{d.message or 'done'} · {human_size(d.total or d.done)}", "2"
989
+ elif d.status == "downloading": # retrying
990
+ note, style = d.message, "33"
991
+ else:
992
+ note, style = d.message or d.status, "2"
993
+ parts.append((" " + ("" if d.status == "done" else percent(d)).rjust(FIELDS[0][0]), ""))
994
+ parts.append((" " + note, style))
995
+ return fit(parts, width)
996
+
997
+ @staticmethod
998
+ def _bar(d: Download, width: int) -> list[tuple[str, str]]:
999
+ if d.total:
1000
+ cells = coverage_map(d.pieces(), d.total, width)
1001
+ elif d.status == "downloading": # unknown size: a pulse that shows it is alive
1002
+ head = int(time.monotonic() * 8) % width
1003
+ cells = [3 if (i - head) % width < 3 else 0 for i in range(width)]
1004
+ else:
1005
+ cells = [0] * width
1006
+ parts: list[tuple[str, str]] = []
1007
+ for shade in cells:
1008
+ style = BAR_COLORS[shade]
1009
+ if d.status not in ("downloading", "verifying") and shade:
1010
+ style = "2;33" if d.status == "paused" else "2;37"
1011
+ if parts and parts[-1][1] == style:
1012
+ parts[-1] = (parts[-1][0] + BAR, style)
1013
+ else:
1014
+ parts.append((BAR, style))
1015
+ return parts
1016
+
1017
+ def _summary(self, width: int) -> str:
1018
+ counts: dict[str, int] = {}
1019
+ for d in self.downloads:
1020
+ counts[d.status] = counts.get(d.status, 0) + 1
1021
+ speed = sum(d.speed() for d in self.downloads if d.status == "downloading") \
1022
+ if counts.get("downloading") else 0
1023
+ parts = [f"{counts.get('done', 0)}/{len(self.downloads)} done"]
1024
+ if counts.get("error"):
1025
+ parts.append(f"{counts['error']} failed")
1026
+ if speed:
1027
+ parts.append(f"{human_size(speed)}/s total")
1028
+ if self.watching:
1029
+ parts.append("watching the clipboard for links · Ctrl+C to stop")
1030
+ elif any(d.status in ("waiting", "connecting", "downloading") for d in self.downloads):
1031
+ parts.append("Ctrl+C to pause")
1032
+ return fit([(" " + " · ".join(parts), "2")], width)
1033
+
1034
+ def _log_changes(self) -> None:
1035
+ for i, d in enumerate(self.downloads):
1036
+ if d.status in ("done", "error", "paused") and self.reported.get(i) != d.status:
1037
+ self.reported[i] = d.status
1038
+ detail = d.message or f"{human_size(d.done)}/{human_size(d.total)}"
1039
+ print(f"[{d.status}] {d.name}: {detail}", file=self.stream, flush=True)
1040
+
1041
+
1042
+ def percent(d: Download) -> str:
1043
+ if d.total:
1044
+ return f"{min(1.0, d.done / d.total) * 100:.1f}%"
1045
+ return "100.0%" if d.status == "done" else ""
1046
+
1047
+
1048
+ def screen_width(text: str) -> int:
1049
+ return sum(char_width(c) for c in re.sub(r"\x1b\[[0-9;?]*[A-Za-z]", "", text))
1050
+
1051
+
1052
+ def fit(parts: list[tuple[str, str]], width: int) -> str:
1053
+ """Join colored pieces, cut at `width` screen columns. Cutting by characters let wide
1054
+ characters in a file name push the row past the edge, where it wrapped and broke the redraw."""
1055
+ out, left = [], width
1056
+ for text, color in parts:
1057
+ kept = []
1058
+ for c in text:
1059
+ w = char_width(c)
1060
+ if w > left:
1061
+ left = 0
1062
+ break
1063
+ kept.append(c)
1064
+ left -= w
1065
+ if kept:
1066
+ text = "".join(kept)
1067
+ out.append(f"\x1b[{color}m{text}\x1b[0m" if color else text)
1068
+ if left <= 0:
1069
+ break
1070
+ return "".join(out)
1071
+
1072
+
1073
+ # ---------------------------------------------------------------- CLI
1074
+
1075
+ def add_network_options(parser: argparse.ArgumentParser) -> None:
1076
+ parser.add_argument("-t", "--timeout", type=float, default=30, help="network timeout in seconds (default: 30)")
1077
+ parser.add_argument("-H", "--header", action="append", default=[], metavar="'NAME: VALUE'",
1078
+ help="extra request header, e.g. cookie or referer (repeatable)")
1079
+ parser.add_argument("-x", "--proxy", metavar="URL",
1080
+ help="proxy for every request: socks5://host:port, socks5h://... or http://host:port")
1081
+
1082
+
1083
+ def add_download_options(parser: argparse.ArgumentParser) -> None:
1084
+ parser.add_argument("-c", "--connections", type=int, default=DEFAULT_CONNECTIONS,
1085
+ help=f"connections per file, up to {MAX_CONNECTIONS} (default: {DEFAULT_CONNECTIONS})")
1086
+ parser.add_argument("-j", "--jobs", type=int, default=3, help="files downloaded at once (default: 3)")
1087
+ parser.add_argument("-l", "--limit", type=parse_rate, metavar="RATE", help="total speed limit, e.g. 500K or 2M")
1088
+ parser.add_argument("-r", "--retries", type=int, default=10, help="retries per connection (default: 10)")
1089
+ add_network_options(parser)
1090
+ parser.add_argument("--at", type=parse_clock, metavar="HH:MM",
1091
+ help="wait until this time of day before starting")
1092
+ parser.add_argument("--until", type=parse_clock, metavar="HH:MM",
1093
+ help="pause at this time of day; with --at, carry on at the next --at, day after day")
1094
+ parser.add_argument("-n", "--notify", action="store_true",
1095
+ help="when downloads end, send a desktop notification, and a Telegram message "
1096
+ "if MOX_TELEGRAM_TOKEN and MOX_TELEGRAM_CHAT are set")
1097
+
1098
+
1099
+ def check_network_options(parser: argparse.ArgumentParser, args) -> None:
1100
+ if args.proxy:
1101
+ try:
1102
+ Proxy(args.proxy)
1103
+ except ValueError as e:
1104
+ parser.error(str(e))
1105
+ headers = {}
1106
+ for header in args.header:
1107
+ name, sep, value = header.partition(":")
1108
+ if not sep or not name.strip():
1109
+ parser.error(f"bad header {header!r}, expected 'Name: value'")
1110
+ headers[name.strip()] = value.strip()
1111
+ args.headers = headers
1112
+
1113
+
1114
+ def check_download_options(parser: argparse.ArgumentParser, args) -> None:
1115
+ if not 1 <= args.connections <= MAX_CONNECTIONS:
1116
+ parser.error(f"--connections must be between 1 and {MAX_CONNECTIONS}")
1117
+ if args.jobs < 1:
1118
+ parser.error("--jobs must be at least 1")
1119
+ if args.at is not None and args.at == args.until:
1120
+ parser.error("--at and --until must differ")
1121
+ check_network_options(parser, args)
1122
+
1123
+
1124
+ def parse_args(argv):
1125
+ parser = argparse.ArgumentParser(
1126
+ prog="mox", description="Fast, resumable, multi-connection downloads in the terminal.",
1127
+ epilog="Interrupted downloads resume automatically: run the same command again. "
1128
+ "Other commands: 'mox resume [DIR]' continues every unfinished download in a folder, "
1129
+ "'mox bench URL' measures how much faster many connections are.")
1130
+ parser.add_argument("urls", nargs="*", metavar="URL")
1131
+ parser.add_argument("-i", "--input", metavar="FILE", help="read URLs from FILE, one per line ('-' for stdin)")
1132
+ parser.add_argument("-d", "--dir", default=".", help="directory to save into (default: current)")
1133
+ parser.add_argument("-o", "--output", metavar="NAME", help="file name to save as (single URL only)")
1134
+ parser.add_argument("--sha256", metavar="HEX", help="verify the file against this checksum (single URL only)")
1135
+ parser.add_argument("-w", "--watch", action="store_true",
1136
+ help="keep running and download every file link you copy to the clipboard")
1137
+ add_download_options(parser)
1138
+ parser.add_argument("-V", "--version", action="version", version=f"mox {VERSION}")
1139
+ args = parser.parse_args(argv)
1140
+
1141
+ urls = list(args.urls)
1142
+ if args.input:
1143
+ source = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8")
1144
+ with source:
1145
+ urls += [line.strip() for line in source if line.strip() and not line.lstrip().startswith("#")]
1146
+ urls = [u if "://" in u else "https://" + u for u in dict.fromkeys(urls)]
1147
+ if not urls and not args.watch:
1148
+ parser.error("give at least one URL (or -i FILE, or --watch)")
1149
+ if (len(urls) > 1 or args.watch) and (args.output or args.sha256):
1150
+ parser.error("--output and --sha256 work with a single URL only")
1151
+ if args.watch and (args.at or args.until):
1152
+ parser.error("--watch cannot be combined with --at or --until")
1153
+ check_download_options(parser, args)
1154
+ args.urls = urls
1155
+ return args
1156
+
1157
+
1158
+ # ---------------------------------------------------------------- scheduling
1159
+
1160
+ def parse_clock(text: str) -> datetime.time:
1161
+ match = re.fullmatch(r"\s*(\d{1,2}):(\d{2})(?::(\d{2}))?\s*", text)
1162
+ if not match or int(match.group(1)) > 23 or int(match.group(2)) > 59 or int(match.group(3) or 0) > 59:
1163
+ raise argparse.ArgumentTypeError(f"invalid time {text!r} (examples: 02:00, 23:30)")
1164
+ return datetime.time(int(match.group(1)), int(match.group(2)), int(match.group(3) or 0))
1165
+
1166
+
1167
+ def show_clock(clock: datetime.time) -> str:
1168
+ return f"{clock:%H:%M:%S}" if clock.second else f"{clock:%H:%M}"
1169
+
1170
+
1171
+ def next_time(clock: datetime.time, now: datetime.datetime) -> datetime.datetime:
1172
+ """The next moment the wall clock shows `clock`: later today, or else tomorrow."""
1173
+ moment = now.replace(hour=clock.hour, minute=clock.minute, second=clock.second, microsecond=0)
1174
+ return moment if moment > now else moment + datetime.timedelta(days=1)
1175
+
1176
+
1177
+ def in_window(now: datetime.datetime, start: datetime.time, end: datetime.time) -> bool:
1178
+ """Whether `now` falls between start and end, for windows that may cross midnight."""
1179
+ t = now.time()
1180
+ return start <= t < end if start < end else (t >= start or t < end)
1181
+
1182
+
1183
+ def wait_until(moment: datetime.datetime) -> bool:
1184
+ """Sleep until `moment`, showing a countdown. False if the user pressed Ctrl+C."""
1185
+ tty = sys.stderr.isatty()
1186
+ if not tty:
1187
+ print(f"Waiting until {moment:%H:%M:%S}", file=sys.stderr, flush=True)
1188
+ try:
1189
+ while (left := (moment - datetime.datetime.now()).total_seconds()) > 0:
1190
+ if tty:
1191
+ sys.stderr.write(f"\r\x1b[2K\x1b[2m… waiting until {moment:%H:%M:%S} "
1192
+ f"({human_time(left)} left) · Ctrl+C to cancel\x1b[0m")
1193
+ sys.stderr.flush()
1194
+ time.sleep(min(1.0, left))
1195
+ except KeyboardInterrupt:
1196
+ return False
1197
+ finally:
1198
+ if tty:
1199
+ sys.stderr.write("\r\x1b[2K")
1200
+ sys.stderr.flush()
1201
+ return True
1202
+
1203
+
1204
+ # ---------------------------------------------------------------- notifications
1205
+
1206
+ def notify(title: str, body: str) -> None:
1207
+ """Desktop notification where a tool for it exists, plus Telegram when configured.
1208
+
1209
+ A notification that fails only prints a warning: it never changes the exit code.
1210
+ """
1211
+ token, chat = os.environ.get("MOX_TELEGRAM_TOKEN"), os.environ.get("MOX_TELEGRAM_CHAT")
1212
+ if token and chat:
1213
+ # MOX_TELEGRAM_API points at a self-hosted Bot API server, if you run one.
1214
+ api = os.environ.get("MOX_TELEGRAM_API", "https://api.telegram.org").rstrip("/")
1215
+ data = urllib.parse.urlencode({"chat_id": chat, "text": f"{title}\n{body}"}).encode()
1216
+ try:
1217
+ _opener.open(urllib.request.Request(f"{api}/bot{token}/sendMessage", data=data), timeout=15).close()
1218
+ except (OSError, http.client.HTTPException) as e:
1219
+ print(f"! Telegram notification failed: {reason(e)}", file=sys.stderr)
1220
+ for command in (["notify-send", "--app-name=mox", title, body],
1221
+ ["termux-notification", "--title", title, "--content", body],
1222
+ ["osascript", "-e", f"display notification {json.dumps(body)} with title {json.dumps(title)}"]):
1223
+ if shutil.which(command[0]):
1224
+ try:
1225
+ subprocess.run(command, capture_output=True, timeout=10)
1226
+ except (OSError, subprocess.SubprocessError):
1227
+ pass
1228
+ break
1229
+
1230
+
1231
+ def notify_summary(downloads: list[Download], outcome: str) -> None:
1232
+ done = [d for d in downloads if d.status == "done"]
1233
+ failed = [d for d in downloads if d.status == "error"]
1234
+ unfinished = [d for d in downloads if d.status in ("paused", "waiting")]
1235
+ if outcome == "deadline" and unfinished:
1236
+ title = f"mox stopped as scheduled, {len(unfinished)} unfinished"
1237
+ elif failed:
1238
+ title = f"mox: {len(done)} done, {len(failed)} failed"
1239
+ else:
1240
+ title = "mox: all downloads finished" if len(done) != 1 else "mox: download finished"
1241
+ names = [d.name for d in done + failed][:5]
1242
+ more = len(done) + len(failed) - len(names)
1243
+ notify(title, "\n".join(names) + (f"\nand {more} more" if more > 0 else ""))
1244
+
1245
+
1246
+ # ---------------------------------------------------------------- clipboard
1247
+
1248
+ URL_PATTERN = re.compile(r"https?://[^\s<>\"'`]+")
1249
+ PAGE_EXTENSIONS = {"", ".html", ".htm", ".php", ".asp", ".aspx", ".jsp", ".cgi"}
1250
+ CLIPBOARD_POLL = 1.0 # seconds between clipboard reads in --watch mode
1251
+
1252
+
1253
+ def clipboard_command() -> list[str] | None:
1254
+ custom = os.environ.get("MOX_CLIPBOARD_COMMAND")
1255
+ if custom:
1256
+ return shlex.split(custom)
1257
+ candidates = [["wl-paste", "--no-newline"]] if os.environ.get("WAYLAND_DISPLAY") else []
1258
+ candidates += [["xclip", "-o", "-selection", "clipboard"], ["xsel", "--clipboard", "--output"],
1259
+ ["pbpaste"], ["termux-clipboard-get"],
1260
+ ["powershell.exe", "-NoProfile", "-Command", "Get-Clipboard"]]
1261
+ return next((c for c in candidates if shutil.which(c[0])), None)
1262
+
1263
+
1264
+ def read_clipboard(command: list[str]) -> str:
1265
+ try:
1266
+ return subprocess.run(command, capture_output=True, text=True, timeout=5).stdout
1267
+ except (OSError, subprocess.SubprocessError):
1268
+ return ""
1269
+
1270
+
1271
+ def links_in(text: str) -> list[str]:
1272
+ """File links in a piece of text. Links to web pages are skipped: copying the address of
1273
+ an article should not download its HTML."""
1274
+ links = []
1275
+ for url in URL_PATTERN.findall(text):
1276
+ url = url.rstrip(".,;:)]}>")
1277
+ name = urllib.parse.urlsplit(url).path.rsplit("/", 1)[-1]
1278
+ if os.path.splitext(name)[1].lower() not in PAGE_EXTENSIONS:
1279
+ links.append(url)
1280
+ return links
1281
+
1282
+
1283
+ # ---------------------------------------------------------------- running downloads
1284
+
1285
+ def run_session(downloads: list[Download], args, limiter: RateLimiter, deadline) -> str:
1286
+ """Run every waiting download. Returns 'finished', 'interrupted' or 'deadline'."""
1287
+ _claimed_paths.clear()
1288
+ stop = threading.Event()
1289
+ display = Display(downloads)
1290
+ pool = ThreadPoolExecutor(max_workers=args.jobs)
1291
+ futures = [pool.submit(d.run, stop, limiter) for d in downloads if d.status == "waiting"]
1292
+ outcome = "finished"
1293
+ try:
1294
+ try:
1295
+ while not all(f.done() for f in futures):
1296
+ if deadline and outcome == "finished" and datetime.datetime.now() >= deadline:
1297
+ outcome = "deadline"
1298
+ stop.set()
1299
+ display.draw()
1300
+ time.sleep(FRAME_INTERVAL)
1301
+ except KeyboardInterrupt:
1302
+ outcome = "interrupted"
1303
+ stop.set()
1304
+ while not all(f.done() for f in futures):
1305
+ display.draw()
1306
+ time.sleep(0.1)
1307
+ pool.shutdown(wait=True)
1308
+ display.draw()
1309
+ finally:
1310
+ display.close()
1311
+ return outcome
1312
+
1313
+
1314
+ def watch_session(downloads: list[Download], args, limiter: RateLimiter, command: list[str]) -> str:
1315
+ """Download what is given, then every new file link that shows up on the clipboard."""
1316
+ _claimed_paths.clear()
1317
+ seen = {d.url for d in downloads} | set(links_in(read_clipboard(command))) # not what was there before
1318
+ stop = threading.Event()
1319
+ display = Display(downloads, watching=True)
1320
+ pool = ThreadPoolExecutor(max_workers=args.jobs)
1321
+ futures = [pool.submit(d.run, stop, limiter) for d in downloads]
1322
+ announced: set[int] = set()
1323
+ last_poll = 0.0
1324
+ try:
1325
+ try:
1326
+ while True:
1327
+ if time.monotonic() - last_poll >= CLIPBOARD_POLL:
1328
+ last_poll = time.monotonic()
1329
+ for url in links_in(read_clipboard(command)):
1330
+ if url not in seen:
1331
+ seen.add(url)
1332
+ d = Download(url, args)
1333
+ downloads.append(d)
1334
+ futures.append(pool.submit(d.run, stop, limiter))
1335
+ if args.notify:
1336
+ for d in downloads:
1337
+ if d.status in ("done", "error") and id(d) not in announced:
1338
+ announced.add(id(d))
1339
+ notify("mox: download finished" if d.status == "done" else "mox: download failed",
1340
+ d.name if d.status == "done" else f"{d.name}: {d.message}")
1341
+ display.draw()
1342
+ time.sleep(FRAME_INTERVAL)
1343
+ except KeyboardInterrupt:
1344
+ stop.set()
1345
+ while not all(f.done() for f in futures):
1346
+ display.draw()
1347
+ time.sleep(0.1)
1348
+ pool.shutdown(wait=True)
1349
+ display.draw()
1350
+ finally:
1351
+ display.close()
1352
+ return "interrupted" if any(d.status in ("paused", "waiting") for d in downloads) else "finished"
1353
+
1354
+
1355
+ def expired_link_hint(d: Download) -> str:
1356
+ folder = os.path.dirname(d.path) if d.path else d.opts.dir
1357
+ return (" If this link worked before, it has probably expired. Get a fresh link and run\n"
1358
+ f" mox -d {shlex.quote(folder or '.')} -o {shlex.quote(d.name)} 'NEW_LINK'\n"
1359
+ " to continue from where it stopped.")
1360
+
1361
+
1362
+ def report(downloads: list[Download], outcome: str, args) -> int:
1363
+ failed = [d for d in downloads if d.status == "error"]
1364
+ unfinished = [d for d in downloads if d.status in ("paused", "waiting")]
1365
+ for d in failed:
1366
+ print(f"✖ {d.name}: {d.message}", file=sys.stderr)
1367
+ if d.error_status in EXPIRED_HTTP:
1368
+ print(expired_link_hint(d), file=sys.stderr)
1369
+ if unfinished and outcome == "deadline":
1370
+ print(f"‖ Stopped at {show_clock(args.until)} as scheduled with {len(unfinished)} download(s) unfinished. "
1371
+ "Run the same command again to continue.", file=sys.stderr)
1372
+ elif unfinished:
1373
+ print(f"‖ Paused {len(unfinished)} download(s). Run the same command again, "
1374
+ "or 'mox resume' in that folder, to continue.", file=sys.stderr)
1375
+ if outcome == "interrupted":
1376
+ return 130
1377
+ if failed:
1378
+ return 1
1379
+ return 3 if unfinished else 0
1380
+
1381
+
1382
+ def run_downloads(downloads: list[Download], args) -> int:
1383
+ """Run downloads, on the --at/--until schedule if there is one, and report how it went."""
1384
+ limiter = RateLimiter(args.limit)
1385
+ while True:
1386
+ now = datetime.datetime.now()
1387
+ if args.at and not (args.until and in_window(now, args.at, args.until)):
1388
+ if not wait_until(next_time(args.at, now)):
1389
+ return 130
1390
+ deadline = next_time(args.until, datetime.datetime.now()) if args.until else None
1391
+ outcome = run_session(downloads, args, limiter, deadline)
1392
+ unfinished = [d for d in downloads if d.status in ("paused", "waiting")]
1393
+ if outcome != "deadline" or not unfinished or not args.at:
1394
+ break
1395
+ print(f"‖ Stopped at {show_clock(args.until)} as scheduled. Carrying on at {show_clock(args.at)}.",
1396
+ file=sys.stderr, flush=True)
1397
+ downloads = [d.restart() if d.status in ("paused", "waiting") else d for d in downloads]
1398
+ if args.notify and outcome != "interrupted":
1399
+ notify_summary(downloads, outcome)
1400
+ return report(downloads, outcome, args)
1401
+
1402
+
1403
+ # ---------------------------------------------------------------- mox resume
1404
+
1405
+ def resume_main(argv) -> int:
1406
+ parser = argparse.ArgumentParser(
1407
+ prog="mox resume", description="Continue every unfinished download in a folder.",
1408
+ epilog="mox remembers each unfinished download in a FILE.mox next to it, link included.")
1409
+ parser.add_argument("dir", nargs="?", default=".", help="folder to look in (default: current)")
1410
+ add_download_options(parser)
1411
+ args = parser.parse_args(argv)
1412
+ check_download_options(parser, args)
1413
+ configure_proxy(args.proxy)
1414
+
1415
+ downloads = []
1416
+ for entry in sorted(os.listdir(args.dir)) if os.path.isdir(args.dir) else []:
1417
+ if not entry.endswith(STATE_SUFFIX):
1418
+ continue
1419
+ try:
1420
+ with open(os.path.join(args.dir, entry), encoding="utf-8") as f:
1421
+ url = json.load(f)["url"]
1422
+ except (OSError, ValueError, KeyError, TypeError):
1423
+ print(f"! skipping {entry}: not a state file mox can read", file=sys.stderr)
1424
+ continue
1425
+ downloads.append(Download(url, args, name=entry[:-len(STATE_SUFFIX)]))
1426
+ if not downloads:
1427
+ print(f"No unfinished downloads in {args.dir}.", file=sys.stderr)
1428
+ return 0
1429
+ return run_downloads(downloads, args)
1430
+
1431
+
1432
+ # ---------------------------------------------------------------- mox bench
1433
+
1434
+ def bench_round(url: str, total: int | None, connections: int, seconds: float, args) -> tuple[float, bool, list[str]]:
1435
+ """Download for `seconds` over `connections` connections, keeping nothing.
1436
+
1437
+ Returns (bytes per second, whether the round was too short to trust, why each refused
1438
+ connection was refused).
1439
+ """
1440
+ received = [0]
1441
+ first_finish: list[tuple[float, int]] = [] # (time, bytes) when the first connection ran out of work
1442
+ refused: list[str] = []
1443
+ lock = threading.Lock()
1444
+ deadline = time.monotonic() + seconds
1445
+
1446
+ def fetch(start: int | None, end: int | None) -> None:
1447
+ link = KeepAlive(url, args.timeout) if start is not None and keep_alive_possible(url) else None
1448
+ try:
1449
+ if link:
1450
+ resp = link.get(build_headers(args.headers, start, end))
1451
+ if resp.status not in (200, 206):
1452
+ raise ServerBusy(resp.status)
1453
+ else:
1454
+ resp = request(url, args.headers, args.timeout, start, end)
1455
+ with resp:
1456
+ while time.monotonic() < deadline:
1457
+ data = resp.read1(CHUNK)
1458
+ if not data:
1459
+ # Only a connection that got through its whole share ends the window.
1460
+ # A refused one (Hetzner answers 429 past two connections) must not:
1461
+ # it would stop the clock at zero bytes.
1462
+ with lock:
1463
+ if not first_finish:
1464
+ first_finish.append((time.monotonic(), received[0]))
1465
+ break
1466
+ with lock:
1467
+ received[0] += len(data)
1468
+ except (RETRYABLE_ERRORS + (FatalError,)) as e:
1469
+ with lock:
1470
+ refused.append(reason(e))
1471
+ finally:
1472
+ if link:
1473
+ link.close()
1474
+
1475
+ if total:
1476
+ bounds = [(total * i // connections, total * (i + 1) // connections - 1) for i in range(connections)]
1477
+ else:
1478
+ bounds = [(None, None)]
1479
+ threads = [threading.Thread(target=fetch, args=b, daemon=True) for b in bounds]
1480
+ started = time.monotonic()
1481
+ for t in threads:
1482
+ t.start()
1483
+ tty = sys.stderr.isatty()
1484
+ label = f"{connections:>3} connection{'s' if connections > 1 else ' '}"
1485
+ while any(t.is_alive() for t in threads):
1486
+ time.sleep(FRAME_INTERVAL)
1487
+ if tty:
1488
+ elapsed = time.monotonic() - started
1489
+ sys.stderr.write(f"\r\x1b[2K{label} {human_size(received[0] / elapsed) + '/s':>11} "
1490
+ f"\x1b[2m{max(0.0, seconds - elapsed):.0f}s left\x1b[0m")
1491
+ sys.stderr.flush()
1492
+ if tty:
1493
+ sys.stderr.write("\r\x1b[2K")
1494
+ # Count only while every connection was busy. Once the fast ones finish their share, the
1495
+ # slow stragglers alone would drag the average down and hide what N connections can do.
1496
+ ended, got = first_finish[0] if first_finish else (time.monotonic(), received[0])
1497
+ window = max(ended - started, 1e-3)
1498
+ return got / window, window < min(1.0, seconds / 2), refused
1499
+
1500
+
1501
+ def bench_main(argv) -> int:
1502
+ parser = argparse.ArgumentParser(
1503
+ prog="mox bench", description="Measure how much faster many connections are than one. "
1504
+ "Nothing is saved to disk.")
1505
+ parser.add_argument("url", metavar="URL")
1506
+ parser.add_argument("-s", "--seconds", type=float, default=5, help="length of each round (default: 5)")
1507
+ parser.add_argument("-c", "--connections", type=int, default=DEFAULT_CONNECTIONS,
1508
+ help=f"most connections to try (default: {DEFAULT_CONNECTIONS})")
1509
+ add_network_options(parser)
1510
+ args = parser.parse_args(argv)
1511
+ if not 1 <= args.connections <= MAX_CONNECTIONS:
1512
+ parser.error(f"--connections must be between 1 and {MAX_CONNECTIONS}")
1513
+ check_network_options(parser, args)
1514
+ configure_proxy(args.proxy)
1515
+ url = args.url if "://" in args.url else "https://" + args.url
1516
+
1517
+ try:
1518
+ final, total, ranges, name, _ = probe(url, args.headers, args.timeout)
1519
+ except (RETRYABLE_ERRORS + (FatalError,)) as e:
1520
+ print(f"✖ {reason(e)}", file=sys.stderr)
1521
+ return 1
1522
+ support = "supports range requests" if ranges else "does not support range requests"
1523
+ print(f"{name} · {human_size(total)} · the server {support}")
1524
+ levels = sorted({1, min(4, args.connections), args.connections}) if ranges else [1]
1525
+ rates: dict[int, float] = {}
1526
+ allowed: int | None = None # how many connections the server accepted, when it refused some
1527
+ rough = False
1528
+ try:
1529
+ for connections in levels:
1530
+ rate, short, refused = bench_round(final, total if ranges else None, connections, args.seconds, args)
1531
+ rates[connections] = rate
1532
+ rough = rough or short
1533
+ gain = f" {rate / rates[1]:4.1f}×" if connections > 1 and rates[1] else ""
1534
+ label = f"{connections:>3} connection{'s' if connections > 1 else ' '}"
1535
+ note = f" \x1b[2m{len(refused)} refused: {refused[0]}\x1b[0m" if refused and sys.stdout.isatty() \
1536
+ else (f" {len(refused)} refused: {refused[0]}" if refused else "")
1537
+ print(f"{label} {human_size(rate) + '/s':>11}{gain}{note}", flush=True)
1538
+ if refused and allowed is None:
1539
+ allowed = connections - len(refused)
1540
+ except KeyboardInterrupt:
1541
+ return 130
1542
+
1543
+ best = max(rates, key=rates.get)
1544
+ gain = rates[best] / rates[1] if rates[1] else 0
1545
+ if not ranges:
1546
+ print("This server sends files in one piece, so extra connections cannot help here.")
1547
+ elif allowed is not None:
1548
+ print(f"This server accepts only about {max(allowed, 1)} connections at once and refuses the rest. "
1549
+ f"mox backs off by itself, but -c {max(allowed, 1)} avoids the refusals.")
1550
+ elif gain >= 1.5:
1551
+ print(f"This server limits each connection. With -c {best}, mox is about {gain:.0f}× faster here "
1552
+ "than a one-connection download such as wget or curl.")
1553
+ else:
1554
+ print("Extra connections barely help here: your line or the server's total is the limit. "
1555
+ "A few connections, -c 4, are plenty.")
1556
+ if rough:
1557
+ print("The file is small enough to finish within a round, so these numbers are rough.")
1558
+ return 0
1559
+
1560
+
1561
+ # ---------------------------------------------------------------- library
1562
+
1563
+ class DownloadError(Exception):
1564
+ """Raised by download() when a file could not be downloaded."""
1565
+
1566
+
1567
+ def download(url: str, directory: str = ".", *, name: str | None = None,
1568
+ connections: int = DEFAULT_CONNECTIONS, limit: int | str | None = None, retries: int = 10,
1569
+ timeout: float = 30, headers: dict | None = None, sha256: str | None = None,
1570
+ proxy: str | None = None, progress=None) -> str:
1571
+ """Download one URL over many connections and return the path of the saved file.
1572
+
1573
+ It resumes a matching unfinished download in `directory`, just like the command does.
1574
+ `limit` is bytes per second, or text such as "2M". `progress`, if given, is called about
1575
+ four times a second with (bytes done, total bytes or None). `proxy` applies to the whole
1576
+ process, since all requests share one opener. Raises DownloadError on failure.
1577
+ """
1578
+ if not 1 <= connections <= MAX_CONNECTIONS:
1579
+ raise ValueError(f"connections must be between 1 and {MAX_CONNECTIONS}")
1580
+ if isinstance(limit, str):
1581
+ try:
1582
+ limit = parse_rate(limit)
1583
+ except argparse.ArgumentTypeError as e:
1584
+ raise ValueError(str(e)) from None
1585
+ configure_proxy(proxy)
1586
+ opts = argparse.Namespace(dir=directory, connections=connections, retries=retries,
1587
+ timeout=timeout, headers=dict(headers or {}), limit=limit)
1588
+ d = Download(url if "://" in url else "https://" + url, opts, name=name, sha256=sha256)
1589
+ stop = threading.Event()
1590
+ worker = threading.Thread(target=d.run, args=(stop, RateLimiter(limit)), daemon=True)
1591
+ worker.start()
1592
+ try:
1593
+ while worker.is_alive():
1594
+ worker.join(FRAME_INTERVAL)
1595
+ if progress:
1596
+ progress(d.done, d.total)
1597
+ except BaseException:
1598
+ stop.set()
1599
+ worker.join()
1600
+ raise
1601
+ finally:
1602
+ if d.path:
1603
+ with _claim_lock:
1604
+ _claimed_paths.discard(os.path.abspath(d.path))
1605
+ if d.status != "done":
1606
+ raise DownloadError(f"{d.name}: {d.message or d.status}")
1607
+ return d.path
1608
+
1609
+
1610
+ # ---------------------------------------------------------------- main
1611
+
1612
+ def main(argv=None) -> int:
1613
+ argv = sys.argv[1:] if argv is None else list(argv)
1614
+ if argv and argv[0] == "resume":
1615
+ return resume_main(argv[1:])
1616
+ if argv and argv[0] == "bench":
1617
+ return bench_main(argv[1:])
1618
+ args = parse_args(argv)
1619
+ configure_proxy(args.proxy)
1620
+ downloads = [Download(url, args, name=args.output, sha256=args.sha256) for url in args.urls]
1621
+ if not args.watch:
1622
+ return run_downloads(downloads, args)
1623
+ command = clipboard_command()
1624
+ if not command:
1625
+ print("✖ --watch needs a clipboard tool: wl-clipboard on Wayland, xclip or xsel on X11, "
1626
+ "termux-api on Termux. Or set MOX_CLIPBOARD_COMMAND.", file=sys.stderr)
1627
+ return 2
1628
+ outcome = watch_session(downloads, args, RateLimiter(args.limit), command)
1629
+ return report(downloads, outcome, args)
1630
+
1631
+
1632
+ if __name__ == "__main__":
1633
+ sys.exit(main())