streamshatter 1.0.0__tar.gz

streamshatter-1.0.0/.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+dist
+cache

streamshatter-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Thomas Xin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

streamshatter-1.0.0/PKG-INFO

@@ -0,0 +1,40 @@
+Metadata-Version: 2.4
+Name: streamshatter
+Version: 1.0.0
+Summary: Multiplexed chunked file downloader
+Author-email: Thomas Xin <thomasxin@gmail.com>
+License: MIT
+License-File: LICENSE
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Requires-Dist: niquests>=3.15.2
+Requires-Dist: urllib3-future>=2.13.908
+Description-Content-Type: text/markdown
+
+# StreamShatter
+Originally a very basic script for downloading files from servers with inconsistent connections, this project has been revisited and modernised to use https://github.com/jawah/niquests for multiplexing performance, both for private testing of niquests' stability, and for general improvements in functionality for those who still have use for such a tool.
+
+StreamShatter takes advantage of the `Range` HTTP header to dynamically allocate multiple chunks, by starting with one streaming request and gradually bisecting it while bandwidth permits, all without restarting the download. This allows for single, large file downloads from hosts that, whether intentionally or unintentionally, have degraded throughputs. The individual chunks also serve as checkpoints for if/when connections are broken.
+
+# Installation
+- Install [python](https://www.python.org) and [pip](https://pip.pypa.io/en/stable/)
+- Install StreamShatter as a package:
+`pip install streamshatter`
+
+## Usage
+```ini
+usage: streamshatter [-h] [-V] [-c CACHE_FOLDER] [-l LIMIT] url [filename]
+
+Multiplexed chunked file downloader
+
+positional arguments:
+  url                   Target URL
+  filename              Output filename
+
+options:
+  -h, --help            show this help message and exit
+  -V, --version         show program's version number and exit
+  -c, --cache-folder CACHE_FOLDER
+                        Folder to store temporary files
+  -l, --limit LIMIT     Limits the amount of chunks to download
+```

streamshatter-1.0.0/README.md

@@ -0,0 +1,27 @@
+# StreamShatter
+Originally a very basic script for downloading files from servers with inconsistent connections, this project has been revisited and modernised to use https://github.com/jawah/niquests for multiplexing performance, both for private testing of niquests' stability, and for general improvements in functionality for those who still have use for such a tool.
+
+StreamShatter takes advantage of the `Range` HTTP header to dynamically allocate multiple chunks, by starting with one streaming request and gradually bisecting it while bandwidth permits, all without restarting the download. This allows for single, large file downloads from hosts that, whether intentionally or unintentionally, have degraded throughputs. The individual chunks also serve as checkpoints for if/when connections are broken.
+
+# Installation
+- Install [python](https://www.python.org) and [pip](https://pip.pypa.io/en/stable/)
+- Install StreamShatter as a package:
+`pip install streamshatter`
+
+## Usage
+```ini
+usage: streamshatter [-h] [-V] [-c CACHE_FOLDER] [-l LIMIT] url [filename]
+
+Multiplexed chunked file downloader
+
+positional arguments:
+  url                   Target URL
+  filename              Output filename
+
+options:
+  -h, --help            show this help message and exit
+  -V, --version         show program's version number and exit
+  -c, --cache-folder CACHE_FOLDER
+                        Folder to store temporary files
+  -l, --limit LIMIT     Limits the amount of chunks to download
+```

streamshatter-1.0.0/pyproject.toml

@@ -0,0 +1,20 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "streamshatter"
+version = "1.0.0"
+dependencies = [
+    "niquests>=3.15.2",
+    "urllib3-future>=2.13.908",
+]
+description = "Multiplexed chunked file downloader"
+readme = "README.md"
+requires-python = ">=3.8"
+authors = [{ name = "Thomas Xin", email = "thomasxin@gmail.com" }]
+license = { text = "MIT" }
+classifiers = ["Programming Language :: Python :: 3"]
+
+[project.scripts]
+streamshatter = "streamshatter:main"

streamshatter-1.0.0/streamshatter.py

@@ -0,0 +1,257 @@
+import asyncio
+import base64
+import hashlib
+import json
+from math import ceil, isfinite
+import os
+import random
+import shutil
+import time
+from urllib.parse import quote_plus
+import niquests
+
+session = niquests.AsyncSession()
+chunk_size = 1048576
+base_chunk = 65536
+COLOURS = ["\x1b[38;5;16m█"]
+COLOURS.extend(f"\x1b[38;5;{i}m█" for i in range(232, 256))
+COLOURS.append("\x1b[38;5;15m█")
+
+def shash(s): return base64.urlsafe_b64encode(hashlib.sha256(s if type(s) is bytes else str(s).encode("utf-8")).digest()).rstrip(b"==").decode("ascii")
+def uhash(s): return min([shash(s), quote_plus(s.removeprefix("https://"))], key=len)
+def header():
+	return {
+		"User-Agent": f"Mozilla/5.{random.randint(1, 9)} (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+		"DNT": "1",
+		"X-Forwarded-For": ".".join(str(random.randint(0, 255)) for _ in range(4)),
+	}
+def nth_file(tag, chunk=0):
+	return base64.urlsafe_b64encode(chunk.to_bytes(ceil(chunk.bit_length() / 8), "big")).rstrip(b"==").decode("ascii") + "~" + tag
+def box(i):
+	if i < 0:
+		return "\x1b[38;5;196m█"
+	return COLOURS[round(min(1, i) * (len(COLOURS) - 1))]
+def time_disp(s, rounded=True):
+	if not isfinite(s):
+		return str(s)
+	if rounded:
+		s = round(s)
+	output = str(s % 60)
+	if len(output) < 2:
+		output = "0" + output
+	if s >= 60:
+		temp = str((s // 60) % 60)
+		if len(temp) < 2 and s >= 3600:
+			temp = "0" + temp
+		output = temp + ":" + output
+		if s >= 3600:
+			temp = str((s // 3600) % 24)
+			if len(temp) < 2 and s >= 86400:
+				temp = "0" + temp
+			output = temp + ":" + output
+			if s >= 86400:
+				output = str(s // 86400) + ":" + output
+	else:
+		output = "0:" + output
+	return output
+def calc_bps(bps):
+	for suffix in ("bps", "kbps", "Mbps", "Gbps", "Tbps", "Pbps", "Ebps", "Zbps", "Ybps"):
+		bps = round(bps, 4)
+		if bps < 1000:
+			if bps.is_integer():
+				bps = int(bps)
+			return f"{bps} {suffix}"
+		bps /= 1000
+	return "ERR"
+def sample(arr, n):
+	while len(arr) > n * 2:
+		arr = [(a + b) / 2 for a, b in zip(arr[::2], arr[1::2])]
+	if len(arr) > n:
+		indices = [x * len(arr) / n for x in range(n)]
+		arr.append(arr[-1])
+		return [arr[int(i)] * (1 - i % 1) + arr[int(i) + 1] * (i % 1) for i in indices]
+	return arr
+def update_progress(ctx, force=False, use_original_timestamp=False):
+	ct = time.perf_counter()
+	if not force and ct - ctx["last"] < 0.1:
+		return
+	maxbar = 64
+	samples = [chunk[-1] / chunk[1] for chunk in ctx["chunkinfo"]]
+	s = "".join(map(box, sample(samples, maxbar)))
+	dt = ct - ctx["start"]
+	timer = time_disp(dt)
+	progress = sum(chunk[-1] for chunk in ctx["chunkinfo"])
+	percentage = round(progress / ctx["size"] * 100, 4)
+	if percentage.is_integer():
+		percentage = int(percentage)
+	if use_original_timestamp:
+		bps = ctx["size"] * 8 / dt
+	else:
+		bps = sum(chunk[3] / max(0.001, ct - chunk[2]) for chunk in ctx["chunkinfo"]) * 8
+	bpst = calc_bps(bps)
+	s2 = f" {timer} ({percentage}%, {bpst})"
+	chars = min(maxbar, len(ctx["chunkinfo"])) + len(s2)
+	s += "\x1b[38;5;7m" + s2
+	s += " " * (100 - chars)
+	print(s, end="\r")
+	if ctx["forkable"] and len(ctx["chunkinfo"]) < ctx["limit"] and ct - ctx["last_split"] > 1 and bps > ctx["last_bps"]:
+		ctx["last_bps"] = bps
+		ctx["last_split"] = time.perf_counter()
+		# Allow no more than 4 stalled/errored requests at a time
+		return sum(chunk[-1] <= 0 for chunk in ctx["chunkinfo"]) < 4
+	elif ct - ctx["last_split"] > 5:
+		ctx["last_bps"] = bps
+		ctx["last_split"] = time.perf_counter()
+
+async def write_request(ctx, chunk, resp, url, method, headers, data, filename):
+	file = os.path.join(ctx["cache_folder"], nth_file(uhash(url), len(ctx["chunkinfo"])))
+	ctx["chunkinfo"].append(chunk)
+	attempts = 0
+	fn = file + "~"
+	with open(fn, "wb+") as f:
+		while True:
+			timeout = (attempts + 1) * 5
+			try:
+				if not resp:
+					resp = await asyncio.wait_for(session.request(method, url, headers=headers, data=data, stream=True, timeout=timeout), timeout=timeout + 1)
+					resp.raise_for_status()
+				if "Range" in headers:
+					assert resp.headers["content-range"].split("/", 1)[0].split(None, 1)[-1] == headers["Range"].split("=", 1)[-1], "Server failed to serve range header as specified!"
+				chunk[-2] = time.perf_counter()
+				it = await asyncio.wait_for(resp.iter_content(base_chunk), timeout=timeout)
+				size = chunk[1]
+				try:
+					while True:
+						data = await asyncio.wait_for(it.__anext__(), timeout=timeout)
+						f.write(data)
+						chunk[-1] = min(max(len(data), chunk[-1] + len(data)), size)
+						chunk[3] += len(data)
+						split = update_progress(ctx)
+						if chunk[-1] == size:
+							break
+						if split and chunk[-1] + chunk_size < size:
+							start = chunk[0]
+							offset = round((chunk[-1] + size) / 2)
+							chunk2 = [start + offset, size - offset, time.perf_counter(), 0, 0]
+							rheaders = headers.copy()
+							rheaders["Range"] = f"bytes={start + offset}-{start + size - 1}"
+							fut = asyncio.create_task(write_request(ctx, chunk2, None, resp.url, method, rheaders, data, filename))
+							fut.start = chunk2[0]
+							ctx["workers"].append(fut)
+							ctx["workers"].sort(key=lambda fut: fut.start)
+							size = chunk[1] = offset
+							chunk[2] = time.perf_counter()
+							chunk[3] = len(data)
+				except (StopIteration, StopAsyncIteration):
+					pass
+				f.flush()
+				f.truncate(size)
+				f.seek(0, os.SEEK_END)
+				assert f.tell() == size, (f.tell, size)
+			except (TimeoutError, asyncio.TimeoutError):
+				pass
+			except Exception as ex:
+				print(repr(ex))
+			else:
+				chunk[-1] = size
+				chunk[2] = time.perf_counter()
+				chunk[3] = base_chunk
+				break
+			finally:
+				if resp:
+					try:
+						await asyncio.wait_for(resp.close(), timeout=timeout)
+					except Exception:
+						pass
+			resp = None
+			f.seek(0)
+			f.truncate(0)
+			chunk[-1] = -0.01
+			chunk[2] = time.perf_counter()
+			chunk[3] = 0
+			update_progress(ctx, force=True)
+			await asyncio.sleep((attempts + random.random()) ** 2 + 1)
+			attempts += 1
+			globals()["session"] = niquests.AsyncSession()
+	os.replace(fn, file)
+	return file
+
+async def parallel_request(url, method="get", headers={}, data=None, filename=None, cache_folder="", limit=1024):
+	t = time.perf_counter()
+	head = header()
+	head.update(headers)
+	resp = await session.request(method, url, headers=head, data=data, stream=True)
+	resp.raise_for_status()
+	filename = filename or resp.headers.get("attachment-filename") or url.rstrip("/").rsplit("/", 1)[-1].split("?", 1)[0]
+	try:
+		size = int(resp.headers.get("content-length") or resp.headers["content-range"].rsplit("/", 1)[-1])
+	except (KeyError, ValueError):
+		size = -1
+	chunk = [0, size, time.perf_counter(), 0, 0, 0]
+	single = limit <= 1 or size <= 0 or "bytes" not in resp.headers.get("accept-ranges", "").casefold()
+	ctx = dict(
+		url=url,
+		start=t,
+		last=0,
+		size=size,
+		last_bps=0,
+		last_split=0,
+		cache_folder=cache_folder,
+		limit=limit,
+		forkable=not single,
+		chunkinfo=[],
+		workers=[],
+	)
+	fut = asyncio.create_task(write_request(ctx, chunk, resp, url, method, headers, data, filename))
+	fut.start = 0
+	ctx["workers"].append(fut)
+	removes = []
+	try:
+		fn = filename + "~"
+		with open(fn, "ab") as f:
+			f.truncate(0)
+			while ctx["workers"]:
+				file = await ctx["workers"].pop(0)
+				with open(file, "rb") as g:
+					shutil.copyfileobj(g, f)
+				removes.append(file)
+		os.replace(fn, filename)
+		update_progress(ctx, force=True, use_original_timestamp=True)
+	finally:
+		for file in removes:
+			os.remove(file)
+
+
+try:
+	from importlib.metadata import version
+	__version__ = version("streamshatter")
+except Exception:
+	__version__ = "0.0.0-unknown"
+
+def main():
+	import argparse
+	parser = argparse.ArgumentParser(
+		prog="streamshatter",
+		description="Multiplexed chunked file downloader",
+	)
+	parser.add_argument("-V", '--version', action='version', version=f'%(prog)s {__version__}')
+	parser.add_argument("-H", '--headers', help="HTTP headers, interpreted as JSON", required=False, default="{}")
+	parser.add_argument("-c", '--cache-folder', help="Folder to store temporary files", required=False, default=os.path.join(__file__.replace("\\", "/").rsplit("/", 1)[0], "cache"))
+	parser.add_argument("-l", '--limit', help="Limits the amount of chunks to download", type=int, required=False, default=1024)
+	parser.add_argument("url", help="Target URL")
+	parser.add_argument("filename", help="Output filename", nargs="?", default="")
+	args = parser.parse_args()
+	if not os.path.exists(args.cache_folder):
+		os.mkdir(args.cache_folder)
+	if os.name == "nt":
+		os.system("color")
+	asyncio.run(parallel_request(
+		url=args.url,
+		filename=args.filename,
+		headers=json.loads(args.headers),
+		cache_folder=args.cache_folder,
+		limit=args.limit,
+	))
+
+if __name__ == "__main__":
+	main()