speedtest-cli-ng 0.1.1__py3-none-any.whl

speedtest/engine/config.py

@@ -0,0 +1,120 @@
+"""
+Retrieves and parses the core configuration XML from speedtest.net.
+"""
+
+import math
+import xml.etree.ElementTree as ET
+from typing import Any
+
+from speedtest.exceptions import ConfigRetrievalError, SpeedtestConfigError
+from speedtest.http.request import build_request, catch_request
+from speedtest.http.response import get_response_stream
+from speedtest.utils.logger import logger
+
+__all__ = ["fetch_config"]
+
+
+def fetch_config(opener: Any) -> dict[str, Any]:
+    """
+    Download the speedtest.net configuration, parse the XML,
+    and return a standardized dictionary of settings.
+    """
+
+    headers = {"Accept-Encoding": "gzip"}
+    request = build_request(
+        "https://www.speedtest.net/speedtest-config.php",
+        headers=headers,
+    )
+
+    uh, e = catch_request(request, opener=opener)
+    if e or not uh:
+        raise ConfigRetrievalError(e) from e
+
+    with uh:
+        if int(getattr(uh, "code", 200)) != 200:
+            raise ConfigRetrievalError(f"HTTP Error {uh.code} while fetching config")  # type: ignore
+
+        try:
+            with get_response_stream(uh) as stream:
+                configxml = stream.read()
+        except (OSError, EOFError) as err:
+            raise ConfigRetrievalError(err) from err
+
+    logger.debug(f"Config XML:\n{configxml.decode(errors='ignore')}")
+
+    try:
+        root = ET.fromstring(configxml)
+    except ET.ParseError as err:
+        raise SpeedtestConfigError(f"Malformed speedtest.net configuration: {err}")
+
+    server_config_node = root.find("server-config")
+    download_node = root.find("download")
+    upload_node = root.find("upload")
+    client_node = root.find("client")
+
+    if not all(
+        [
+            server_config_node is not None,
+            download_node is not None,
+            upload_node is not None,
+            client_node is not None,
+        ]
+    ):
+        raise SpeedtestConfigError("Missing expected XML tags in the config payload.")
+
+    server_config = server_config_node.attrib  # type: ignore
+    download = download_node.attrib  # type: ignore
+    upload = upload_node.attrib  # type: ignore
+    client = client_node.attrib  # type: ignore
+
+    ignore_servers = [int(i) for i in server_config.get("ignoreids", "").split(",") if i.strip()]
+
+    ratio = int(upload.get("ratio", 5))
+    upload_max = int(upload.get("maxchunkcount", 50))
+    up_sizes = [32768, 65536, 131072, 262144, 524288, 1048576, 7340032]
+
+    slice_idx = max(0, ratio - 1)
+
+    sizes = {
+        "upload": up_sizes[slice_idx:],
+        "download": [350, 500, 750, 1000, 1500, 2000, 2500, 3000, 3500, 4000],
+    }
+
+    size_count = len(sizes["upload"])
+    if size_count == 0:
+        raise SpeedtestConfigError("Upload size list evaluated to empty based on configured ratio.")
+
+    upload_count = math.ceil(upload_max / size_count)
+
+    counts = {"upload": upload_count, "download": int(download.get("threadsperurl", 4))}
+
+    threads = {
+        "upload": int(upload.get("threads", 4)),
+        "download": int(server_config.get("threadcount", 4)) * 2,
+    }
+
+    length = {
+        "upload": int(upload.get("testlength", 10)),
+        "download": int(download.get("testlength", 10)),
+    }
+
+    try:
+        lat_lon = (float(client["lat"]), float(client["lon"]))
+    except (ValueError, KeyError) as err:
+        raise SpeedtestConfigError(
+            f"Unknown location: lat={client.get('lat')} lon={client.get('lon')}"
+        ) from err
+
+    parsed_config = {
+        "client": client,
+        "ignore_servers": ignore_servers,
+        "sizes": sizes,
+        "counts": counts,
+        "threads": threads,
+        "length": length,
+        "upload_max": upload_count * size_count,
+        "lat_lon": lat_lon,
+    }
+
+    logger.debug(f"Config:\n{parsed_config}")
+    return parsed_config

speedtest/engine/results.py

@@ -0,0 +1,195 @@
+"""
+Manages the aggregation, formatting, and submission of speedtest results.
+"""
+
+import csv
+import json
+from datetime import UTC, datetime
+from hashlib import md5
+from io import StringIO
+from typing import Any
+from urllib.parse import parse_qs, urlencode
+from urllib.request import OpenerDirector
+
+from speedtest.exceptions import ShareResultsConnectFailure, ShareResultsSubmitFailure
+from speedtest.http.handlers import build_opener
+from speedtest.http.request import build_request, catch_request
+
+__all__ = ["SpeedtestResults"]
+
+
+class SpeedtestResults:
+    """
+    Class for holding the results of a speedtest, including:
+
+    * Download speed
+    * Upload speed
+    * Ping/Latency to test server
+    * Data about the server that the test was run against
+
+    Additionally, this class can return result data as a dictionary or CSV,
+    as well as submit a POST of the result data to the speedtest.net API
+    to get a share results image link.
+    """
+
+    def __init__(
+        self,
+        download: float = 0.0,
+        upload: float = 0.0,
+        ping: float = 0.0,
+        server: dict[str, Any] | None = None,
+        client: dict[str, str] | None = None,
+        opener: OpenerDirector | None = None,
+    ):
+        self.download = download
+        self.upload = upload
+        self.ping = ping
+        self.server = server or {}
+        self.client = client or {}
+
+        self._share: str | None = None
+
+        # Generate a clean ISO 8601 UTC timestamp
+        self.timestamp = datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
+
+        self.bytes_received: int = 0
+        self.bytes_sent: int = 0
+
+        self._opener = opener or build_opener()
+
+    def __repr__(self) -> str:
+        return repr(self.to_dict())
+
+    def share(self) -> str:
+        """POST data to the speedtest.net API to obtain a share results link."""
+
+        if self._share:
+            return self._share
+
+        download = round(self.download / 1000.0)
+        ping = round(self.ping)
+        upload = round(self.upload / 1000.0)
+
+        hash_str = f"{ping}-{upload}-{download}-297aae72"
+        hash_val = md5(hash_str.encode()).hexdigest()
+
+        # We use a list of tuples instead of a dict because the speedtest API
+        # expects parameters in a strict sequential order. urlencode preserves this.
+        api_parameters = [
+            ("recommendedserverid", self.server.get("id", "")),
+            ("ping", ping),
+            ("screenresolution", ""),
+            ("promo", ""),
+            ("download", download),
+            ("screendpi", ""),
+            ("upload", upload),
+            ("testmethod", "http"),
+            ("hash", hash_val),
+            ("touchscreen", "none"),
+            ("startmode", "pingselect"),
+            ("accuracy", 1),
+            ("bytesreceived", self.bytes_received),
+            ("bytessent", self.bytes_sent),
+            ("serverid", self.server.get("id", "")),
+        ]
+
+        api_data = urlencode(api_parameters).encode()
+        headers = {"Referer": "https://c.speedtest.net/flash/speedtest.swf"}
+
+        request = build_request(
+            "https://www.speedtest.net/api/api.php",
+            data=api_data,
+            headers=headers,
+        )
+
+        f, e = catch_request(request, opener=self._opener)
+
+        if e or not f:
+            raise ShareResultsConnectFailure(e or "Failed to connect to Share API")
+
+        # Context manager strictly handles network cleanup
+        with f:
+            code = int(getattr(f, "code", 500))
+            if code != 200:
+                raise ShareResultsSubmitFailure(f"Could not submit results. HTTP {code}")
+
+            response = f.read()
+
+        # Safely decode ignoring corrupted bytes
+        qsargs = parse_qs(response.decode(errors="ignore"))
+        resultid = qsargs.get("resultid")
+
+        if not resultid or len(resultid) != 1:
+            raise ShareResultsSubmitFailure("Could not parse result ID from API response")
+
+        self._share = f"https://www.speedtest.net/result/{resultid[0]}.png"
+        return self._share
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return dictionary of result data cleanly formatted."""
+
+        return {
+            "download": self.download,
+            "upload": self.upload,
+            "ping": self.ping,
+            "server": self.server,
+            "timestamp": self.timestamp,
+            "bytes_sent": self.bytes_sent,
+            "bytes_received": self.bytes_received,
+            "share": self._share,
+            "client": self.client,
+        }
+
+    @staticmethod
+    def csv_header(delimiter: str = ",") -> str:
+        """Return CSV Headers."""
+
+        row = [
+            "Server ID",
+            "Sponsor",
+            "Server Name",
+            "Timestamp",
+            "Distance",
+            "Ping",
+            "Download",
+            "Upload",
+            "Share",
+            "IP Address",
+        ]
+
+        out = StringIO()
+        writer = csv.writer(out, delimiter=delimiter, lineterminator="")
+        writer.writerow(row)
+
+        return out.getvalue()
+
+    def csv(self, delimiter: str = ",") -> str:
+        """Return data in CSV format."""
+
+        data = self.to_dict()
+        server = data.get("server", {})
+
+        row = [
+            server.get("id", ""),
+            server.get("sponsor", ""),
+            server.get("name", ""),
+            data.get("timestamp", ""),
+            server.get("d", ""),
+            data.get("ping", ""),
+            data.get("download", ""),
+            data.get("upload", ""),
+            self._share or "",
+            self.client.get("ip", ""),
+        ]
+
+        out = StringIO()
+        writer = csv.writer(out, delimiter=delimiter, lineterminator="")
+        writer.writerow(row)
+
+        return out.getvalue()
+
+    def json(self, pretty: bool = False) -> str:
+        """Return data in JSON format."""
+
+        kwargs: dict[str, Any] = {"indent": 4, "sort_keys": True} if pretty else {}
+        return json.dumps(self.to_dict(), **kwargs)

speedtest/engine/servers.py

@@ -0,0 +1,189 @@
+"""
+Handles fetching, distance calculation, and latency ranking of speedtest.net servers.
+"""
+
+import math
+import time
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any
+from urllib.request import OpenerDirector
+
+from speedtest.exceptions import ServersRetrievalError, SpeedtestBestServerFailure
+from speedtest.http.request import build_request, build_user_agent, catch_request
+from speedtest.http.response import get_response_stream
+from speedtest.utils.logger import logger
+
+__all__ = ["fetch_servers", "get_best_server"]
+
+
+def _calculate_distance(origin: tuple[float, float], destination: tuple[float, float]) -> float:
+    """Determine distance between 2 sets of [lat, lon] in km using the Haversine formula."""
+
+    lat1, lon1 = origin
+    lat2, lon2 = destination
+    radius = 6371.0  # km
+
+    dlat = math.radians(lat2 - lat1)
+    dlon = math.radians(lon2 - lon1)
+
+    a = (math.sin(dlat / 2) ** 2) + (
+        math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * (math.sin(dlon / 2) ** 2)
+    )
+    c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+
+    return radius * c
+
+
+def fetch_servers(
+    opener: OpenerDirector,
+    lat_lon: tuple[float, float],
+    ignore_servers: list[int] | None = None,
+) -> dict[float, list[dict[str, Any]]]:
+    """
+    Fetch the server list from speedtest.net, parse the XML, calculate
+    distances from the client, and return a dictionary grouped and sorted by distance.
+    """
+
+    ignore_servers = ignore_servers or []
+    urls = [
+        "https://c.speedtest.net/speedtest-servers-static.php",
+        "https://www.speedtest.net/speedtest-servers-static.php",
+    ]
+
+    servers_dict: defaultdict[float, list[dict[str, Any]]] = defaultdict(list)
+
+    for url in urls:
+        request = build_request(url)
+        uh, e = catch_request(request, opener=opener)
+
+        if e or not uh:
+            continue
+
+        with uh:
+            if int(getattr(uh, "code", 500)) != 200:
+                continue
+
+            try:
+                with get_response_stream(uh) as stream:
+                    serversxml = stream.read()
+            except (OSError, EOFError):
+                continue
+
+        logger.debug(f"Servers XML:\n{serversxml.decode(errors='ignore')}")
+
+        try:
+            root = ET.fromstring(serversxml)
+        except ET.ParseError:
+            continue
+
+        for server in root.findall(".//server"):
+            attrib = server.attrib
+            server_id = int(attrib.get("id", 0))
+
+            if not server_id or server_id in ignore_servers:
+                continue
+
+            try:
+                lat_str, lon_str = attrib.get("lat"), attrib.get("lon")
+                if not lat_str or not lon_str:
+                    continue
+
+                server_lat_lon = (float(lat_str), float(lon_str))
+                distance = _calculate_distance(lat_lon, server_lat_lon)
+            except (ValueError, TypeError):
+                continue
+
+            attrib["d"] = distance  # type: ignore
+            servers_dict[distance].append(attrib)
+
+        if servers_dict:
+            break
+
+    if not servers_dict:
+        raise ServersRetrievalError("Failed to retrieve or parse speedtest server list.")
+
+    logger.debug(f"Discovered {sum(len(s) for s in servers_dict.values())} servers.")
+
+    return dict(servers_dict)
+
+
+def _ping_server(
+    server: dict[str, Any],
+    opener: OpenerDirector,
+    headers: dict[str, Any] | None = None,
+    pings: int = 3,
+) -> tuple[dict[str, Any], float]:
+    """Ping a single server multiple times and return the average latency."""
+
+    headers = headers or {}
+    url: str = server.get("url", "").replace("upload.php", "latency.txt")
+
+    if not url:
+        return server, 3600.0
+
+    latencies = []
+
+    for i in range(pings):
+        request = build_request(url, headers=headers, bump=str(i))
+        start = time.monotonic()
+
+        uh, e = catch_request(request, opener=opener)
+
+        if e or not uh:
+            latencies.append(3600.0)
+            continue
+
+        with uh:
+            if int(getattr(uh, "code", 500)) != 200:
+                latencies.append(3600.0)
+                continue
+
+            latency = time.monotonic() - start
+            latencies.append(latency)
+
+    avg_latency = sum(latencies) / len(latencies)
+    server["latency"] = avg_latency
+    server["latency_ms"] = avg_latency * 1000.0
+
+    return server, avg_latency
+
+
+def get_best_server(
+    closest_servers: list[dict[str, Any]], opener: OpenerDirector
+) -> dict[str, Any]:
+    """
+    Concurrently ping the closest servers to determine which has the lowest latency.
+    """
+
+    best_server = None
+    lowest_latency = float("inf")
+
+    user_agent = build_user_agent()
+    headers = {"User-Agent": user_agent}
+
+    with ThreadPoolExecutor(max_workers=len(closest_servers)) as executor:
+        future_to_server = {
+            executor.submit(_ping_server, server, opener, headers): server
+            for server in closest_servers
+        }
+
+        for future in as_completed(future_to_server):
+            try:
+                server, latency = future.result()
+                if latency < lowest_latency:
+                    lowest_latency = latency
+                    best_server = server
+            except Exception as e:
+                logger.debug(f"Server ping generated an exception: {e}")
+
+    if best_server is None:
+        raise SpeedtestBestServerFailure("Unable to determine best server via latency pings.")
+
+    logger.debug(
+        f"Best server selected: {best_server.get('sponsor')} "
+        f"({best_server.get('name')}) with latency {best_server.get('latency_ms', 0.0):.4f} ms"
+    )
+
+    return best_server

speedtest/engine/transfer.py

@@ -0,0 +1,157 @@
+"""
+Handles multi-threaded execution of download and upload tests.
+"""
+
+import os
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any
+from urllib.request import OpenerDirector, Request
+
+from speedtest.http.request import build_request
+from speedtest.http.workers import HTTPUploaderData, download_worker, upload_worker
+from speedtest.utils.logger import logger
+
+__all__ = ["run_download_test", "run_upload_test"]
+
+
+def run_download_test(
+    best_server_url: str,
+    config: dict[str, Any],
+    opener: OpenerDirector | None,
+    shutdown_event: threading.Event | None,
+    threads: int | None = None,
+) -> tuple[int, float]:
+    """
+    Execute a multi-threaded download speed test against the target server.
+    Returns a tuple of (bytes_received, download_speed_bps).
+    """
+
+    urls: list[str] = []
+
+    base_url = os.path.dirname(best_server_url)
+
+    sizes = config.get("sizes", {}).get("download", [])
+    counts = config.get("counts", {}).get("download", 4)
+
+    for size in sizes:
+        for _ in range(counts):
+            urls.append(f"{base_url}/random{size}x{size}.jpg")
+
+    requests = [build_request(url, bump=str(i)) for i, url in enumerate(urls)]
+    max_threads = threads or config.get("threads", {}).get("download", 4)
+
+    bytes_received = 0
+    start = time.monotonic()
+
+    test_length = config.get("length", {}).get("download", 10)
+
+    with ThreadPoolExecutor(max_workers=max_threads) as executor:
+        futures = [
+            executor.submit(
+                download_worker,
+                req,
+                start,
+                test_length,
+                opener=opener,
+                shutdown_event=shutdown_event,
+            )
+            for req in requests
+        ]
+
+        for future in as_completed(futures):
+            try:
+                bytes_received += future.result()
+            except Exception as e:
+                logger.debug(f"Download thread failed: {e}")
+
+    stop = time.monotonic()
+
+    elapsed = max(stop - start, 0.001)
+    download_speed = (bytes_received / elapsed) * 8.0
+
+    # Legacy behavior: Adapt upload thread count dynamically based on download performance
+    if download_speed > 100000 and "threads" in config:
+        config["threads"]["upload"] = 8
+
+    return bytes_received, download_speed
+
+
+def run_upload_test(
+    best_server_url: str,
+    config: dict[str, Any],
+    opener: OpenerDirector | None,
+    shutdown_event: threading.Event | None,
+    threads: int | None = None,
+) -> tuple[int, float]:
+    """
+    Execute a multi-threaded upload speed test against the target server.
+    Returns a tuple of (bytes_sent, upload_speed_bps).
+    """
+
+    raw_sizes = [
+        size
+        for size in config.get("sizes", {}).get("upload", [])
+        for _ in range(config.get("counts", {}).get("upload", 1))
+    ]
+
+    # Truncate to the exact maximum needed BEFORE looping to save RAM overhead
+    request_count = config.get("upload_max", len(raw_sizes))
+    sizes = raw_sizes[:request_count]
+
+    requests: list[Request] = []
+    payloads: list[HTTPUploaderData] = []
+
+    test_length = config.get("length", {}).get("upload", 10)
+
+    # Prepare requests and allocate payloads before starting the clock
+    for size in sizes:
+        data = HTTPUploaderData(
+            length=size,
+            start_time=0.0,  # Dummy value; will be updated right before execution
+            timeout=test_length,
+            shutdown_event=shutdown_event,
+        )
+
+        headers = {"Content-length": str(size)}
+        req = build_request(best_server_url, data, headers=headers)
+
+        requests.append(req)
+        payloads.append(data)
+
+    max_threads = threads or config.get("threads", {}).get("upload", 4)
+    bytes_sent = 0
+
+    start = time.monotonic()
+
+    with ThreadPoolExecutor(max_workers=max_threads) as executor:
+        futures = []
+
+        for req, payload in zip(requests, payloads):
+            # Stamp the real start time immediately before submission
+            payload.start_time = start
+
+            futures.append(
+                executor.submit(
+                    upload_worker,
+                    req,
+                    payload,
+                    test_length,
+                    opener=opener,
+                    shutdown_event=shutdown_event,
+                )
+            )
+
+        for future in as_completed(futures):
+            try:
+                bytes_sent += future.result()
+            except Exception as e:
+                logger.debug(f"Upload thread failed: {e}")
+
+    stop = time.monotonic()
+
+    elapsed = max(stop - start, 0.001)
+    upload_speed = (bytes_sent / elapsed) * 8.0
+
+    return bytes_sent, upload_speed