mithwire 0.51.2__tar.gz → 0.51.4__tar.gz

{mithwire-0.51.2/mithwire.egg-info → mithwire-0.51.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mithwire
-Version: 0.51.2
+Version: 0.51.4
 Summary: Dual-mode anti-detect browser framework — CDP-native stealth for local machines, binary-level patching for production servers. No WebDriver, no chromedriver.
 Author: Elendil Technologies
 Maintainer: Elendil Technologies

{mithwire-0.51.2 → mithwire-0.51.4}/mithwire/__init__.py

@@ -36,4 +36,4 @@ __all__ = [
     "run_stealth_diagnostic",
 ]
 
-__version__ = "0.51.2"
+__version__ = "0.51.4"

{mithwire-0.51.2 → mithwire-0.51.4}/mithwire/proxy/health.py

@@ -24,6 +24,12 @@ Notes on the implementation:
   ``http`` endpoint 301-redirects to it), so we point at the ``http`` URL and
   let urllib follow the redirect through the proxy automatically — that
   exercises the proxy's CONNECT + TLS path the same way the real browser will.
+* **Fallback for residential/mobile proxies**: many residential gateways don't
+  support CONNECT well (slow or timing out). When the primary HTTPS probe
+  fails on timeout or connection error (NOT auth 407), the probe automatically
+  retries with a plain-HTTP target (``ip-api.com/json/``) that requires only
+  an absolute-form GET — no CONNECT tunnel needed. The response is normalized
+  to the ipapi.is schema so downstream identity alignment is unaffected.
 * The synchronous ``urlopen`` runs in a worker thread; the surrounding
   ``asyncio.wait_for`` enforces an async-side deadline regardless of any
   socket-level timer.
@@ -49,13 +55,13 @@ from .config import ProxyConfig
 
 logger = logging.getLogger(__name__)
 
-# ipapi.is returns a JSON object with ``ip`` plus
-# ``location.{country,country_code,city,timezone,latitude,longitude}``. We use
-# the ``http://`` URL on purpose: urllib's ProxyHandler issues an absolute-form
-# GET to the proxy for plain HTTP, AND transparently follows the 301-to-https
-# the public service returns — so we end up exercising the proxy's CONNECT +
-# TLS path in production, which is exactly what the real browser does too.
+# Primary probe target.  ipapi.is now 301-redirects http → https.  When urllib
+# follows that redirect it attempts an HTTP CONNECT tunnel for the TLS leg,
+# which many residential/mobile proxies don't support on their HTTP port —
+# causing a silent hang.  We disable auto-redirect and, if the primary hangs,
+# fall back to ip-api.com which still serves plain HTTP without redirecting.
 _PROBE_TARGET_URL = "http://api.ipapi.is/"
+_FALLBACK_PROBE_URL = "http://ip-api.com/json/"
 
 _DEFAULT_TIMEOUT_SECONDS = 8.0
 
@@ -161,12 +167,38 @@ async def probe_proxy(
     :class:`ProxyHealthError` with a redacted, actionable message on any
     failure — bad host/port, refused TCP, HTTP 407 (wrong credentials), or an
     unparseable response. The browser never starts on failure.
+
+    For HTTP/HTTPS proxies the primary probe goes to ipapi.is (HTTPS via
+    CONNECT). If the CONNECT path times out or fails — common with
+    residential/mobile gateways — an automatic fallback probe hits a plain-HTTP
+    endpoint (ip-api.com) that only requires an absolute-form GET, no CONNECT.
+    Auth failures (407) are never retried via fallback.
     """
     timeout = max(1.0, float(timeout_seconds))
     if proxy.is_socks:
         await _check_tcp(proxy, timeout=timeout)
         return {}
-    return await _http_egress_probe(proxy, timeout=timeout)
+
+    try:
+        return await _http_egress_probe(proxy, timeout=timeout)
+    except ProxyHealthError as primary_exc:
+        if _is_auth_failure(primary_exc):
+            raise
+        logger.info(
+            "Primary probe (HTTPS/CONNECT) failed for %s: %s — "
+            "retrying with plain-HTTP fallback.",
+            proxy.redacted(),
+            primary_exc,
+        )
+        try:
+            return await _http_egress_probe_plaintext(proxy, timeout=timeout)
+        except ProxyHealthError:
+            raise primary_exc from None
+
+
+def _is_auth_failure(exc: ProxyHealthError) -> bool:
+    """True when the error clearly indicates bad credentials (407)."""
+    return "407" in str(exc)
 
 
 async def _check_tcp(proxy: ProxyConfig, *, timeout: float) -> None:
@@ -214,23 +246,107 @@ def _proxy_url_for_urllib(proxy: ProxyConfig) -> str:
     return f"http://{netloc}"
 
 
+class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
+    """Prevent urllib from following redirects (avoids CONNECT tunneling)."""
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):  # noqa: ARG002
+        return None
+
+
+def _normalize_ip_api_response(data: dict[str, Any]) -> dict[str, Any]:
+    """Map an ip-api.com response to the ipapi.is schema."""
+    return {
+        "ip": data.get("query") or data.get("ip", ""),
+        "location": {
+            "country": data.get("country"),
+            "country_code": data.get("countryCode") or data.get("country_code"),
+            "city": data.get("city"),
+            "timezone": data.get("timezone"),
+            "latitude": data.get("lat") or data.get("latitude"),
+            "longitude": data.get("lon") or data.get("longitude"),
+        },
+    }
+
+
+def _parse_egress_response(
+    proxy: ProxyConfig,
+    status: int,
+    body: bytes,
+    *,
+    schema: str = "ipapi",
+) -> dict[str, Any]:
+    """Validate and parse a successful egress probe response.
+
+    ``schema`` selects the normalisation: ``"ipapi"`` (api.ipapi.is, the
+    default) passes through as-is; ``"ip-api"`` maps ``http://ip-api.com``
+    fields to the canonical schema.
+    """
+    if status == 407:
+        raise ProxyHealthError(
+            f"Proxy {proxy.redacted()} rejected the supplied credentials "
+            "(HTTP 407 Proxy Authentication Required). Double-check username "
+            "and password."
+        )
+    if not (200 <= status < 300):
+        raise ProxyHealthError(
+            f"Proxy {proxy.redacted()} returned HTTP {status} for the egress "
+            f"probe."
+        )
+    if not body:
+        raise ProxyHealthError(
+            f"Proxy {proxy.redacted()} returned a 2xx response with an empty "
+            "body — egress probe could not read identity."
+        )
+
+    text = body.decode("utf-8", errors="replace").strip()
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError as exc:
+        snippet = text[:200].replace("\n", " ")
+        raise ProxyHealthError(
+            f"Proxy {proxy.redacted()} returned non-JSON for the egress probe "
+            f"(first 200 chars: {snippet!r})."
+        ) from exc
+
+    if not isinstance(data, dict):
+        raise ProxyHealthError(
+            f"Proxy {proxy.redacted()} returned a JSON value that is not an "
+            "object."
+        )
+
+    if schema == "ip-api":
+        data = _normalize_ip_api_response(data)
+
+    ip_field = data.get("ip")
+    if not (isinstance(ip_field, str) and ip_field.strip()):
+        raise ProxyHealthError(
+            f"Proxy {proxy.redacted()} responded but the egress probe could "
+            "not determine an exit IP (response did not match the expected "
+            "schema)."
+        )
+
+    return data
+
+
 async def _http_egress_probe(proxy: ProxyConfig, *, timeout: float) -> dict[str, Any]:
-    """Drive a GET to ipapi.is through the proxy and return the egress JSON.
+    """Drive a GET to an IP-info service through the proxy and return egress JSON.
 
-    Implementation lives in a sync helper so we can lean on urllib's proxy
-    support (absolute-form GET for HTTP targets, CONNECT + TLS for HTTPS
-    targets, transparent 301 follow). The helper runs in a worker thread; the
-    outer ``wait_for`` keeps the async timeout authoritative.
+    Tries ``api.ipapi.is`` first (with redirects disabled to avoid CONNECT
+    tunneling through proxies that only support absolute-form HTTP). Falls back
+    to ``ip-api.com`` (plain HTTP, no redirect) when the primary times out or
+    fails — a common scenario with residential/mobile HTTP proxies.
     """
     proxy_url = _proxy_url_for_urllib(proxy)
 
-    def _fetch() -> tuple[int, bytes]:
-        handler = urllib.request.ProxyHandler({"http": proxy_url, "https": proxy_url})
-        opener = urllib.request.build_opener(handler)
+    def _fetch(url: str, *, follow_redirects: bool = False) -> tuple[int, bytes]:
+        handlers: list = [urllib.request.ProxyHandler({"http": proxy_url, "https": proxy_url})]
+        if not follow_redirects:
+            handlers.append(_NoRedirectHandler)
+        opener = urllib.request.build_opener(*handlers)
         request = urllib.request.Request(
-            _PROBE_TARGET_URL,
+            url,
             headers={
-                "User-Agent": "mithwire-mcp/proxy-probe",
+                "User-Agent": "mithwire/proxy-probe",
                 "Accept": "application/json, */*",
                 "Connection": "close",
             },
@@ -240,29 +356,50 @@ async def _http_egress_probe(proxy: ProxyConfig, *, timeout: float) -> dict[str,
             with opener.open(request, timeout=timeout) as response:
                 return int(response.status), response.read()
         except urllib.error.HTTPError as exc:
-            # Capture the body too — proxies often explain a 4xx in HTML.
             try:
                 body = exc.read()
             except Exception:  # noqa: BLE001
                 body = b""
             return int(exc.code), body
 
+    primary_exc: Exception | None = None
+
+    # --- Attempt 1: api.ipapi.is (redirects disabled) ---
     try:
         status, body = await asyncio.wait_for(
-            asyncio.to_thread(_fetch),
-            timeout=timeout + 5.0,
+            asyncio.to_thread(_fetch, _PROBE_TARGET_URL, follow_redirects=False),
+            timeout=timeout + 2.0,
         )
+        if 300 <= status < 400:
+            logger.debug(
+                "Primary probe returned %d redirect — falling back to ip-api.com",
+                status,
+            )
+            raise ProxyHealthError("primary probe redirected")
+        return _parse_egress_response(proxy, status, body)
+    except (ProxyHealthError, asyncio.TimeoutError, urllib.error.URLError,
+            ConnectionError, OSError) as exc:
+        primary_exc = exc
+        logger.debug("Primary egress probe failed (%s), trying fallback", exc)
+
+    # --- Attempt 2: ip-api.com (plain HTTP, no redirect) ---
+    try:
+        status, body = await asyncio.wait_for(
+            asyncio.to_thread(_fetch, _FALLBACK_PROBE_URL, follow_redirects=True),
+            timeout=timeout + 2.0,
+        )
+        return _parse_egress_response(proxy, status, body, schema="ip-api")
     except asyncio.TimeoutError as exc:
         raise ProxyHealthError(
             f"Proxy {proxy.redacted()} did not complete the egress probe "
-            f"within {timeout:.1f}s."
+            f"within {timeout:.1f}s (both primary and fallback timed out)."
         ) from exc
     except urllib.error.URLError as exc:
         reason = exc.reason
         if isinstance(reason, socket.timeout):
             raise ProxyHealthError(
                 f"Proxy {proxy.redacted()} timed out during the egress probe "
-                f"(socket timeout)."
+                f"(socket timeout on fallback)."
             ) from exc
         raise ProxyHealthError(
             f"Could not reach proxy {proxy.redacted()}: {reason}"
@@ -272,50 +409,101 @@ async def _http_egress_probe(proxy: ProxyConfig, *, timeout: float) -> dict[str,
             f"Proxy {proxy.redacted()} failed mid-probe: {exc}"
         ) from exc
 
-    if status == 407:
-        raise ProxyHealthError(
-            f"Proxy {proxy.redacted()} rejected the supplied credentials "
-            "(HTTP 407 Proxy Authentication Required). Double-check username "
-            "and password."
-        )
-    if not (200 <= status < 300):
-        raise ProxyHealthError(
-            f"Proxy {proxy.redacted()} returned HTTP {status} for the egress "
-            f"probe."
-        )
 
-    if not body:
-        raise ProxyHealthError(
-            f"Proxy {proxy.redacted()} returned a 2xx response with an empty "
-            "body — egress probe could not read identity."
+async def _http_egress_probe_plaintext(
+    proxy: ProxyConfig, *, timeout: float
+) -> dict[str, Any]:
+    """Last-resort egress probe using raw sockets over plain HTTP.
+
+    When even urllib's ip-api.com fallback hangs (residential proxies that
+    keep connections alive regardless of ``Connection: close``), this probe
+    bypasses urllib entirely. It opens a raw TCP socket to the proxy, sends an
+    absolute-form ``GET`` with ``Proxy-Connection: Keep-Alive``, and reads
+    exactly ``Content-Length`` bytes — avoiding the EOF-wait hang.
+    """
+    import base64
+
+    def _fetch() -> tuple[int, bytes]:
+        auth_header = ""
+        if proxy.has_auth:
+            token = base64.b64encode(
+                f"{proxy.username}:{proxy.password}".encode()
+            ).decode("ascii")
+            auth_header = f"Proxy-Authorization: Basic {token}\r\n"
+
+        request = (
+            f"GET {_FALLBACK_PROBE_URL} HTTP/1.1\r\n"
+            f"Host: ip-api.com\r\n"
+            f"{auth_header}"
+            "User-Agent: mithwire-mcp/proxy-probe\r\n"
+            "Accept: application/json\r\n"
+            "Proxy-Connection: Keep-Alive\r\n"
+            "\r\n"
         )
 
-    text = body.decode("utf-8", errors="replace").strip()
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.settimeout(timeout)
+        try:
+            sock.connect((proxy.host, proxy.port))
+            sock.sendall(request.encode("ascii"))
+
+            buf = b""
+            while b"\r\n\r\n" not in buf:
+                chunk = sock.recv(4096)
+                if not chunk:
+                    raise ProxyHealthError(
+                        f"Proxy {proxy.redacted()} closed connection before "
+                        "sending response headers (raw-socket probe)."
+                    )
+                buf += chunk
+
+            header_end = buf.index(b"\r\n\r\n")
+            headers_raw = buf[:header_end].decode("ascii", errors="replace")
+            body_so_far = buf[header_end + 4 :]
+
+            status_code = 0
+            content_length = -1
+            for line in headers_raw.split("\r\n"):
+                if line.startswith("HTTP/"):
+                    status_code = int(line.split()[1])
+                if line.lower().startswith("content-length:"):
+                    content_length = int(line.split(":", 1)[1].strip())
+
+            if content_length >= 0:
+                while len(body_so_far) < content_length:
+                    chunk = sock.recv(4096)
+                    if not chunk:
+                        break
+                    body_so_far += chunk
+            else:
+                while True:
+                    chunk = sock.recv(4096)
+                    if not chunk:
+                        break
+                    body_so_far += chunk
+
+            return status_code, body_so_far
+        finally:
+            sock.close()
+
     try:
-        data = json.loads(text)
-    except json.JSONDecodeError as exc:
-        snippet = text[:200].replace("\n", " ")
+        status, body = await asyncio.wait_for(
+            asyncio.to_thread(_fetch),
+            timeout=timeout + 5.0,
+        )
+    except asyncio.TimeoutError as exc:
         raise ProxyHealthError(
-            f"Proxy {proxy.redacted()} returned non-JSON for the egress probe "
-            f"(first 200 chars: {snippet!r})."
+            f"Proxy {proxy.redacted()} did not complete the raw-socket egress "
+            f"probe within {timeout:.1f}s."
         ) from exc
-
-    if not isinstance(data, dict):
-        raise ProxyHealthError(
-            f"Proxy {proxy.redacted()} returned a JSON value that is not an "
-            "object."
-        )
-
-    if not (isinstance(data.get("ip"), str) and data["ip"].strip()):
-        # ipapi.is always includes the resolved ip on success; a missing/blank
-        # one means we either hit a captive portal or got a different service.
+    except ProxyHealthError:
+        raise
+    except (ConnectionError, OSError) as exc:
         raise ProxyHealthError(
-            f"Proxy {proxy.redacted()} responded but the egress probe could "
-            "not determine an exit IP (response did not match the expected "
-            "ipapi.is schema)."
-        )
+            f"Proxy {proxy.redacted()} failed during raw-socket probe: {exc}"
+        ) from exc
 
-    return data
+    return _parse_egress_response(proxy, status, body, schema="ip-api")
 
 
 def egress_summary(data: dict[str, Any] | None) -> dict[str, Any] | None:

{mithwire-0.51.2 → mithwire-0.51.4}/mithwire/stealth/controller.py

@@ -837,7 +837,12 @@ class Stealth:
                 # height; in headed (Xvfb) mode the real window geometry does
                 # this naturally, so we just set screen dims + DPR.
                 chrome_h = 85 + int(dpr * 10)  # 85-100 px, realistic range
-                vw = sw
+                # In headed mode the real OS window drives the layout viewport;
+                # width/height=0 disables the CDP override so innerWidth/Height
+                # follow the physical window while screen_width/screen_height
+                # and device_scale_factor still get spoofed.  In headless there
+                # is no real window, so we must set explicit dimensions.
+                vw = sw if self.headless else 0
                 vh = max(sh - chrome_h, sh // 2) if self.headless else 0
                 await self.tab.send(
                     emu.set_device_metrics_override(

{mithwire-0.51.2 → mithwire-0.51.4/mithwire.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mithwire
-Version: 0.51.2
+Version: 0.51.4
 Summary: Dual-mode anti-detect browser framework — CDP-native stealth for local machines, binary-level patching for production servers. No WebDriver, no chromedriver.
 Author: Elendil Technologies
 Maintainer: Elendil Technologies

{mithwire-0.51.2 → mithwire-0.51.4}/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "mithwire" # Required
 
-version = "0.51.2"  # Required
+version = "0.51.4"  # Required
 
 description = "Dual-mode anti-detect browser framework — CDP-native stealth for local machines, binary-level patching for production servers. No WebDriver, no chromedriver."
 readme = {file = "README.md", content-type = "text/markdown"} # Optional