evo-cli 0.32.0__tar.gz → 0.33.0__tar.gz

{evo_cli-0.32.0 → evo_cli-0.33.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evo_cli
-Version: 0.32.0
+Version: 0.33.0
 Summary: Evolution CLI - a developer toolbox for setting up dev machines
 Author: maycuatroi
 Project-URL: Homepage, https://github.com/maycuatroi/evo-cli

evo_cli-0.33.0/evo_cli/VERSION

@@ -0,0 +1 @@
+0.33.0

{evo_cli-0.32.0 → evo_cli-0.33.0}/evo_cli/commands/route/__init__.py

@@ -15,7 +15,7 @@ edit can touch it.
 
 - `check <host>` - what stands in the way, and which of the two it is
 - `probe <host>` - every address for the name, ranked by what answers
-- `open <host>` - apply the fix the diagnosis calls for
+- `open <host>` - apply the fix the diagnosis calls for, and see to what its page loads
 - `close <host>` - undo it again
 - `status` - which fixes are currently in place
 

evo_cli-0.33.0/evo_cli/commands/route/_page.py

@@ -0,0 +1,96 @@
+"""What a page loads besides itself: a site is not open until its stylesheets and images are."""
+
+from __future__ import annotations
+
+import ipaddress
+import urllib.parse
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+from html.parser import HTMLParser
+
+from evo_cli.commands.route import _probe
+
+# Far past any <head>, and the rest of a long page repeats the same hosts.
+MAX_PAGE = 2 * 1024 * 1024
+WORKERS = 8
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
+    "(KHTML, like Gecko) Version/17.6 Safari/605.1.15"
+)
+
+# The <link> kinds a browser acts on by itself, unlike canonical or alternate.
+LINK_RELS = {"stylesheet", "preload", "modulepreload", "preconnect", "dns-prefetch", "icon"}
+SOURCE_TAGS = {"script", "img", "source", "video", "audio", "iframe"}
+
+# Under a country code these are a public suffix, not a site: example.com.vn, example.co.uk.
+COUNTRY_SECOND_LEVEL = {"ac", "co", "com", "edu", "gov", "net", "org"}
+
+
+class _AssetParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.urls = []
+
+    def handle_starttag(self, tag, attrs):
+        fields = dict(attrs)
+        if tag == "link":
+            if LINK_RELS & set((fields.get("rel") or "").lower().split()):
+                self.urls.append(fields.get("href"))
+        elif tag in SOURCE_TAGS:
+            self.urls.append(fields.get("src"))
+            self.urls.extend(part.split()[0] for part in (fields.get("srcset") or "").split(",") if part.strip())
+
+
+def _is_ip(name):
+    try:
+        ipaddress.ip_address(name)
+    except ValueError:
+        return False
+    return True
+
+
+def asset_hosts(html, base_url):
+    """The names a browser would connect to for this page, ignoring what only a click would reach."""
+    parser = _AssetParser()
+    parser.feed(html)
+    hosts = []
+    for url in parser.urls:
+        try:
+            name = urllib.parse.urlsplit(urllib.parse.urljoin(base_url, url or "")).hostname
+        except ValueError:
+            continue
+        if name and "." in name and not _is_ip(name) and name not in hosts:
+            hosts.append(name)
+    return hosts
+
+
+def fetch(host, proxy=None, timeout=6.0):
+    """Read the front page of ``host``, through ``proxy`` when there is one; returns where it ended up and the HTML."""
+    opener = urllib.request.build_opener(urllib.request.ProxyHandler({"http": proxy, "https": proxy} if proxy else {}))
+    request = urllib.request.Request(f"https://{host}/", headers={"User-Agent": USER_AGENT, "Accept": "text/html"})
+    with opener.open(request, timeout=timeout) as response:
+        return response.geturl(), response.read(MAX_PAGE).decode("utf-8", "replace")
+
+
+def registrable(host):
+    labels = host.split(".")
+    keep = 3 if len(labels[-1]) == 2 and labels[-2] in COUNTRY_SECOND_LEVEL else 2
+    return ".".join(labels[-keep:])
+
+
+def covers(opened, host):
+    return any(host == name or host.endswith(f".{name}") for name in opened)
+
+
+def blocked_hosts(host, proxy, opened, timeout=6.0):
+    """The hosts the page at ``host`` needs that a browser cannot reach, leaving out what ``opened`` already covers.
+
+    The site's own family is in the running because a page calls its API and sign-in
+    hosts from script, where no tag shows them.
+    """
+    final_url, html = fetch(host, proxy=proxy, timeout=timeout)
+    names = dict.fromkeys([*asset_hosts(html, final_url), registrable(host)])
+    names = [name for name in names if not covers([host, *opened], name)]
+    with ThreadPoolExecutor(max_workers=WORKERS) as pool:
+        alive = list(pool.map(lambda name: _probe.reachable(name, timeout=timeout), names))
+    return [name for name, ok in zip(names, alive) if not ok]

{evo_cli-0.32.0 → evo_cli-0.33.0}/evo_cli/commands/route/_probe.py

@@ -13,6 +13,7 @@ CONTROL_SNI = "example.com"
 
 OPEN = "open"
 DNS_POISONED = "dns-poisoned"
+ADDRESS_BLOCKED = "address-blocked"
 SNI_BLOCKED = "sni-blocked"
 IP_BLOCKED = "ip-blocked"
 UNREACHABLE = "unreachable"
@@ -20,6 +21,7 @@ UNREACHABLE = "unreachable"
 VERDICT_TEXT = {
     OPEN: "reachable - nothing in the way",
     DNS_POISONED: "DNS is poisoned, the route itself is open",
+    ADDRESS_BLOCKED: "the address your system resolves is blocked, another one works",
     SNI_BLOCKED: "DPI blocks the TLS handshake by server name",
     IP_BLOCKED: "the address itself is blocked or dead",
     UNREACHABLE: "no address answers at all",
@@ -103,24 +105,38 @@ def rank(host, candidates, timeout=6.0):
     return rows
 
 
+def reachable(host, timeout=6.0):
+    """Whether a browser going straight to what the system resolves gets a handshake."""
+    ips = [ip for ip in _dns.system_resolve(host) or [] if not _dns.is_bogus(ip)]
+    return any(tls_probe(ip, host, timeout=timeout)["tls"] for ip in ips)
+
+
+def candidates(answers):
+    """The system's own addresses first, since those are what a browser connects to, then every trusted one."""
+    system = [ip for ip in answers["system"] or [] if not _dns.is_bogus(ip)]
+    return system + [ip for ip in answers["trusted"] if ip not in system]
+
+
 def diagnose(host, timeout=6.0, include_vn=True):
     answers = _dns.gather(host, include_vn=include_vn)
     system_ips = answers["system"] or []
     trusted = answers["trusted"]
 
-    poisoned = bool(system_ips) and all(_dns.is_bogus(ip) for ip in system_ips)
+    # A name the system cannot resolve at all, while trusted resolvers can, is as dead to a browser as a bogus one.
+    poisoned = (bool(system_ips) and all(_dns.is_bogus(ip) for ip in system_ips)) or (not system_ips and bool(trusted))
     if not poisoned and system_ips and trusted:
         poisoned = not set(system_ips) & set(trusted) and any(_dns.is_bogus(ip) for ip in system_ips)
 
-    candidates = trusted or [ip for ip in system_ips if not _dns.is_bogus(ip)]
-    probes = rank(host, candidates, timeout=timeout) if candidates else []
+    addresses = candidates(answers)
+    probes = rank(host, addresses, timeout=timeout) if addresses else []
     working = [row for row in probes if row["tls"]]
+    system_ok = any(row["ip"] in system_ips for row in working)
 
     control = None
-    if candidates and not working:
-        control = tls_probe(candidates[0], CONTROL_SNI, timeout=timeout)
+    if addresses and not working:
+        control = tls_probe(addresses[0], CONTROL_SNI, timeout=timeout)
 
-    verdict = _verdict(candidates, probes, working, control, poisoned)
+    verdict = _verdict(addresses, probes, working, control, poisoned, system_ok)
 
     return {
         "host": host,
@@ -128,7 +144,7 @@ def diagnose(host, timeout=6.0, include_vn=True):
         "poisoned": poisoned,
         "system_ips": system_ips,
         "answers": answers["resolvers"],
-        "candidates": candidates,
+        "candidates": addresses,
         "probes": probes,
         "working": working,
         "control": control,
@@ -136,11 +152,14 @@ def diagnose(host, timeout=6.0, include_vn=True):
     }
 
 
-def _verdict(candidates, probes, working, control, poisoned):
+def _verdict(candidates, probes, working, control, poisoned, system_ok=True):
     if not candidates:
         return UNREACHABLE
     if working:
-        return DNS_POISONED if poisoned else OPEN
+        if poisoned:
+            return DNS_POISONED
+        # Some address works, but not the one the system hands the browser: the name only opens once it is pinned.
+        return OPEN if system_ok else ADDRESS_BLOCKED
     if control and control["tls"]:
         return SNI_BLOCKED
     if any(row["tcp"] for row in probes):

{evo_cli-0.32.0 → evo_cli-0.33.0}/evo_cli/commands/route/_spoofdpi.py

@@ -84,12 +84,13 @@ def opened():
         lines = hosts_path().read_text(encoding="utf-8").splitlines()
     except FileNotFoundError:
         return []
-    return [line for line in lines if line and not line.startswith(("#", "*."))]
+    return [line for line in lines if line and not line.startswith(("#", "*"))]
 
 
 def _write_hosts(hosts):
     home().mkdir(parents=True, exist_ok=True)
-    lines = [pattern for host in sorted(set(hosts)) for pattern in (host, f"*.{host}")]
+    # `*` stands for one label and `**` for any number: a CDN name like store.fastly.example.com needs the second.
+    lines = [pattern for host in sorted(set(hosts)) for pattern in (host, f"**.{host}")]
     hosts_path().write_text("".join(f"{line}\n" for line in lines), encoding="utf-8")
 
 
@@ -310,15 +311,15 @@ def service_state():
     }
 
 
-def ensure(host):
-    """Put ``host`` through the bypass: install SpoofDPI, (re)start its agent, point the system proxy at it."""
+def ensure(*hosts):
+    """Put ``hosts`` through the bypass: install SpoofDPI, (re)start its agent, point the system proxy at it."""
     conflicts = _proxy_conflicts()
     if conflicts:
         raise RuntimeError(f"A proxy is already set on {', '.join(conflicts)}; evo will not replace it.")
     exe = executable() or install()
     home().mkdir(parents=True, exist_ok=True)
     config_path().write_text(CONFIG, encoding="utf-8")
-    _write_hosts([*opened(), host])
+    _write_hosts([*opened(), *hosts])
     if not _start(reload=_write_agent(exe)):
         return service_state()
     set_system_proxy()

{evo_cli-0.32.0 → evo_cli-0.33.0}/evo_cli/commands/route/check.py

@@ -24,6 +24,7 @@ EPILOG = Text.from_markup(
 VERDICT_STYLE = {
     _probe.OPEN: "success",
     _probe.DNS_POISONED: "warning",
+    _probe.ADDRESS_BLOCKED: "warning",
     _probe.SNI_BLOCKED: "error",
     _probe.IP_BLOCKED: "error",
     _probe.UNREACHABLE: "error",
@@ -32,6 +33,7 @@ VERDICT_STYLE = {
 ADVICE = {
     _probe.OPEN: "Nothing to do.",
     _probe.DNS_POISONED: "Run `evo route open {host}` to pin the working address, or fix DNS at the source.",
+    _probe.ADDRESS_BLOCKED: "Run `evo route open {host}` to pin the working address.",
     _probe.SNI_BLOCKED: "hosts and DNS cannot help here. Run `evo route open {host}` to bring up the DPI bypass.",
     _probe.IP_BLOCKED: "The address answers TCP but nothing completes - you need a tunnel or VPN.",
     _probe.UNREACHABLE: "No address resolved or answered. Check the name, then your link.",
@@ -92,6 +94,9 @@ def render(report):
         success(f"{host} is reachable. {message}")
     elif verdict == _probe.DNS_POISONED:
         warning(f"DNS hands back a bogus address for {host}, but {report['best']} works. {message}")
+    elif verdict == _probe.ADDRESS_BLOCKED:
+        dead = ", ".join(report["system_ips"])
+        warning(f"{host} resolves to {dead}, which is blocked, but {report['best']} works. {message}")
     else:
         error(f"{host}: {_probe.VERDICT_TEXT[verdict]}. {message}")
 

{evo_cli-0.32.0 → evo_cli-0.33.0}/evo_cli/commands/route/fix.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
+import http.client
 import sys
 
 import rich_click as click
 from rich.text import Text
 
-from evo_cli.commands.route import _dpi, _hosts, _probe, _spoofdpi
+from evo_cli.commands.route import _dpi, _hosts, _page, _probe, _spoofdpi
 from evo_cli.console import console, error, info, step, success, warning
 
 OPEN_EPILOG = Text.from_markup(
@@ -13,9 +14,12 @@ OPEN_EPILOG = Text.from_markup(
     "  [cyan]evo route open store.steampowered.com[/cyan]     pick the fix that fits\n"
     "  [cyan]evo route open medium.com --dry-run[/cyan]       say what it would do\n"
     "  [cyan]evo route open x.com --method hosts[/cyan]       force the address pin\n"
-    "  [cyan]evo route open x.com --method dpi[/cyan]         force the DPI bypass\n\n"
+    "  [cyan]evo route open x.com --method dpi[/cyan]         force the DPI bypass\n"
+    "  [cyan]evo route open x.com --host-only[/cyan]          leave what its page loads alone\n\n"
     "[dim]Diagnoses first, then applies only the fix that matches: pinning an address\n"
     "for a poisoned name, or the packet-level bypass for a name-keyed block.\n"
+    "Then reads the page: the bypass takes in the blocked hosts it loads from,\n"
+    "whole domains at a time because CDNs rotate; a pin only names them.\n"
     "Needs administrator rights because it edits hosts / installs a service;\n"
     "on macOS the bypass is SpoofDPI as your own LaunchAgent and needs no sudo.[/dim]"
 )
@@ -98,6 +102,51 @@ def _apply_dpi(report, dry_run):
     return state
 
 
+def _page_hosts(host, proxy, timeout):
+    """The blocked hosts ``host``'s page needs, or None when the page cannot be read."""
+    with console.status(f"[info]reading the page at {host}...[/info]", spinner="dots"):
+        try:
+            return _page.blocked_hosts(host, proxy, _spoofdpi.opened(), timeout=timeout)
+        except (OSError, http.client.HTTPException, ValueError) as exc:
+            warning(f"Could not read the page at {host}, so what it loads is unchecked: {exc}")
+            return None
+
+
+def _open_page_hosts(host, timeout):
+    """The macOS bypass lists hosts by name, so everything the page loads has to go on the list too."""
+    proxy = (_spoofdpi.LISTEN_HOST, _spoofdpi.LISTEN_PORT)
+    blocked = _page_hosts(host, f"http://{_spoofdpi.listen()}", timeout)
+    if blocked is None:
+        return
+    if not blocked:
+        success(f"Nothing the page at {host} loads is blocked")
+        return
+
+    info(f"The page at {host} also needs {', '.join(blocked)}")
+    domains = sorted({_page.registrable(name) for name in blocked})
+    try:
+        state = _spoofdpi.ensure(*domains)
+    except RuntimeError as exc:
+        error(str(exc))
+        sys.exit(1)
+    if not state["running"]:
+        error(f"SpoofDPI would not restart - see {_spoofdpi.log_path()}")
+        sys.exit(1)
+
+    with console.status("[info]re-checking through the proxy...[/info]", spinner="dots"):
+        failed = [name for name in blocked if not _probe.proxy_probe(name, proxy, timeout=timeout)["tls"]]
+    if failed:
+        warning(f"Still failing through the bypass: {', '.join(failed)}. See {_spoofdpi.log_path()}")
+    success(f"Added {', '.join(domains)} to the bypass - `evo route close <host>` takes one back out")
+
+
+def _warn_page_hosts(host, timeout):
+    """A pin cannot reach past the name it pins, and the bypass will not start under sudo, so this only reports."""
+    blocked = _page_hosts(host, None, timeout)
+    if blocked:
+        warning(f"The page at {host} also needs {', '.join(blocked)} - run `evo route open <host>` on each")
+
+
 @click.command("open", epilog=OPEN_EPILOG)
 @click.argument("host")
 @click.option(
@@ -109,14 +158,16 @@ def _apply_dpi(report, dry_run):
 )
 @click.option("-n", "--dry-run", is_flag=True, help="Say what would change, change nothing.")
 @click.option("-t", "--timeout", default=6.0, show_default=True, help="Per-connection timeout (seconds).")
-def open_route(host, method, dry_run, timeout):
+@click.option("--host-only", is_flag=True, help="Open HOST alone, not what its page loads from.")
+def open_route(host, method, dry_run, timeout, host_only):
     """Open a route to `HOST` by applying the fix its diagnosis calls for.
 
     A poisoned name gets its working address pinned in hosts. A name-keyed DPI
     block gets the packet-level bypass, because no amount of hosts or DNS
     editing survives a filter that reads the server name out of the handshake.
 
-    Re-checks afterwards and reports whether the route actually opened.
+    Re-checks afterwards and reports whether the route actually opened, then
+    reads the page, because a site with its CDN still blocked is not open.
     """
     step(f"evo route open {host}")
 
@@ -130,7 +181,7 @@ def open_route(host, method, dry_run, timeout):
         if verdict == _probe.OPEN:
             success(f"{host} is already reachable - nothing to do.")
             return report
-        if verdict == _probe.DNS_POISONED:
+        if verdict in (_probe.DNS_POISONED, _probe.ADDRESS_BLOCKED):
             method = "hosts"
         elif verdict == _probe.SNI_BLOCKED:
             method = "dpi"
@@ -144,15 +195,22 @@ def open_route(host, method, dry_run, timeout):
         _apply_dpi(report, dry_run)
 
     if dry_run:
+        if method == "dpi" and _dpi.is_macos() and not host_only:
+            info("Would then read its page and add the blocked hosts it loads from")
         return report
 
     step("Verify")
     if method == "dpi" and _dpi.is_macos():
-        return _verify_through_proxy(host, timeout)
+        row = _verify_through_proxy(host, timeout)
+        if row["tls"] and not host_only:
+            _open_page_hosts(host, timeout)
+        return row
     with console.status(f"[info]re-checking {host}...[/info]", spinner="dots"):
         after = _probe.diagnose(host, timeout=timeout)
     if after["verdict"] in (_probe.OPEN, _probe.DNS_POISONED) and after["working"]:
         success(f"{host} is reachable now ({after['best']})")
+        if method == "hosts" and not host_only:
+            _warn_page_hosts(host, timeout)
     else:
         warning(f"{host} still reports [bold]{after['verdict']}[/bold]. Run `evo route check {host}` for detail.")
     return after

{evo_cli-0.32.0 → evo_cli-0.33.0}/evo_cli/commands/route/probe.py

@@ -37,7 +37,7 @@ def probe(host, timeout, show_all, no_vn, as_json):
         with console.status(f"[info]collecting addresses for {host}...[/info]", spinner="dots"):
             answers = _dns.gather(host, include_vn=not no_vn)
 
-        candidates = answers["trusted"] or [ip for ip in (answers["system"] or []) if not _dns.is_bogus(ip)]
+        candidates = _probe.candidates(answers)
         if not candidates:
             error(f"No usable address for {host}.")
             sys.exit(1)
@@ -61,13 +61,13 @@ def probe(host, timeout, show_all, no_vn, as_json):
     table.add_column("time")
     table.add_column("seen from")
 
-    origin = _origin_map(answers["resolvers"])
+    origin = _origin_map({"system": answers["system"], **answers["resolvers"]})
     for row in shown:
         table.add_row(
             row["ip"],
             "[success]ok[/success]" if row["tls"] else "[error]blocked[/error]",
             f"{row['ms']:.0f}ms" if row["ms"] is not None else "-",
-            ", ".join(origin.get(row["ip"], [])) or "[dim]system[/dim]",
+            ", ".join(origin.get(row["ip"], [])),
         )
     console.print()
     console.print(table)

{evo_cli-0.32.0 → evo_cli-0.33.0}/evo_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evo_cli
-Version: 0.32.0
+Version: 0.33.0
 Summary: Evolution CLI - a developer toolbox for setting up dev machines
 Author: maycuatroi
 Project-URL: Homepage, https://github.com/maycuatroi/evo-cli

{evo_cli-0.32.0 → evo_cli-0.33.0}/evo_cli.egg-info/SOURCES.txt

@@ -82,6 +82,7 @@ evo_cli/commands/route/__init__.py
 evo_cli/commands/route/_dns.py
 evo_cli/commands/route/_dpi.py
 evo_cli/commands/route/_hosts.py
+evo_cli/commands/route/_page.py
 evo_cli/commands/route/_probe.py
 evo_cli/commands/route/_spoofdpi.py
 evo_cli/commands/route/check.py