evo-cli 0.32.1__tar.gz → 0.33.0__tar.gz

{evo_cli-0.32.1 → evo_cli-0.33.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evo_cli
-Version: 0.32.1
+Version: 0.33.0
 Summary: Evolution CLI - a developer toolbox for setting up dev machines
 Author: maycuatroi
 Project-URL: Homepage, https://github.com/maycuatroi/evo-cli

evo_cli-0.33.0/evo_cli/VERSION

@@ -0,0 +1 @@
+0.33.0

{evo_cli-0.32.1 → evo_cli-0.33.0}/evo_cli/commands/route/__init__.py

@@ -15,7 +15,7 @@ edit can touch it.
 
 - `check <host>` - what stands in the way, and which of the two it is
 - `probe <host>` - every address for the name, ranked by what answers
-- `open <host>` - apply the fix the diagnosis calls for
+- `open <host>` - apply the fix the diagnosis calls for, and see to what its page loads
 - `close <host>` - undo it again
 - `status` - which fixes are currently in place
 

evo_cli-0.33.0/evo_cli/commands/route/_page.py

@@ -0,0 +1,96 @@
+"""What a page loads besides itself: a site is not open until its stylesheets and images are."""
+
+from __future__ import annotations
+
+import ipaddress
+import urllib.parse
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+from html.parser import HTMLParser
+
+from evo_cli.commands.route import _probe
+
+# Far past any <head>, and the rest of a long page repeats the same hosts.
+MAX_PAGE = 2 * 1024 * 1024
+WORKERS = 8
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
+    "(KHTML, like Gecko) Version/17.6 Safari/605.1.15"
+)
+
+# The <link> kinds a browser acts on by itself, unlike canonical or alternate.
+LINK_RELS = {"stylesheet", "preload", "modulepreload", "preconnect", "dns-prefetch", "icon"}
+SOURCE_TAGS = {"script", "img", "source", "video", "audio", "iframe"}
+
+# Under a country code these are a public suffix, not a site: example.com.vn, example.co.uk.
+COUNTRY_SECOND_LEVEL = {"ac", "co", "com", "edu", "gov", "net", "org"}
+
+
+class _AssetParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.urls = []
+
+    def handle_starttag(self, tag, attrs):
+        fields = dict(attrs)
+        if tag == "link":
+            if LINK_RELS & set((fields.get("rel") or "").lower().split()):
+                self.urls.append(fields.get("href"))
+        elif tag in SOURCE_TAGS:
+            self.urls.append(fields.get("src"))
+            self.urls.extend(part.split()[0] for part in (fields.get("srcset") or "").split(",") if part.strip())
+
+
+def _is_ip(name):
+    try:
+        ipaddress.ip_address(name)
+    except ValueError:
+        return False
+    return True
+
+
+def asset_hosts(html, base_url):
+    """The names a browser would connect to for this page, ignoring what only a click would reach."""
+    parser = _AssetParser()
+    parser.feed(html)
+    hosts = []
+    for url in parser.urls:
+        try:
+            name = urllib.parse.urlsplit(urllib.parse.urljoin(base_url, url or "")).hostname
+        except ValueError:
+            continue
+        if name and "." in name and not _is_ip(name) and name not in hosts:
+            hosts.append(name)
+    return hosts
+
+
+def fetch(host, proxy=None, timeout=6.0):
+    """Read the front page of ``host``, through ``proxy`` when there is one; returns where it ended up and the HTML."""
+    opener = urllib.request.build_opener(urllib.request.ProxyHandler({"http": proxy, "https": proxy} if proxy else {}))
+    request = urllib.request.Request(f"https://{host}/", headers={"User-Agent": USER_AGENT, "Accept": "text/html"})
+    with opener.open(request, timeout=timeout) as response:
+        return response.geturl(), response.read(MAX_PAGE).decode("utf-8", "replace")
+
+
+def registrable(host):
+    labels = host.split(".")
+    keep = 3 if len(labels[-1]) == 2 and labels[-2] in COUNTRY_SECOND_LEVEL else 2
+    return ".".join(labels[-keep:])
+
+
+def covers(opened, host):
+    return any(host == name or host.endswith(f".{name}") for name in opened)
+
+
+def blocked_hosts(host, proxy, opened, timeout=6.0):
+    """The hosts the page at ``host`` needs that a browser cannot reach, leaving out what ``opened`` already covers.
+
+    The site's own family is in the running because a page calls its API and sign-in
+    hosts from script, where no tag shows them.
+    """
+    final_url, html = fetch(host, proxy=proxy, timeout=timeout)
+    names = dict.fromkeys([*asset_hosts(html, final_url), registrable(host)])
+    names = [name for name in names if not covers([host, *opened], name)]
+    with ThreadPoolExecutor(max_workers=WORKERS) as pool:
+        alive = list(pool.map(lambda name: _probe.reachable(name, timeout=timeout), names))
+    return [name for name, ok in zip(names, alive) if not ok]

{evo_cli-0.32.1 → evo_cli-0.33.0}/evo_cli/commands/route/_probe.py

@@ -105,6 +105,12 @@ def rank(host, candidates, timeout=6.0):
     return rows
 
 
+def reachable(host, timeout=6.0):
+    """Whether a browser going straight to what the system resolves gets a handshake."""
+    ips = [ip for ip in _dns.system_resolve(host) or [] if not _dns.is_bogus(ip)]
+    return any(tls_probe(ip, host, timeout=timeout)["tls"] for ip in ips)
+
+
 def candidates(answers):
     """The system's own addresses first, since those are what a browser connects to, then every trusted one."""
     system = [ip for ip in answers["system"] or [] if not _dns.is_bogus(ip)]

{evo_cli-0.32.1 → evo_cli-0.33.0}/evo_cli/commands/route/_spoofdpi.py

@@ -84,12 +84,13 @@ def opened():
         lines = hosts_path().read_text(encoding="utf-8").splitlines()
     except FileNotFoundError:
         return []
-    return [line for line in lines if line and not line.startswith(("#", "*."))]
+    return [line for line in lines if line and not line.startswith(("#", "*"))]
 
 
 def _write_hosts(hosts):
     home().mkdir(parents=True, exist_ok=True)
-    lines = [pattern for host in sorted(set(hosts)) for pattern in (host, f"*.{host}")]
+    # `*` stands for one label and `**` for any number: a CDN name like store.fastly.example.com needs the second.
+    lines = [pattern for host in sorted(set(hosts)) for pattern in (host, f"**.{host}")]
     hosts_path().write_text("".join(f"{line}\n" for line in lines), encoding="utf-8")
 
 
@@ -310,15 +311,15 @@ def service_state():
     }
 
 
-def ensure(host):
-    """Put ``host`` through the bypass: install SpoofDPI, (re)start its agent, point the system proxy at it."""
+def ensure(*hosts):
+    """Put ``hosts`` through the bypass: install SpoofDPI, (re)start its agent, point the system proxy at it."""
     conflicts = _proxy_conflicts()
     if conflicts:
         raise RuntimeError(f"A proxy is already set on {', '.join(conflicts)}; evo will not replace it.")
     exe = executable() or install()
     home().mkdir(parents=True, exist_ok=True)
     config_path().write_text(CONFIG, encoding="utf-8")
-    _write_hosts([*opened(), host])
+    _write_hosts([*opened(), *hosts])
     if not _start(reload=_write_agent(exe)):
         return service_state()
     set_system_proxy()

{evo_cli-0.32.1 → evo_cli-0.33.0}/evo_cli/commands/route/fix.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
+import http.client
 import sys
 
 import rich_click as click
 from rich.text import Text
 
-from evo_cli.commands.route import _dpi, _hosts, _probe, _spoofdpi
+from evo_cli.commands.route import _dpi, _hosts, _page, _probe, _spoofdpi
 from evo_cli.console import console, error, info, step, success, warning
 
 OPEN_EPILOG = Text.from_markup(
@@ -13,9 +14,12 @@ OPEN_EPILOG = Text.from_markup(
     "  [cyan]evo route open store.steampowered.com[/cyan]     pick the fix that fits\n"
     "  [cyan]evo route open medium.com --dry-run[/cyan]       say what it would do\n"
     "  [cyan]evo route open x.com --method hosts[/cyan]       force the address pin\n"
-    "  [cyan]evo route open x.com --method dpi[/cyan]         force the DPI bypass\n\n"
+    "  [cyan]evo route open x.com --method dpi[/cyan]         force the DPI bypass\n"
+    "  [cyan]evo route open x.com --host-only[/cyan]          leave what its page loads alone\n\n"
     "[dim]Diagnoses first, then applies only the fix that matches: pinning an address\n"
     "for a poisoned name, or the packet-level bypass for a name-keyed block.\n"
+    "Then reads the page: the bypass takes in the blocked hosts it loads from,\n"
+    "whole domains at a time because CDNs rotate; a pin only names them.\n"
     "Needs administrator rights because it edits hosts / installs a service;\n"
     "on macOS the bypass is SpoofDPI as your own LaunchAgent and needs no sudo.[/dim]"
 )
@@ -98,6 +102,51 @@ def _apply_dpi(report, dry_run):
     return state
 
 
+def _page_hosts(host, proxy, timeout):
+    """The blocked hosts ``host``'s page needs, or None when the page cannot be read."""
+    with console.status(f"[info]reading the page at {host}...[/info]", spinner="dots"):
+        try:
+            return _page.blocked_hosts(host, proxy, _spoofdpi.opened(), timeout=timeout)
+        except (OSError, http.client.HTTPException, ValueError) as exc:
+            warning(f"Could not read the page at {host}, so what it loads is unchecked: {exc}")
+            return None
+
+
+def _open_page_hosts(host, timeout):
+    """The macOS bypass lists hosts by name, so everything the page loads has to go on the list too."""
+    proxy = (_spoofdpi.LISTEN_HOST, _spoofdpi.LISTEN_PORT)
+    blocked = _page_hosts(host, f"http://{_spoofdpi.listen()}", timeout)
+    if blocked is None:
+        return
+    if not blocked:
+        success(f"Nothing the page at {host} loads is blocked")
+        return
+
+    info(f"The page at {host} also needs {', '.join(blocked)}")
+    domains = sorted({_page.registrable(name) for name in blocked})
+    try:
+        state = _spoofdpi.ensure(*domains)
+    except RuntimeError as exc:
+        error(str(exc))
+        sys.exit(1)
+    if not state["running"]:
+        error(f"SpoofDPI would not restart - see {_spoofdpi.log_path()}")
+        sys.exit(1)
+
+    with console.status("[info]re-checking through the proxy...[/info]", spinner="dots"):
+        failed = [name for name in blocked if not _probe.proxy_probe(name, proxy, timeout=timeout)["tls"]]
+    if failed:
+        warning(f"Still failing through the bypass: {', '.join(failed)}. See {_spoofdpi.log_path()}")
+    success(f"Added {', '.join(domains)} to the bypass - `evo route close <host>` takes one back out")
+
+
+def _warn_page_hosts(host, timeout):
+    """A pin cannot reach past the name it pins, and the bypass will not start under sudo, so this only reports."""
+    blocked = _page_hosts(host, None, timeout)
+    if blocked:
+        warning(f"The page at {host} also needs {', '.join(blocked)} - run `evo route open <host>` on each")
+
+
 @click.command("open", epilog=OPEN_EPILOG)
 @click.argument("host")
 @click.option(
@@ -109,14 +158,16 @@ def _apply_dpi(report, dry_run):
 )
 @click.option("-n", "--dry-run", is_flag=True, help="Say what would change, change nothing.")
 @click.option("-t", "--timeout", default=6.0, show_default=True, help="Per-connection timeout (seconds).")
-def open_route(host, method, dry_run, timeout):
+@click.option("--host-only", is_flag=True, help="Open HOST alone, not what its page loads from.")
+def open_route(host, method, dry_run, timeout, host_only):
     """Open a route to `HOST` by applying the fix its diagnosis calls for.
 
     A poisoned name gets its working address pinned in hosts. A name-keyed DPI
     block gets the packet-level bypass, because no amount of hosts or DNS
     editing survives a filter that reads the server name out of the handshake.
 
-    Re-checks afterwards and reports whether the route actually opened.
+    Re-checks afterwards and reports whether the route actually opened, then
+    reads the page, because a site with its CDN still blocked is not open.
     """
     step(f"evo route open {host}")
 
@@ -144,15 +195,22 @@ def open_route(host, method, dry_run, timeout):
         _apply_dpi(report, dry_run)
 
     if dry_run:
+        if method == "dpi" and _dpi.is_macos() and not host_only:
+            info("Would then read its page and add the blocked hosts it loads from")
         return report
 
     step("Verify")
     if method == "dpi" and _dpi.is_macos():
-        return _verify_through_proxy(host, timeout)
+        row = _verify_through_proxy(host, timeout)
+        if row["tls"] and not host_only:
+            _open_page_hosts(host, timeout)
+        return row
     with console.status(f"[info]re-checking {host}...[/info]", spinner="dots"):
         after = _probe.diagnose(host, timeout=timeout)
     if after["verdict"] in (_probe.OPEN, _probe.DNS_POISONED) and after["working"]:
         success(f"{host} is reachable now ({after['best']})")
+        if method == "hosts" and not host_only:
+            _warn_page_hosts(host, timeout)
     else:
         warning(f"{host} still reports [bold]{after['verdict']}[/bold]. Run `evo route check {host}` for detail.")
     return after

{evo_cli-0.32.1 → evo_cli-0.33.0}/evo_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evo_cli
-Version: 0.32.1
+Version: 0.33.0
 Summary: Evolution CLI - a developer toolbox for setting up dev machines
 Author: maycuatroi
 Project-URL: Homepage, https://github.com/maycuatroi/evo-cli

{evo_cli-0.32.1 → evo_cli-0.33.0}/evo_cli.egg-info/SOURCES.txt

@@ -82,6 +82,7 @@ evo_cli/commands/route/__init__.py
 evo_cli/commands/route/_dns.py
 evo_cli/commands/route/_dpi.py
 evo_cli/commands/route/_hosts.py
+evo_cli/commands/route/_page.py
 evo_cli/commands/route/_probe.py
 evo_cli/commands/route/_spoofdpi.py
 evo_cli/commands/route/check.py

{evo_cli-0.32.1 → evo_cli-0.33.0}/tests/test_route.py

@@ -8,7 +8,7 @@ import pytest
 from click.testing import CliRunner
 
 from evo_cli.cli import cli
-from evo_cli.commands.route import _dns, _dpi, _hosts, _probe, _spoofdpi
+from evo_cli.commands.route import _dns, _dpi, _hosts, _page, _probe, _spoofdpi
 
 
 def test_route_registered():
@@ -257,10 +257,16 @@ def _fake_networksetup(proxies, calls):
 def test_spoofdpi_host_list_round_trips_with_subdomains(tmp_path, monkeypatch):
     monkeypatch.setenv("EVO_ROUTE_HOME", str(tmp_path))
     _spoofdpi._write_hosts(["medium.com", "x.com", "medium.com"])
-    assert (tmp_path / "hosts.txt").read_text() == "medium.com\n*.medium.com\nx.com\n*.x.com\n"
+    assert (tmp_path / "hosts.txt").read_text() == "medium.com\n**.medium.com\nx.com\n**.x.com\n"
     assert _spoofdpi.opened() == ["medium.com", "x.com"]
 
 
+def test_spoofdpi_reads_a_host_list_from_before_the_double_star(tmp_path, monkeypatch):
+    monkeypatch.setenv("EVO_ROUTE_HOME", str(tmp_path))
+    (tmp_path / "hosts.txt").write_text("medium.com\n*.medium.com\n")
+    assert _spoofdpi.opened() == ["medium.com"]
+
+
 def test_spoofdpi_config_bypasses_only_the_opened_hosts():
     """Every other name goes straight through, so names only a VPN resolves keep working."""
     assert 'mode = "system"' in _spoofdpi.CONFIG
@@ -339,6 +345,154 @@ def test_proxy_probe_reports_a_refused_connect():
     assert "403" in row["error"]
 
 
+PAGE_HTML = """
+<html><head>
+<link rel="canonical" href="https://other.example/">
+<link rel="stylesheet" href="https://store.fastly.steamstatic.com/a.css">
+<link rel="preconnect" href="//shared.fastly.steamstatic.com">
+<link rel="icon" href="/favicon.ico">
+<script src="https://cdn.fastly.steamstatic.com/a.js"></script>
+</head><body>
+<a href="https://steamcommunity.com/">a link is not a load</a>
+<img src="data:image/gif;base64,R0lGODlhAQABAAAAACw=">
+<img src="https://1.2.3.4/a.png" srcset="https://img.example.org/a.png 1x, https://img2.example.org/a.png 2x">
+</body></html>
+"""
+
+
+def test_asset_hosts_are_what_a_browser_loads_by_itself():
+    assert _page.asset_hosts(PAGE_HTML, "https://store.steampowered.com/") == [
+        "store.fastly.steamstatic.com",
+        "shared.fastly.steamstatic.com",
+        "store.steampowered.com",
+        "cdn.fastly.steamstatic.com",
+        "img.example.org",
+        "img2.example.org",
+    ]
+
+
+@pytest.mark.parametrize(
+    "host, family",
+    [
+        ("store.fastly.steamstatic.com", "steamstatic.com"),
+        ("medium.com", "medium.com"),
+        ("static.example.com.vn", "example.com.vn"),
+        ("www.example.co.uk", "example.co.uk"),
+        ("a.b.example.io", "example.io"),
+    ],
+)
+def test_registrable(host, family):
+    assert _page.registrable(host) == family
+
+
+def _fake_page(monkeypatch, html, dead=(), final_url="https://store.steampowered.com/"):
+    seen = []
+
+    def reachable(name, timeout=6.0):
+        seen.append(name)
+        return name not in dead
+
+    monkeypatch.setattr(_page, "fetch", lambda host, proxy=None, timeout=6.0: (final_url, html))
+    monkeypatch.setattr(_probe, "reachable", reachable)
+    return seen
+
+
+def test_blocked_hosts_keeps_only_what_fails_and_is_not_covered(monkeypatch):
+    dead = {"store.fastly.steamstatic.com", "cdn.fastly.steamstatic.com", "steampowered.com"}
+    seen = _fake_page(monkeypatch, PAGE_HTML, dead)
+    blocked = _page.blocked_hosts("store.steampowered.com", None, ["example.org"])
+    assert blocked == ["store.fastly.steamstatic.com", "cdn.fastly.steamstatic.com", "steampowered.com"]
+    # The host itself and what the bypass already covers read as blocked over a raw socket, so they are not asked.
+    assert "store.steampowered.com" not in seen and "img.example.org" not in seen
+
+
+def test_reachable_ignores_a_bogus_answer_and_tries_each_address(monkeypatch):
+    monkeypatch.setattr(_dns, "system_resolve", lambda host: ["127.0.0.1", "1.2.3.4", "5.6.7.8"])
+    monkeypatch.setattr(_probe, "tls_probe", lambda ip, sni, timeout=6.0: _probe_row(ip, tls=ip == "5.6.7.8"))
+    assert _probe.reachable("example.org")
+    monkeypatch.setattr(_dns, "system_resolve", lambda host: ["127.0.0.1"])
+    assert not _probe.reachable("example.org")
+    monkeypatch.setattr(_dns, "system_resolve", lambda host: None)
+    assert not _probe.reachable("example.org")
+
+
+def _fake_macos_bypass(monkeypatch, tmp_path, proxy_fails=()):
+    """A sni-blocked host on macOS; ``ensure`` only records what it was asked to open."""
+    opened = []
+
+    def ensure(*hosts):
+        opened.append(hosts)
+        return {"running": True, "proxy": ["Wi-Fi"]}
+
+    monkeypatch.setenv("EVO_ROUTE_HOME", str(tmp_path))
+    monkeypatch.setattr(_dpi, "is_macos", lambda: True)
+    monkeypatch.setattr(_dpi, "is_admin", lambda: False)
+    monkeypatch.setattr(_probe, "diagnose", lambda host, timeout: {"host": host, "verdict": _probe.SNI_BLOCKED})
+    monkeypatch.setattr(_spoofdpi, "ensure", ensure)
+    monkeypatch.setattr(_spoofdpi, "browser_overrides", lambda: [])
+    monkeypatch.setattr(
+        _probe, "proxy_probe", lambda host, proxy, timeout=6.0: _probe_row(host, tls=host not in proxy_fails)
+    )
+    return opened
+
+
+def test_open_adds_the_domains_a_page_needs_to_the_bypass(monkeypatch, tmp_path):
+    opened = _fake_macos_bypass(monkeypatch, tmp_path)
+    _fake_page(monkeypatch, PAGE_HTML, {"store.fastly.steamstatic.com", "cdn.fastly.steamstatic.com"})
+    result = CliRunner().invoke(cli, ["route", "open", "store.steampowered.com"])
+    assert result.exit_code == 0, result.output
+    assert opened == [("store.steampowered.com",), ("steamstatic.com",)]
+
+
+def test_open_host_only_never_reads_the_page(monkeypatch, tmp_path):
+    opened = _fake_macos_bypass(monkeypatch, tmp_path)
+    seen = _fake_page(monkeypatch, PAGE_HTML, {"store.fastly.steamstatic.com"})
+    result = CliRunner().invoke(cli, ["route", "open", "store.steampowered.com", "--host-only"])
+    assert result.exit_code == 0, result.output
+    assert opened == [("store.steampowered.com",)]
+    assert seen == []
+
+
+def test_open_says_when_a_page_host_stays_blocked(monkeypatch, tmp_path):
+    _fake_macos_bypass(monkeypatch, tmp_path, proxy_fails={"cdn.fastly.steamstatic.com"})
+    _fake_page(monkeypatch, PAGE_HTML, {"cdn.fastly.steamstatic.com"})
+    result = CliRunner().invoke(cli, ["route", "open", "store.steampowered.com"])
+    assert result.exit_code == 0, result.output
+    assert "Still failing through the bypass" in result.output
+
+
+def test_open_survives_a_page_it_cannot_read(monkeypatch, tmp_path):
+    opened = _fake_macos_bypass(monkeypatch, tmp_path)
+
+    def refuse(host, proxy=None, timeout=6.0):
+        raise OSError("HTTP Error 403: Forbidden")
+
+    monkeypatch.setattr(_page, "fetch", refuse)
+    result = CliRunner().invoke(cli, ["route", "open", "medium.com"])
+    assert result.exit_code == 0, result.output
+    assert opened == [("medium.com",)]
+    assert "unchecked" in result.output
+
+
+def test_open_with_a_pin_reports_the_page_hosts_it_cannot_reach(monkeypatch, tmp_path):
+    monkeypatch.setenv("EVO_ROUTE_HOME", str(tmp_path))
+    monkeypatch.setattr(_dpi, "is_admin", lambda: True)
+    monkeypatch.setattr(_hosts, "pin", lambda host, ip: {host: ip})
+    report = {
+        "host": "www.example.org",
+        "verdict": _probe.ADDRESS_BLOCKED,
+        "best": "5.6.7.8",
+        "working": [_probe_row("5.6.7.8", tls=True)],
+    }
+    after = {**report, "verdict": _probe.OPEN}
+    reports = iter([report, after])
+    monkeypatch.setattr(_probe, "diagnose", lambda host, timeout: next(reports))
+    _fake_page(monkeypatch, PAGE_HTML, {"cdn.fastly.steamstatic.com"})
+    result = CliRunner().invoke(cli, ["route", "open", "www.example.org"])
+    assert result.exit_code == 0, result.output
+    assert "cdn.fastly.steamstatic.com" in result.output
+
+
 def test_open_on_linux_names_the_supported_systems(monkeypatch):
     monkeypatch.setattr(_dpi, "is_macos", lambda: False)
     monkeypatch.setattr(_dpi, "is_windows", lambda: False)

evo_cli-0.32.1/evo_cli/VERSION

@@ -1 +0,0 @@
-0.32.1