mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/netxray/register.py

@@ -0,0 +1,93 @@
+"""Network X-ray's routes, for the Mac app's session only: no agent or MCP tool, so no hostname reaches a model."""
+
+from ..api_errors import APIError
+from ..api_routes import Response, Route
+from ..api_routes import register as add_route
+from ..harness_api import STAY_ON_MAC
+from . import job, target
+
+APP_ONLY = "Only the Mobster app can record your iPhone's network traffic. Open Mobster and ask there."
+
+
+def _app_only(request):
+    # Fail closed: the capture is of everything the phone does, so only the Mac app's own session starts or reads it.
+    if not request.app_session:
+        raise APIError(APP_ONLY, 403, "app_only")
+
+
+def _board(request):
+    board = getattr(request.runtime, "netxray_board", None)
+    if board is None:
+        raise APIError("Network X-ray isn't running", 503, "xray_unavailable")
+    return board
+
+
+def parse(body):
+    """(device, seconds, wifi, key) from a start request; ValueError says what's wrong."""
+    body = body if isinstance(body, dict) else {}
+    device, seconds, wifi, key = (body.get("device"), body.get("seconds", job.DEFAULT_SECONDS),
+                                  body.get("wifi", False), body.get("key"))
+    if not isinstance(device, str) or not 1 <= len(device) <= 128:
+        raise ValueError("device is the phone's id, UDID or name")
+    if isinstance(seconds, bool) or not isinstance(seconds, int) or not job.MIN_SECONDS <= seconds <= job.MAX_SECONDS:
+        raise ValueError(f"seconds is {job.MIN_SECONDS} to {job.MAX_SECONDS}")
+    if not isinstance(wifi, bool):
+        raise ValueError("wifi is true or false")
+    if key is not None and (not isinstance(key, str) or not 1 <= len(key) <= 100):
+        raise ValueError("key is 1 to 100 characters")
+    return device, seconds, wifi, key
+
+
+def start(request):
+    _app_only(request)
+    device, seconds, wifi, key = parse(request.body)
+    board = _board(request)
+    try:
+        phone = target.pick(device, wifi=wifi)
+    except target.TargetError as error:
+        raise APIError(str(error), 400, "xray_device") from None
+    gate = getattr(request.runtime, "plan_gate", None)
+    if gate is not None:
+        gate.require()
+    started = board.start(phone, seconds, key=f"{phone.udid}-{key}" if key else None)
+    return Response.json({"jobId": started.id}, 202)
+
+
+def status(request):
+    _app_only(request)
+    found = _board(request).get(request.params["id"])
+    if found is None:
+        raise APIError("There is no such X-ray", 404, "xray_not_found")
+    return Response.json(found.view())
+
+
+def stop(request):
+    _app_only(request)
+    return Response.json(_board(request).stop(request.params["id"]).view())
+
+
+def open_board(runtime):
+    runtime.netxray_board = job.Board(runtime)
+
+
+def close_board(runtime):
+    board = getattr(runtime, "netxray_board", None)
+    if board is not None:
+        board.close()
+        runtime.netxray_board = None
+
+
+def stay_on_mac(value, runtime):
+    if value is not True:
+        raise ValueError(f"{STAY_ON_MAC} is true or left out")
+    return True
+
+
+def register(api):
+    """``api`` is mobile_agent.harness_api."""
+    # The finished report's run stays on this Mac: nothing shares, sends or reads it but the app.
+    api.register_run_field(STAY_ON_MAC, stay_on_mac)
+    api.register_service(open_board, close_board)
+    add_route(Route("netxray", "POST", "/api/xray/network", start))
+    add_route(Route("netxray", "GET", "/api/xray/network/{id}", status, body="none"))
+    add_route(Route("netxray", "POST", "/api/xray/network/{id}/stop", stop, body="none"))

mobile_agent/netxray/report.py

@@ -0,0 +1,214 @@
+"""netxray.json (``mobster.netxray/1``), report.html and report.md; names from the traffic are data, always escaped."""
+
+import json
+import time
+from pathlib import Path
+
+from .. import links
+from ..verify.report import BASE_STYLE, CSP, _e, brand, icon
+from .analyze import headline
+from .trackers import CATEGORY_LABELS, TRACKING
+
+CATEGORY_TONE = {"advertising": "fail", "analytics": "warn", "attribution": "review", "engagement": "warn",
+                 "crash": "none", "social": "none", "platform": "none", "cdn": "none"}
+
+STYLE = BASE_STYLE + """
+.stats { display: grid; grid-template-columns: repeat(5, minmax(0, 1fr)); gap: 1px; margin: 0 0 12px;
+  border-radius: var(--radius-card); overflow: hidden; }
+.stat { padding: 14px 16px; background: var(--card); min-width: 0; }
+.stat dt { font-size: 12px; line-height: 18px; color: var(--ink-3); }
+.stat dd { margin: 2px 0 0; font-size: 20px; line-height: 28px; font-weight: var(--weight-medium);
+  font-variant-numeric: tabular-nums; }
+.stat.alert dd { color: var(--coral); }
+table.hosts { width: 100%; border-collapse: collapse; font-size: 13px; line-height: 20px; }
+table.hosts th { text-align: left; font-weight: var(--weight-medium); color: var(--ink-3); font-size: 12px;
+  padding: 10px 12px 6px; white-space: nowrap; }
+table.hosts td { padding: 8px 12px; border-top: 1px solid var(--line); vertical-align: middle; }
+table.hosts td.num, table.hosts th.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
+table.hosts .host { font-family: var(--font-mono); font-size: 12.5px; overflow-wrap: anywhere; }
+table.hosts .company { color: var(--ink-2); white-space: nowrap; }
+table.hosts tr.tracking .host { color: var(--ink); }
+.bar { display: block; height: 3px; margin-top: 4px; border-radius: 2px; background: var(--tone, var(--ink-4));
+  opacity: .55; min-width: 2px; }
+.hosts-card { padding: 4px 4px 8px; overflow-x: auto; }
+.tag.tone.none::before { background: var(--ink-4); }
+.unnamed { margin: 8px 16px 4px; font-size: 13px; color: var(--ink-3); }
+.privacy { margin: 16px 0 0; max-width: 720px; font-size: 13px; line-height: 20px; color: var(--ink-2); }
+.synthetic { --tone: var(--amber); }
+@media (max-width: 760px) {
+  .stats { grid-template-columns: repeat(2, minmax(0, 1fr)); }
+  table.hosts .hide-narrow { display: none; }
+  table.hosts td:first-child { min-width: 150px; }
+  table.hosts td, table.hosts th { padding-left: 8px; padding-right: 8px; }
+}
+"""
+
+
+def size(count):
+    """Bytes as people read them: 812 B, 14.2 KB, 3.1 MB."""
+    if count < 1000:
+        return f"{count} B"
+    if count < 1_000_000:
+        return f"{count / 1000:.1f} KB"
+    return f"{count / 1_000_000:.1f} MB"
+
+
+def category_tag(category):
+    if not category:
+        return '<span class="quiet">Unknown</span>'
+    label = CATEGORY_LABELS.get(category, category)
+    return f'<span class="tag tone {CATEGORY_TONE.get(category, "none")}">{_e(label)}</span>'
+
+
+def _app_section(app, seconds, index, *, lede=True):
+    domains = app["domains"]
+    biggest = max([d["up"] + d["down"] for d in domains] or [1]) or 1
+    parts = [f'<section aria-labelledby="app-{index}"><h2 id="app-{index}">{_e(app["name"])}'
+             f'<span class="count">{len(domains)} domain{"s" * (len(domains) != 1)}</span></h2>']
+    if lede:
+        parts.append(f'<p class="lede" style="margin-bottom:16px">{_e(headline(app, seconds))}</p>')
+    via = ", ".join(app["via"])
+    stats = [("Domains", app["domain_count"], False), ("Companies", len(app["companies"]), False),
+             ("Ads, analytics, attribution", len(app["tracking"]), bool(app["tracking"])),
+             ("Sent", size(app["up"]), False), ("Received", size(app["down"]), False)]
+    parts.append('<dl class="stats">' + "".join(
+        f'<div class="stat{" alert" * alert}"><dt>{_e(label)}</dt><dd>{_e(value)}</dd></div>'
+        for label, value, alert in stats) + "</dl>")
+    if domains:
+        rows = []
+        for domain in domains:
+            total = domain["up"] + domain["down"]
+            tone = CATEGORY_TONE.get(domain["category"] or "", "none")
+            tracking = " tracking" if domain["category"] in TRACKING else ""
+            rows.append(
+                f'<tr class="{tone}{tracking}"><td><span class="host">{_e(domain["host"])}</span>'
+                f'<span class="bar" style="width:{max(1, round(100 * total / biggest))}%"></span></td>'
+                f'<td class="company hide-narrow">{_e(domain["company"])}</td>'
+                f'<td>{category_tag(domain["category"])}</td>'
+                f'<td class="num">{_e(size(domain["up"]))}</td><td class="num">{_e(size(domain["down"]))}</td>'
+                f'<td class="num hide-narrow">{domain["connections"]}</td>'
+                f'<td class="num hide-narrow">{domain["first"]:.1f} s</td></tr>')
+        parts.append('<div class="card hosts-card"><table class="hosts"><thead><tr><th>Host</th>'
+                     '<th class="hide-narrow">Company</th>'
+                     '<th>Category</th><th class="num">Sent</th><th class="num">Received</th>'
+                     '<th class="num hide-narrow">Connections</th><th class="num hide-narrow">First seen</th>'
+                     '</tr></thead><tbody>' + "".join(rows) + "</tbody></table>")
+        unnamed = app["unnamed"]
+        if unnamed["connections"]:
+            parts.append(f'<p class="unnamed">{unnamed["connections"]} more connection'
+                         f'{"s" * (unnamed["connections"] != 1)} with no hostname in the capture '
+                         f'({_e(size(unnamed["up"] + unnamed["down"]))}). Their addresses aren’t shown.</p>')
+        parts.append("</div>")
+    if app["kind"] == "ios" and app["processes"]:
+        parts.append(f'<p class="privacy">These are parts of iOS itself: {_e(", ".join(app["processes"]))}.</p>')
+    if app.get("note"):
+        parts.append(f'<p class="privacy">{_e(app["note"])}</p>')
+    if app["kind"] == "process" and app["processes"]:
+        parts.append(f'<p class="privacy">Process: {_e(", ".join(app["processes"]))}.</p>')
+    if via:
+        parts.append(f'<p class="privacy">Some of this traffic went through {_e(via)} on the app’s behalf.</p>')
+    parts.append("</section>")
+    return "".join(parts)
+
+
+def render_html(result, *, title="Network X-ray", synthetic=False):
+    capture = result["capture"]
+    apps = result["apps"]
+    seconds = capture["seconds"]
+    started = time.strftime("%d %b %Y %H:%M", time.localtime(capture["started"])) if capture["started"] else "-"
+    focus = apps[0] if len(apps) == 1 else None
+    heading = headline(focus, seconds) if focus else (
+        f"{len(apps)} process{'es' * (len(apps) != 1)} talked to "
+        f"{len({d['host'] for a in apps for d in a['domains']})} domains")
+    chips = ['<span class="tag">Metadata only</span>']
+    if synthetic:
+        chips.append('<span class="tag tone synthetic">Synthetic capture</span>')
+    if capture["truncated"]:
+        chips.append('<span class="tag tone warn">The capture ends mid-packet</span>')
+    if not capture["with_process"]:
+        chips.append('<span class="tag tone warn">No process names in this capture</span>')
+    meta = [("clock", f"{seconds:.0f} s captured"), ("calendar", started),
+            ("phone", f"{capture['packets']:,} packets")]
+    parts = [
+        "<!doctype html>", '<html lang="en"><head><meta charset="utf-8">',
+        f'<meta http-equiv="Content-Security-Policy" content="{CSP}">',
+        '<meta name="viewport" content="width=device-width, initial-scale=1">',
+        '<meta name="color-scheme" content="light dark">',
+        f"<title>{_e(title)}</title><style>{STYLE}</style></head><body><main>",
+        brand("network x-ray"),
+        '<header class="head"><div class="chips">', "".join(chips), "</div>",
+        f"<h1>{_e(heading)}</h1>",
+        '<p class="lede">Hostnames, sizes and times, read from the phone’s traffic. Nothing was decrypted '
+        "and no payload was kept.</p>",
+        '<ul class="meta">' + "".join(f"<li>{icon(name)}{_e(text)}</li>" for name, text in meta) + "</ul></header>",
+    ]
+    for index, app in enumerate(apps):
+        if app["domains"] or app["unnamed"]["connections"]:
+            parts.append(_app_section(app, seconds, index, lede=focus is None))
+    quiet = [app["name"] for app in apps if not app["domains"] and not app["unnamed"]["connections"]]
+    local = result["local"]
+    notes = []
+    if local["packets"]:
+        notes.append(f"{local['packets']:,} packets ({size(local['bytes'])}) stayed on the local network. "
+                     "Local names and addresses aren’t shown.")
+    if quiet:
+        notes.append("Looked up names but opened no connection: " + ", ".join(quiet) + ".")
+    if notes:
+        parts.append('<section><h2>Also in the capture</h2>' + "".join(f'<p class="privacy">{_e(n)}</p>'
+                                                                       for n in notes) + "</section>")
+    parts.append(f'<footer class="foot"><span><a href="{links.MADE_WITH_XRAY}">Mobster</a> Network X-ray</span>'
+                 '<span>Hostnames come from DNS '
+                 'answers, TLS and QUIC server names and HTTP Host headers</span>'
+                 '<span>Companies and categories come from Mobster’s own list</span></footer>')
+    parts.append("</main></body></html>")
+    return "".join(parts)
+
+
+def _md(text):
+    """Text from the capture, safe inside a Markdown table cell or line."""
+    text = str(text)
+    for char in "\\`*_[]<>|#":
+        text = text.replace(char, "\\" + char)
+    return text.replace("\n", " ")
+
+
+def render_markdown(result, *, synthetic=False):
+    capture = result["capture"]
+    seconds = capture["seconds"]
+    lines = ["# Network X-ray", ""]
+    if synthetic:
+        lines += ["Synthetic capture.", ""]
+    lines += [f"{seconds:.0f} s captured, {capture['packets']:,} packets. Hostnames, sizes and times only: nothing was "
+              "decrypted and no payload was kept.", ""]
+    for app in result["apps"]:
+        if not app["domains"] and not app["unnamed"]["connections"]:
+            continue
+        lines += [f"## {_md(app['name'])}", "", _md(headline(app, seconds)), ""]
+        if app["domains"]:
+            lines += ["| Host | Company | Category | Sent | Received | Connections |", "|---|---|---|---:|---:|---:|"]
+        for domain in app["domains"]:
+            category = CATEGORY_LABELS.get(domain["category"], "") if domain["category"] else "Unknown"
+            lines.append(f"| `{domain['host']}` | {_md(domain['company'])} | {category} | {size(domain['up'])} | "
+                         f"{size(domain['down'])} | {domain['connections']} |")
+        if app["unnamed"]["connections"]:
+            lines += ["", f"{app['unnamed']['connections']} {'more ' * bool(app['domains'])}connections with no "
+                          f"hostname in the capture ({size(app['unnamed']['up'] + app['unnamed']['down'])})."]
+        if app.get("note"):
+            lines += ["", _md(app["note"])]
+        if app["kind"] in ("ios", "process") and app["processes"]:
+            lines += ["", _md(("Parts of iOS: " if app["kind"] == "ios" else "Process: ")
+                              + ", ".join(app["processes"]) + ".")]
+        lines.append("")
+    return "\n".join(lines)
+
+
+def write(result, out_dir, *, synthetic=False):
+    """Write netxray.json, report.html and report.md into ``out_dir``; their paths."""
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    out.chmod(0o700)
+    paths = {"json": out / "netxray.json", "html": out / "report.html", "markdown": out / "report.md"}
+    paths["json"].write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    paths["html"].write_text(render_html(result, synthetic=synthetic), encoding="utf-8")
+    paths["markdown"].write_text(render_markdown(result, synthetic=synthetic), encoding="utf-8")
+    return paths

mobile_agent/netxray/store.py

@@ -0,0 +1,48 @@
+"""Where a Network X-ray lives: a private folder (0700) on this Mac. The capture stays only while read, or when kept."""
+
+import time
+from pathlib import Path
+
+from .. import paths
+
+
+def root():
+    return Path(paths.user_data_dir()) / "xray" / "network"
+
+
+def new_folder(now=None, base=None):
+    """A new private folder named for the time. Never reuses one: a second in the same second gets -2, -3."""
+    base = Path(base) if base else root()
+    stamp = time.strftime("%Y%m%d-%H%M%S", time.localtime(now if now is not None else time.time()))
+    for number in range(1, 100):
+        folder = base / (stamp if number == 1 else f"{stamp}-{number}")
+        try:
+            base.mkdir(parents=True, exist_ok=True)
+            folder.mkdir(mode=0o700)
+        except FileExistsError:
+            continue
+        folder.chmod(0o700)
+        return folder
+    raise FileExistsError("Too many X-rays in one second")
+
+
+def capture_path(folder):
+    return Path(folder) / "capture.pcapng"
+
+
+def discard_capture(folder):
+    """Delete the capture (and nothing else) in ``folder``."""
+    capture_path(folder).unlink(missing_ok=True)
+
+
+def conclude(folder, names, *, keep=False, only=None, source=None):
+    # The folder's capture goes whatever happens, so a failed analysis leaves no packet metadata behind.
+    from . import analyze, report, summary
+    try:
+        result = analyze.analyze_file(source or capture_path(folder), names=names)
+        if only:
+            result = summary.only(result, only)
+        return result, report.write(result, folder)
+    finally:
+        if not keep:
+            discard_capture(folder)

mobile_agent/netxray/summary.py

@@ -0,0 +1,157 @@
+"""What Network X-ray says about an analysis. Every string has already passed the secret filter."""
+
+from .. import result_blocks
+from . import attribution
+from .analyze import headline
+from .trackers import CATEGORY_LABELS, TRACKING
+
+# What a company that does several things is shown as: the one a person would want to know first.
+CATEGORY_ORDER = ("advertising", "attribution", "analytics", "engagement", "crash", "social", "platform", "cdn")
+NOT_SEEN = ("Names sent through encrypted DNS appear only when the connection itself names the server. iCloud Private "
+            "Relay and Encrypted Client Hello hide the server's name, so those connections are counted without one.")
+
+
+def _plural(count, one, many):
+    return f"{count} {one if count == 1 else many}"
+
+
+def named(result):
+    """The apps with something to say: a domain, or a connection with no hostname."""
+    return [app for app in result["apps"] if app["domains"] or app["unnamed"]["connections"]]
+
+
+def only(result, name):
+    """``result`` with only the app called ``name`` (any case), or the process that sent it.
+    The local total goes too."""
+    wanted = name.strip().casefold()
+    kept = [app for app in result["apps"]
+            if app["name"].casefold() == wanted or any(process.casefold() == wanted for process in app["processes"])]
+    return {**result, "apps": kept, "local": {"packets": 0, "bytes": 0}}
+
+
+def lines(result):
+    """One sentence per app that contacted a domain, then one for traffic nobody could be blamed for."""
+    seconds = result["capture"]["seconds"]
+    out = [headline(app, seconds) for app in result["apps"] if app["domain_count"]
+           and app["kind"] != attribution.UNATTRIBUTED_KIND]
+    out += [headline(app, seconds) for app in result["apps"]
+            if app["kind"] == attribution.UNATTRIBUTED_KIND and app["connections"]]
+    return out
+
+
+def companies(result):
+    """[{"name", "category", "categoryLabel", "apps", "domains", "bytes"}] across all apps, biggest first."""
+    found = {}
+    for app in result["apps"]:
+        for domain in app["domains"]:
+            entry = found.setdefault(domain["company"], {"name": domain["company"], "categories": set(), "apps": [],
+                                                         "domains": set(), "bytes": 0})
+            if domain["category"]:
+                entry["categories"].add(domain["category"])
+            if app["name"] not in entry["apps"]:
+                entry["apps"].append(app["name"])
+            entry["domains"].add(domain["host"])
+            entry["bytes"] += domain["up"] + domain["down"]
+    out = []
+    for entry in found.values():
+        category = next((name for name in CATEGORY_ORDER if name in entry["categories"]), None)
+        out.append({"name": entry["name"], "category": category,
+                    "categoryLabel": CATEGORY_LABELS.get(category, "Other"),
+                    "apps": sorted(entry["apps"]), "domains": len(entry["domains"]), "bytes": entry["bytes"]})
+    out.sort(key=lambda item: (-item["bytes"], item["name"].lower()))
+    return out
+
+
+def live(result):
+    """The numbers the Mac app shows while a capture runs: {"apps", "companies", "totals"}."""
+    apps = [{"name": app["name"], "kind": app["kind"], "process": ", ".join(app["processes"]),
+             "domains": app["domain_count"], "companies": len(app["companies"]), "tracking": len(app["tracking"]),
+             "bytes": app["up"] + app["down"]} for app in named(result)]
+    shared = companies(result)
+    unattributed = next((app for app in result["apps"] if app["kind"] == attribution.UNATTRIBUTED_KIND), None)
+    hosts = {domain["host"] for app in result["apps"] for domain in app["domains"]}
+    tracking = {domain["host"] for app in result["apps"] for domain in app["domains"]
+                if domain["category"] in TRACKING}
+    return {"apps": apps, "companies": shared,
+            "totals": {"apps": len([app for app in apps if app["kind"] != attribution.UNATTRIBUTED_KIND]),
+                       "domains": len(hosts), "companies": len(shared), "tracking": len(tracking),
+                       "bytes": sum(app["up"] + app["down"] for app in result["apps"]),
+                       "unattributed": {"bytes": unattributed["up"] + unattributed["down"] if unattributed else 0,
+                                        "connections": unattributed["connections"] if unattributed else 0},
+                       "packets": result["capture"]["packets"]}}
+
+
+def answer(result):
+    """The task's one-line answer: counts across all apps, then the traffic that couldn't be blamed."""
+    seconds = max(1, round(result["capture"]["seconds"]))
+    total = live(result)["totals"]
+    shown = [app for app in named(result) if app["kind"] != attribution.UNATTRIBUTED_KIND and app["domain_count"]]
+    if not shown and not total["unattributed"]["bytes"]:
+        return f"In {seconds} s your iPhone contacted no domain Mobster could name."
+    text = (f"In {seconds} s {_plural(len(shown), 'app', 'apps')} contacted "
+            f"{_plural(total['domains'], 'domain', 'domains')} "
+            f"at {_plural(total['companies'], 'company', 'companies')}")
+    if total["tracking"]:
+        text += (f". {total['tracking']} of those domains {'is' if total['tracking'] == 1 else 'are'} ads, "
+                 "analytics or attribution")
+    text += "."
+    if total["unattributed"]["bytes"]:
+        text += " Mobster couldn't tell which app sent some of the traffic, so it is listed as Unattributed."
+    return text
+
+
+def _kb(count):
+    return round(count / 1000, 1)
+
+
+def blocks(result, stamp=None, kept=False):
+    """The finished task's blocks: who each app talked to, every domain, and what was and wasn't seen."""
+    people = []
+    for app in named(result):
+        grouped = {}
+        for domain in app["domains"]:
+            entry = grouped.setdefault(domain["company"], {"categories": set(), "domains": 0, "up": 0, "down": 0})
+            if domain["category"]:
+                entry["categories"].add(domain["category"])
+            entry["domains"] += 1
+            entry["up"] += domain["up"]
+            entry["down"] += domain["down"]
+        for company, entry in grouped.items():
+            what = ", ".join(CATEGORY_LABELS[name] for name in CATEGORY_ORDER if name in entry["categories"])
+            people.append([app["name"], company, what or "Unknown", entry["domains"], _kb(entry["up"]),
+                           _kb(entry["down"])])
+    people.sort(key=lambda row: (-(row[4] + row[5]), row[0].lower(), row[1].lower()))
+    hosts = [[app["name"], domain["host"], domain["company"],
+              CATEGORY_LABELS.get(domain["category"], "Unknown") if domain["category"] else "Unknown",
+              domain["connections"], _kb(domain["up"] + domain["down"])]
+             for app in result["apps"] for domain in app["domains"]]
+    hosts.sort(key=lambda row: (-row[5], row[0].lower(), row[1]))
+    out = []
+    if people:
+        out.append(result_blocks.table(
+            [{"key": "app", "label": "App"}, {"key": "company", "label": "Company", "type": "brand"},
+             {"key": "does", "label": "What it does"}, {"key": "domains", "label": "Domains", "type": "number"},
+             {"key": "sent", "label": "Sent (KB)", "type": "number"},
+             {"key": "received", "label": "Received (KB)", "type": "number"}],
+            people, title="Who your apps talked to", id="companies"))
+        out.append(result_blocks.table(
+            [{"key": "app", "label": "App"}, {"key": "domain", "label": "Domain"},
+             {"key": "company", "label": "Company", "type": "brand"}, {"key": "category", "label": "Category"},
+             {"key": "connections", "label": "Connections", "type": "number"},
+             {"key": "bytes", "label": "Bytes (KB)", "type": "number"}],
+            hosts, title="Every domain", id="domains"))
+    notes = ["Mobster read hostnames, sizes and times from your iPhone's network traffic. It decrypted nothing and "
+             "kept no page content, URL path or address."]
+    for app in result["apps"]:
+        if app.get("note") and (app["up"] or app["down"] or app["connections"]):
+            notes.append(f"**{app['name']}:** {app['note']} ({_kb(app['up'] + app['down'])} KB in "
+                         f"{_plural(app['connections'], 'connection', 'connections')}.)")
+    if result["local"]["packets"]:
+        notes.append(f"{result['local']['packets']:,} packets stayed on your local network. Their names and addresses "
+                     "aren't shown.")
+    notes.append("**What Mobster couldn't see.** " + NOT_SEEN)
+    if stamp:
+        notes.append(f"Saved on this Mac, in Mobster's data folder under `xray/network/{stamp}`. " + (
+            "The capture file is kept there too." if kept else "The capture file was deleted after the analysis."))
+    out.append(result_blocks.markdown("\n\n".join(notes), title="What was read", id="notes"))
+    return out

mobile_agent/netxray/target.py

@@ -0,0 +1,67 @@
+"""Which phone to capture, and which apps it has. Simulators have no pcapd service, so they are refused."""
+
+from dataclasses import dataclass
+
+from . import attribution
+
+
+class TargetError(Exception):
+    """The phone can't be captured; the message says why and what to do."""
+
+
+@dataclass(frozen=True)
+class Phone:
+    id: str
+    name: str
+    udid: str
+    network: bool        # capture over Wi-Fi (mobster-pcap --network), not the cable
+
+
+def on_cable(record):
+    """Whether ``record`` (devices.discover) is on a cable now."""
+    if "transport" in record:
+        return record["transport"] == "usb"
+    return record.get("state") != "disconnected"
+
+
+def pick(query=None, *, wifi=False, records=None):
+    """The Phone to capture."""
+    from .. import devices
+    records = devices.discover(probe=False) if records is None else records
+    if query:
+        try:
+            record = devices.resolve(records, query)
+        except LookupError as error:
+            raise TargetError(str(error)) from None
+        if record.get("kind") == "simulator":
+            raise TargetError(f"“{record['name']}” is a simulator. Network X-ray reads a real iPhone's traffic: "
+                              "name an iPhone.")
+        if record.get("kind") != "usb" or not record.get("udid"):
+            raise TargetError(f"“{record['name']}” isn't an iPhone Mobster set up. Network X-ray needs the phone's "
+                              "UDID: plug it in once with a cable.")
+    else:
+        phones = [item for item in records
+                  if item.get("kind") == "usb" and item.get("udid") and (wifi or on_cable(item))]
+        if not phones:
+            raise TargetError("Mobster sees no iPhone. Plug one in, or name one with --device.")
+        if len(phones) > 1:
+            raise TargetError(f"{len(phones)} iPhones are connected, so Mobster won't guess which one to capture. "
+                              "Name one with --device: " + ", ".join(f"{item['name']} ({item['id']})"
+                                                                     for item in phones[:8]) + ".")
+        record = phones[0]
+    return Phone(record["id"], record.get("name") or record["id"], record["udid"], bool(wifi or not on_cable(record)))
+
+
+def phone_names(phone, *, tool=None, runner=None):
+    """attribution.Names for ``phone``, from the apps it has installed (user and system, with their executables)."""
+    from .. import installed_apps
+    from ..mac_tools import tool as find_tool
+    kwargs = {"runner": runner} if runner else {}
+    binary = tool or find_tool("ideviceinstaller")
+    rows = []
+    for kind in ("user", "system"):
+        try:
+            rows += installed_apps.list_apps(binary, phone.udid, kind, network=phone.network, **kwargs)
+        except installed_apps.ListingError:
+            pass
+    return attribution.Names(attribution.executables_from(rows))

mobile_agent/netxray/tool.py

@@ -0,0 +1,66 @@
+"""Find (or build) ``mobster-pcap``, Mobster's own pcapd client."""
+
+import hashlib
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+NAME = "mobster-pcap"
+SOURCE = Path(__file__).resolve().with_name(f"{NAME}.c")
+LIBRARIES = ("libimobiledevice-1.0", "libplist-2.0")
+
+
+def _cached():
+    if not SOURCE.is_file():
+        return None
+    from ..paths import user_data_dir
+    digest = hashlib.sha256(SOURCE.read_bytes()).hexdigest()[:8]
+    return Path(user_data_dir()) / "helpers" / f"{NAME}-{digest}"
+
+
+def find():
+    """The app's bundled tool or a previously built copy; None when neither exists."""
+    from ..mac_tools import bundled_tools_dir, executable
+    folder = bundled_tools_dir()
+    if folder is not None and (path := executable(folder, NAME)):
+        return str(path)
+    cached = _cached()
+    if cached is not None and cached.is_file() and os.access(cached, os.X_OK):
+        return str(cached)
+    return None
+
+
+def build():
+    """Compile the packaged tool source against Homebrew's libimobiledevice."""
+    target = _cached()
+    if target is None:
+        raise LookupError("This Mobster has no mobster-pcap source to build")
+    if target.is_file():
+        return str(target)
+    pkg_config = shutil.which("pkg-config") or next(
+        (path for path in ("/opt/homebrew/bin/pkg-config", "/usr/local/bin/pkg-config") if os.access(path, os.X_OK)),
+        None)
+    if not pkg_config:
+        raise LookupError("pkg-config isn't installed. Run `brew install pkg-config libimobiledevice`.")
+    try:
+        flags = subprocess.run([pkg_config, "--cflags", "--libs", *LIBRARIES], capture_output=True, text=True,
+                               timeout=30, check=True).stdout.split()
+    except (OSError, subprocess.SubprocessError):
+        raise LookupError("libimobiledevice isn't installed. Run `brew install libimobiledevice`.") from None
+    target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    partial = target.with_suffix(".partial")
+    try:
+        subprocess.run(["xcrun", "clang", "-O2", str(SOURCE), "-o", str(partial), *flags], capture_output=True,
+                       timeout=120, check=True)
+    except (OSError, subprocess.SubprocessError):
+        partial.unlink(missing_ok=True)
+        raise LookupError("mobster-pcap couldn't be built here. Install Xcode's command line tools with "
+                          "`xcode-select --install`.") from None
+    os.replace(partial, target)
+    return str(target)
+
+
+def locate():
+    """``find()``, else ``build()``. LookupError says what is missing."""
+    return find() or build()