six-degrees 0.3.0 → 0.4.0

package/.next/standalone/scripts/scrape.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-6 Degrees LinkedIn Scraper
+Six Degrees LinkedIn scanner
 
 Drives Chrome with a persistent profile. You log into LinkedIn by hand once;
 the session is reused on every later run.
@@ -37,7 +37,7 @@ from urllib.parse import parse_qs, urlparse
 
 # Local helper: download + compress avatars into permanent WebP files so they
 # don't break every ~3 weeks when LinkedIn's signed CDN URLs expire.
-from image_store import localize_images
+from image_store import TryLater, localize_images, store_avatar
 
 # --- Config ---
 def _load_env_local():
@@ -592,10 +592,14 @@ class LinkedInPushedBack(Exception):
 
 
 class BudgetReached(Exception):
-    """Today's or this month's search budget is used. Raised after saving."""
+    """Today's or this month's search budget, or today's profile views, is used.
+
+    Raised after saving. `kind` is "daily", "monthly" or "profiles"; the
+    profile cap is checked before a profile opens, so for it `found` is 0.
+    """
 
     def __init__(self, found=0, kind="daily"):
-        super().__init__(f"the {kind} search budget")
+        super().__init__("today's profile views" if kind == "profiles" else f"the {kind} search budget")
         self.found = found
         self.kind = kind
 
@@ -779,6 +783,90 @@ def push_company(people, company_name):
     return inserted
 
 
+# Failures in a row that say nothing about a link (offline, LinkedIn's image
+# server busy or down) before Save photos stops: the rest would fail the same way.
+PHOTO_TRIES_IN_A_ROW = 3
+
+
+def save_waiting_photos(say_none=False):
+    """Save the photos still kept as links, once each. Returns how many were saved.
+
+    The app shows only photos saved on this computer, never LinkedIn's link, so
+    browsing never goes online. Older versions stored the link until a scan
+    replaced it, and a copy of a network made by one carries them too; those
+    people show initials until this runs, at the end of a scan or from Save
+    photos on the Scan page (--save-photos). Each link is tried until there's an
+    answer: saved, it becomes the file's path; a definite no (expired, not
+    LinkedIn's, not a picture, or the same picture as someone else's), the app
+    forgets it. A photo already saved for that person is used without fetching.
+
+    A failure that says nothing about the link (TryLater: offline, LinkedIn's
+    image server busy or down) keeps it for next time. Forgetting on those wiped
+    good links whenever this ran offline, and a person's photo then came back
+    only by scanning them again (TRAPS §7). A few in a row, or nothing but
+    those, raise TryLater once the answers there were are recorded, so the
+    run ends with the reason.
+    """
+    try:
+        resp = requests.get(f"{APP_URL}/api/update-images", headers=app_headers(json_body=False), timeout=30)
+    except requests.RequestException as e:
+        raise SaveFailed(f"the app could not be reached ({e})")
+    if resp.status_code != 200:
+        raise SaveFailed(f"the app answered {resp.status_code}: {resp.text[:160]}")
+    waiting = resp.json().get("waiting", [])
+    if not waiting:
+        if say_none:
+            print("No photos are waiting to be saved.")
+        return 0
+
+    people = len({w.get("profileUrl") for w in waiting})
+    print(f"\nSaving {people} profile photo{'s' if people != 1 else ''} kept as links to LinkedIn...")
+    saved, forget, kept = [], [], set()
+    in_a_row, why = 0, ""
+    for w in waiting:
+        if stop_requested() or in_a_row >= PHOTO_TRIES_IN_A_ROW:
+            break
+        purl, iurl = w.get("profileUrl"), w.get("imageUrl")
+        if not purl or not iurl:
+            continue
+        try:
+            local = store_avatar(iurl, purl, overwrite=False)
+        except TryLater as e:
+            print(f"  Not saved this time: {purl} — {e}")
+            kept.add(purl)
+            in_a_row, why = in_a_row + 1, str(e)
+            continue
+        in_a_row = 0
+        if local:
+            saved.append({"profileUrl": purl, "imageUrl": local})
+        else:
+            forget.append({"profileUrl": purl, "imageUrl": iurl})
+
+    # What was answered is recorded even after a stop, so it isn't tried again.
+    for i in range(0, max(len(saved), len(forget)), 100):
+        r = requests.post(
+            f"{APP_URL}/api/update-images",
+            headers=app_headers(),
+            json={"images": saved[i:i+100], "forget": forget[i:i+100]},
+            timeout=60,
+        )
+        if r.status_code != 200:
+            raise SaveFailed(f"the app answered {r.status_code}: {r.text[:160]}")
+
+    done = len({x["profileUrl"] for x in saved})
+    lost = len({x["profileUrl"] for x in forget} - {x["profileUrl"] for x in saved})
+    later = len(kept - {x["profileUrl"] for x in saved})
+    print(f"  Saved {done} photo{'s' if done != 1 else ''} to this computer"
+          + (f"; {lost} couldn't be (an expired link, a link that isn't LinkedIn's or isn't a picture, "
+             "or the same picture as someone else's), so those people show initials" if lost else "")
+          + (f"; {later} couldn't be fetched this time, and their links are kept to try again" if later else "")
+          + ".")
+    if not stop_requested() and (in_a_row >= PHOTO_TRIES_IN_A_ROW or (kept and not saved and not forget)):
+        raise TryLater(f"{why}. Nothing was forgotten: the photos still waiting are tried again "
+                       "with Save photos, and at the end of your next scan")
+    return done
+
+
 def scrape_full(headless=False):
     """Full first-time scrape: uses search page to get ALL connections with images.
     Paginates through /search/results/people/?network=["F"] (10 results per page).
@@ -1266,16 +1354,24 @@ def note_unclear(profile_url, clear=False):
 # day's or the month's budget is used. A cooldown lock is set when LinkedIn
 # pushes back, and nothing searches until it lifts.
 #
+# Profile views have a cap of their own, because they are what LinkedIn
+# restricted an account for (TRAPS §16). Every circle scan that opens someone's
+# profile takes one, and any two opens are at least PROFILE_GAP apart, timed
+# from the record, so starting scan after scan can't open profiles back to back.
+#
 # These belong to the LinkedIn account (the one Chrome profile), not to a
 # profile in the app: three app profiles must not triple the budget. The app's
 # Scan page reads and edits the same files (lib/linkedin-limits.js).
 #
 #   linkedin-activity.json  {"searches": [epoch s, ...], "profiles": [...]}
-#   scan-limits.json        {"daily": 50, "monthly": 250}   0 = no cap
+#   scan-limits.json        {"daily": 50, "monthly": 250, "profiles": 50}
+#                           0 = no cap on searches; profile views always have one
 #   linkedin-cooldown.json  {"until": epoch s, "reason": "...", "set_at": ...}
 # ---------------------------------------------------------------------------
 DEFAULT_DAILY_SEARCHES = 50
 DEFAULT_MONTHLY_SEARCHES = 250
+DEFAULT_DAILY_PROFILES = 50
+PROFILE_GAP = 60                  # seconds between any two profile opens, whatever opened them
 DAY_SECONDS = 24 * 3600
 PACIFIC = "America/Los_Angeles"
 
@@ -1370,7 +1466,8 @@ def _read_activity():
         except OSError:
             pass
         print("  (the record of LinkedIn searches couldn't be read; counting today as used)")
-        fresh = {"searches": [now] * max(1, search_limits()["daily"] or 1), "profiles": []}
+        lim = search_limits()
+        fresh = {"searches": [now] * max(1, lim["daily"] or 1), "profiles": [now] * lim["profiles"]}
         try:
             _write_json_atomic(path, fresh)
         except Exception:
@@ -1380,17 +1477,43 @@ def _read_activity():
 
 def charge_linkedin(kind="searches", n=1):
     """Write down n searches (or profile views) made just now."""
-    now = time.time()
+    with _Locked("linkedin-activity"):
+        _charge(_read_activity(), kind, n, time.time())
+
+
+def _charge(data, kind, n, now):
+    """Add n of `kind` at `now` to the record and write it. The caller holds the lock."""
     keep_from = month_start_pacific(now) - 32 * DAY_SECONDS
+    data[kind] = [t for t in data.get(kind, []) if t >= keep_from] + [now] * n
+    other = "profiles" if kind == "searches" else "searches"
+    data[other] = [t for t in data.get(other, []) if t >= keep_from]
+    try:
+        _write_json_atomic(_home() / "linkedin-activity.json", data)
+    except Exception as exc:
+        print(f"  (could not write down LinkedIn activity: {exc})")
+
+
+def take_profile_view(now=None):
+    """Write down one profile view, if the cap and the gap allow one now.
+
+    Returns 0 once it is written down and a profile may open, the whole
+    seconds still to wait when the last view was under PROFILE_GAP ago, or
+    None when the last 24 hours' views are used. The check and the write are
+    one step under the lock, so two runs can't both take the last view.
+    """
     with _Locked("linkedin-activity"):
+        now = now if now is not None else time.time()
         data = _read_activity()
-        data[kind] = [t for t in data.get(kind, []) if t >= keep_from] + [now] * n
-        other = "profiles" if kind == "searches" else "searches"
-        data[other] = [t for t in data.get(other, []) if t >= keep_from]
-        try:
-            _write_json_atomic(_home() / "linkedin-activity.json", data)
-        except Exception as exc:
-            print(f"  (could not write down LinkedIn activity: {exc})")
+        views = [t for t in data["profiles"] if t > now - DAY_SECONDS]
+        if len(views) >= search_limits()["profiles"]:
+            return None
+        # A time ahead of this clock (another computer's, imported) is left
+        # out of the gap: waiting for it could never end.
+        last = max((t for t in views if t <= now), default=None)
+        if last is not None and now - last < PROFILE_GAP:
+            return int(last + PROFILE_GAP - now) + 1
+        _charge(data, "profiles", 1, now)
+        return 0
 
 
 def search_limits():
@@ -1405,7 +1528,9 @@ def search_limits():
         except (TypeError, ValueError):
             return default
     return {"daily": num(data.get("daily"), DEFAULT_DAILY_SEARCHES),
-            "monthly": num(data.get("monthly"), DEFAULT_MONTHLY_SEARCHES)}
+            "monthly": num(data.get("monthly"), DEFAULT_MONTHLY_SEARCHES),
+            # 0 is no cap for searches, never for profile views.
+            "profiles": num(data.get("profiles"), DEFAULT_DAILY_PROFILES) or DEFAULT_DAILY_PROFILES}
 
 
 def linkedin_usage(now=None):
@@ -1427,6 +1552,11 @@ def searches_left(now=None):
     return (left_day, "daily") if left_day <= left_month else (left_month, "monthly")
 
 
+def profiles_left(now=None):
+    """How many more profiles may be opened: the cap less the last 24 hours' views."""
+    return max(0, search_limits()["profiles"] - linkedin_usage(now)["profiles_today"])
+
+
 def _when(ts):
     t = datetime.fromtimestamp(ts)
     hour = t.hour % 12 or 12
@@ -1440,6 +1570,14 @@ def _day(ts):
 
 def budget_message(kind):
     lim = search_limits()
+    if kind == "profiles":
+        # The one that stops counting soonest frees the next view.
+        now = time.time()
+        views = sorted(t for t in _read_activity()["profiles"] if t > now - DAY_SECONDS)
+        extra = len(views) - lim["profiles"]
+        free = f" The next one frees up at {_when(views[extra] + DAY_SECONDS)}." if extra >= 0 else ""
+        return (f"Today's profile views ({lim['profiles']}) are used — they count the last 24 hours.{free} "
+                f"No profile was opened and nothing was recorded, so the next run starts with the same person.")
     if kind == "monthly":
         when = _day(next_month_start_pacific())
         return (f"This month's search budget ({lim['monthly']}) is used. It starts again on {when} "
@@ -1751,6 +1889,68 @@ BRIDGE_RESULTS_JS = r"""
             }
           });
 
+          // How many mutual connections each result shares with you: the line
+          // under its card, "Maya Chen and 23 other mutual connections". Read
+          // from the page's text in order, each count going to the result named
+          // above it. A name two results share ("LinkedIn Member") takes none:
+          // no count is better than someone else's. English only; any other
+          // wording reads as no count, never a wrong one.
+          // mutualCount: begin
+          const mutualCount = (text) => {
+            const l = String(text || '').replace(/\s+/g, ' ').trim();
+            if (!/\bmutual connections?$/i.test(l) || l.length > 300) return null;
+            const num = (s) => Number(String(s).replace(/,/g, ''));
+            const names = (s) => s.split(/\s*,\s*|\s+and\s+/).filter(Boolean).length;
+            let m = l.match(/^([\d,]+) mutual connections?$/i);
+            if (m) return num(m[1]);
+            m = l.match(/^(.*?)\s*\band ([\d,]+) others? mutual connections?$/i);
+            if (m) return Math.max(1, names(m[1])) + num(m[2]);
+            if (/\bis a mutual connection$/i.test(l)) return 1;
+            m = l.match(/^(.+?)\s+are mutual connections$/i);
+            if (m) return names(m[1]);
+            return null;
+          };
+          // mutualCount: end
+          const byName = new Map();
+          for (const r of byHref.values()) {
+            if (r.name) byName.set(r.name, byName.has(r.name) ? null : r);
+          }
+          const nameAt = (l) => {
+            if (byName.has(l)) return byName.get(l);
+            for (const [name, r] of byName) {
+              if (l.startsWith(name) && /^\s*[\u2022\u00b7|,(]/.test(l.slice(name.length))) return r;
+            }
+            return undefined;
+          };
+          // The rest of a mutual line when the names are drawn above it, each on
+          // a line of its own ("Maya Chen" / "and 23 other mutual connections").
+          const tailOf = (l) => /^(and [\d,]+ others? mutual connections?|is a mutual connection|are mutual connections)$/i.test(l);
+          let current = null;
+          const give = (n) => {
+            if (current && current.mutualCount == null && Number.isFinite(n) && n >= 1) current.mutualCount = n;
+          };
+          const lines = (root.innerText || '').split('\n').map(clean).filter(Boolean);
+          for (let i = 0; i < lines.length; i++) {
+            const l = lines[i];
+            // Those names are links to your own connections, so they're on the
+            // page as people too, but they are the line, not a new result: read
+            // them with the rest of it, and the count stays with the card it's on.
+            let j = i;
+            while (j < lines.length && j - i < 3 && !tailOf(lines[j]) && nameAt(lines[j]) !== undefined) j++;
+            if (j > i && j < lines.length && tailOf(lines[j])) {
+              give(mutualCount(`${lines.slice(i, j).join(', ')} ${lines[j]}`));
+              i = j;
+              continue;
+            }
+            const n = mutualCount(l);
+            if (n != null) {
+              give(n);
+              continue;
+            }
+            const hit = nameAt(l);
+            if (hit !== undefined) current = hit;
+          }
+
           const results = [...byHref.values()]
             .filter((r) => r.name && r.name.length >= 2)
             .map(({ named, ...r }) => r);
@@ -2046,6 +2246,30 @@ def _page_limit_hint(next_page):
     return f"Run again with --deeper, and no --max-pages, to carry on from page {next_page}."
 
 
+def _wait_for_profile_view(page):
+    """Take one of today's profile views, first waiting out PROFILE_GAP.
+
+    None once it is taken and the profile may open; "budget" when the last 24
+    hours' views are used; "stopped" when Stop, or a closed window, came
+    during the wait. Nothing else is written down.
+    """
+    def tick(left):
+        if not _window_closed(page):          # a closed window sets the stop
+            print(f"    {left}s to go", flush=True)
+
+    while True:
+        wait = take_profile_view()
+        if wait is None:
+            return "budget"
+        if not wait:
+            return None
+        print(f"  Waiting {wait}s before opening a profile: at least {PROFILE_GAP}s pass "
+              f"between any two.", flush=True)
+        if not interruptible_sleep(wait, on_tick=tick, step=10) or _window_closed(page):
+            print("  Stopped before opening their profile.")
+            return "stopped"
+
+
 def _scrape_one_bridge(page, bridge_name, bridge_id, profile_url, max_pages=LINKEDIN_MAX_PAGES,
                        start_page=1, urn=None, on_save=None):
     """
@@ -2060,6 +2284,7 @@ def _scrape_one_bridge(page, bridge_name, bridge_id, profile_url, max_pages=LINK
       urn      the id LinkedIn searches their connections by
       found    how many people were read in all
       limited  LinkedIn's monthly search limit ended the read
+      budget   "daily", "monthly" or "profiles": that budget ended it (status "budget")
 
     Timing is calibrated from successful runs:
     - 30s profile render wait (LinkedIn is slow)
@@ -2076,9 +2301,17 @@ def _scrape_one_bridge(page, bridge_name, bridge_id, profile_url, max_pages=LINK
         # one fewer profile view, which is what LinkedIn counts most.
         print(f"  Going straight to {bridge_name}'s connections (their search id is known).")
     else:
-        # Step 1: Navigate to profile (wait_until="commit" — don't wait for full load)
+        # Step 1: take one of today's profile views, at least PROFILE_GAP after
+        # the last one. With none left nothing is opened or recorded: the batch
+        # ends (BudgetReached, which says why) and the next run tries them again.
+        why = _wait_for_profile_view(page)
+        if why == "budget":
+            reach["budget"] = "profiles"
+        if why:
+            return [], why, reach
+
+        # Navigate to profile (wait_until="commit" — don't wait for full load)
         print(f"  Opening {bridge_name}'s profile...")
-        charge_linkedin("profiles")
         try:
             page.goto(profile_url, wait_until="commit")
         except Exception as e:
@@ -2437,6 +2670,10 @@ def scrape_bridge(bridge_name, headless=False, max_pages=LINKEDIN_MAX_PAGES, dee
                   f"Choose more than {max_pages} pages to read further.")
             return _read([], "finished")
         start_page, urn = nxt, (entry or {}).get("urn")
+    # Without their search id, the read opens their profile: with no profile
+    # views left, stop before a browser opens.
+    if not urn and profiles_left() <= 0:
+        raise BudgetReached(0, "profiles")
 
     print(f"\n=== {'Carrying on with' if start_page > 1 else 'Scraping'} connections of {bridge_name} ===\n")
 
@@ -2506,6 +2743,17 @@ def scrape_bridge(bridge_name, headless=False, max_pages=LINKEDIN_MAX_PAGES, dee
         # link; a profile that didn't render is "unclear" and marks nothing.
         mark_bridge_hidden(profile_url, bridge_name)
         print("  Their connections are not visible any more — keeping what is mapped.")
+    elif status in ("private", "empty") and profile_url and not stop_requested():
+        # A read from page 1 that found their list hidden, or LinkedIn showing no
+        # one in it. Noted here rather than by the batch alone: a one-person scan
+        # (a card's Scan or Rescan, the Degrees panel's Ready to scan) that found
+        # it and said nothing left them "not scanned yet", offered again forever.
+        # A stop that landed meanwhile concludes nothing, as in the batch. TRAPS §15.
+        record_bridge_skip(profile_url, bridge_name,
+                           "no visible connections" if status == "private"
+                           else "LinkedIn showed no one in their list")
+        print("  Their connections are hidden — noted, and skipped from now on."
+              if status == "private" else "  LinkedIn shows no one in their list — noted, and skipped from now on.")
 
     if status == "stopped" or stop_requested():
         return _read(read, "stopped")
@@ -2914,6 +3162,8 @@ def auto_bridge_all(headless=False, log_fn=None, retry_private=False, max_bridge
     use, lim = linkedin_usage(), search_limits()
     log(f"Search budget: {use['searches_today']} of {lim['daily'] or 'no limit'} used today, "
         f"{use['searches_month']} of {lim['monthly'] or 'no limit'} this month")
+    log(f"Profile views: {use['profiles_today']} of {lim['profiles']} in the last 24 hours, "
+        f"at least {PROFILE_GAP}s apart")
 
     # Get all degree-1 connections (filtered by active user)
     D1_LIMIT = 20000
@@ -3095,15 +3345,11 @@ def auto_bridge_all(headless=False, log_fn=None, retry_private=False, max_bridge
                 results.append({"name": name, "tier": tier, "found": 0, "status": "private"})
                 if from_page > 1:
                     log("  Their connections are hidden now — keeping what's mapped")
-                elif url:
-                    record_bridge_skip(url, name, "no visible connections")
-                    log("  Connections are hidden — noted, and skipped from now on")
+                # From page 1, scrape_bridge has noted them in bridge-skips.json.
             elif status == "empty":
+                # scrape_bridge has noted them in bridge-skips.json, and said so.
                 healthy = True
                 results.append({"name": name, "tier": tier, "found": 0, "status": "private"})
-                if url:
-                    record_bridge_skip(url, name, "LinkedIn showed no one in their list")
-                log("  LinkedIn shows no one in their list — noted, and skipped from now on")
             elif status == "finished":
                 healthy = True
                 results.append({"name": name, "tier": tier, "found": 0, "status": "finished"})
@@ -3126,9 +3372,12 @@ def auto_bridge_all(headless=False, log_fn=None, retry_private=False, max_bridge
             log("Stopped.")
             raise
         except BudgetReached as exc:
-            results.append({"name": name, "tier": tier, "found": exc.found, "status": "done" if exc.found else "unclear"})
+            # The profile cap stops before their profile opens: nothing was
+            # tried, so they aren't counted, and the next run tries them again.
+            if exc.found or exc.kind != "profiles":
+                results.append({"name": name, "tier": tier, "found": exc.found, "status": "done" if exc.found else "unclear"})
             log("  " + budget_message(exc.kind))
-            stopped_early = f"the {exc.kind} search budget"
+            stopped_early = str(exc)
             break
         except CoolingDown as exc:
             log("  " + str(exc))
@@ -3224,6 +3473,8 @@ def rescrape_bridge(bridge_name, headless=False, max_pages=LINKEDIN_MAX_PAGES):
         raise CoolingDown(cd)
     if searches_left()[0] <= 0:
         raise BudgetReached(0, searches_left()[1])
+    if profiles_left() <= 0:              # a fresh read opens their profile
+        raise BudgetReached(0, "profiles")
 
     print(f"Deleting old cluster data...")
     bridge_id = delete_bridge_cluster(bridge_name)
@@ -3423,7 +3674,7 @@ def run_server(port=5555):
 
     server = HTTPServer(("127.0.0.1", port), ScrapeHandler)
     print(f"\n{'='*50}")
-    print(f"  6 Degrees Scraper Server")
+    print(f"  Six Degrees scanner server")
     print(f"  Running on http://localhost:{port}")
     print(f"{'='*50}")
     print(f"\n  Open your visualization and use the Setup page buttons.")
@@ -3438,7 +3689,7 @@ def run_server(port=5555):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        description="6 Degrees LinkedIn Scraper",
+        description="Six Degrees LinkedIn scanner",
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog="""
 Examples:
@@ -3482,9 +3733,16 @@ Examples:
                         help="Carry on with people already mapped, from the page their last read "
                              "stopped at. With --auto-bridge: alongside new people. With --bridge: that person.")
     parser.add_argument("--bridge-url", type=str,
-                        help="Carry on with one person, found by their LinkedIn profile URL (implies --deeper)")
+                        help="Carry on with one person, found by their LinkedIn profile URL (implies "
+                             "--deeper, unless --from-start)")
+    parser.add_argument("--from-start", action="store_true",
+                        help="With --bridge-url: read their list from page 1 instead, as --bridge does. "
+                             "The app's Scan buttons use it, since two connections can share a name")
     parser.add_argument("--only-unfinished", action="store_true",
                         help="With --auto-bridge: only people whose read was cut short (Resume all)")
+    parser.add_argument("--save-photos", action="store_true",
+                        help="Save the profile photos still kept as links to LinkedIn, then exit "
+                             "(every scan also does this at its end)")
     parser.add_argument("--headless", action="store_true", help="Run browser in headless mode")
     args = parser.parse_args()
     args.max_pages = max(1, min(LINKEDIN_MAX_PAGES, args.max_pages))
@@ -3493,6 +3751,8 @@ Examples:
     def _say_why(exc_type, exc, tb):
         if issubclass(exc_type, SaveFailed):
             print(f"\n  The app could not save what was scraped: {exc}\n", file=sys.stderr)
+        elif issubclass(exc_type, TryLater):
+            print(f"\n  The photos weren't saved: {exc}.\n", file=sys.stderr)
         elif issubclass(exc_type, NotSignedIn):
             print("\n  LinkedIn isn't signed in, so nothing was read. Sign in with --login, then run it again.\n",
                   file=sys.stderr)
@@ -3519,6 +3779,10 @@ Examples:
     if args.login:
         raise SystemExit(0 if open_login_window() else 1)
 
+    if args.save_photos:
+        save_waiting_photos(say_none=True)
+        raise SystemExit(0)
+
     if not args.server:
         resolve_active_user()
 
@@ -3552,7 +3816,10 @@ Examples:
         elif args.rescrape:
             rescrape_bridge(args.rescrape, headless=args.headless, max_pages=args.max_pages)
         elif args.bridge_url:
-            scrape_bridge(None, headless=args.headless, max_pages=args.max_pages, deeper=True,
+            # Carries on unless --from-start. A server started before a pull (TRAPS §22)
+            # still sends Resume as a bare --bridge-url, and a read from page 1 would
+            # spend searches on pages already read.
+            scrape_bridge(None, headless=args.headless, max_pages=args.max_pages, deeper=not args.from_start,
                           profile_url=args.bridge_url)
         elif args.bridge:
             scrape_bridge(args.bridge, headless=args.headless, max_pages=args.max_pages, deeper=args.deeper)
@@ -3569,6 +3836,15 @@ Examples:
                 print("\nNo connections yet — walking your whole connections list.")
                 print("This takes a couple of minutes and captures photos.\n")
                 scrape_connections(headless=args.headless, full_walk=True)
+
+        # Photos an older version kept as links, saved once the scan's own are
+        # (so a photo it just read wins). Not after a stop, which ends the run
+        # now, and a failure here doesn't undo a scan that worked.
+        if not args.server and not stop_requested():
+            try:
+                save_waiting_photos()
+            except (SaveFailed, TryLater) as e:
+                print(f"  The photos kept as links weren't saved this time: {e}")
     except (BudgetReached, CoolingDown) as exc:
         print("\n  " + (budget_message(exc.kind) if isinstance(exc, BudgetReached) else str(exc)))
         raise SystemExit(0)

package/.next/standalone/server.js

@@ -9,7 +9,7 @@ const currentPort = parseInt(process.env.PORT, 10) || 3000
 const hostname = process.env.HOSTNAME || '0.0.0.0'
 
 let keepAliveTimeout = parseInt(process.env.KEEP_ALIVE_TIMEOUT, 10)
-const nextConfig = {"env":{},"webpack":null,"typescript":{"ignoreBuildErrors":false},"typedRoutes":false,"distDir":"./.next","cleanDistDir":true,"assetPrefix":"","cacheMaxMemorySize":52428800,"configOrigin":"next.config.mjs","useFileSystemPublicRoutes":true,"generateEtags":true,"pageExtensions":["tsx","ts","jsx","js"],"instrumentationClientInject":[],"poweredByHeader":true,"compress":true,"images":{"deviceSizes":[640,750,828,1080,1200,1920,2048,3840],"imageSizes":[32,48,64,96,128,256,384],"path":"/_next/image","loader":"default","loaderFile":"","domains":[],"disableStaticImages":false,"minimumCacheTTL":14400,"formats":["image/webp"],"maximumRedirects":3,"maximumResponseBody":50000000,"dangerouslyAllowLocalIP":false,"dangerouslyAllowSVG":false,"contentSecurityPolicy":"script-src 'none'; frame-src 'none'; sandbox;","contentDispositionType":"attachment","localPatterns":[{"pathname":"**","search":""}],"remotePatterns":[],"qualities":[75],"unoptimized":false,"customCacheHandler":false},"devIndicators":{"position":"bottom-left"},"onDemandEntries":{"maxInactiveAge":60000,"pagesBufferLength":5},"basePath":"","sassOptions":{},"trailingSlash":false,"i18n":null,"productionBrowserSourceMaps":false,"excludeDefaultMomentLocales":true,"reactProductionProfiling":false,"reactStrictMode":null,"reactMaxHeadersLength":6000,"httpAgentOptions":{"keepAlive":true},"logging":{"serverFunctions":true,"browserToTerminal":"warn"},"compiler":{},"expireTime":31536000,"staticPageGenerationTimeout":60,"output":"standalone","modularizeImports":{"@mui/icons-material":{"transform":"@mui/icons-material/{{member}}"},"lodash":{"transform":"lodash/{{member}}"}},"outputFileTracingRoot":"/home/runner/work/six-degrees/six-degrees","enablePrerenderSourceMaps":true,"cacheComponents":false,"cacheLife":{"default":{"stale":300,"revalidate":900,"expire":4294967294},"seconds":{"stale":30,"revalidate":1,"expire":60},"minutes":{"stale":300,"revalidate":60,"expire":3600},"hours":{"stale":300,"revalidate":3600,"expire":86400},"days":{"stale":300,"revalidate":86400,"expire":604800},"weeks":{"stale":300,"revalidate":604800,"expire":2592000},"max":{"stale":300,"revalidate":2592000,"expire":31536000}},"cacheHandlers":{},"experimental":{"appNewScrollHandler":true,"coldCacheBadge":false,"devValidationWorker":true,"useSkewCookie":false,"cssChunking":true,"multiZoneDraftMode":false,"appNavFailHandling":false,"prerenderEarlyExit":true,"serverMinification":true,"linkNoTouchStart":false,"caseSensitiveRoutes":false,"cachedNavigations":false,"dynamicOnHover":false,"useOffline":false,"varyParams":true,"optimisticRouting":true,"instrumentationClientRouterTransitionEvents":false,"prefetchInlining":{"maxSize":2048,"maxBundleSize":10240},"preloadEntriesOnStart":true,"clientRouterFilter":true,"clientRouterFilterRedirects":false,"fetchCacheKeyPrefix":"","proxyPrefetch":"flexible","optimisticClientCache":true,"manualClientBasePath":false,"cpus":3,"memoryBasedWorkersCount":false,"imgOptConcurrency":null,"imgOptOperationCache":null,"imgOptTimeoutInSeconds":7,"imgOptMaxInputPixels":268402689,"imgOptSequentialRead":null,"isrFlushToDisk":true,"workerThreads":false,"optimizeCss":false,"nextScriptWorkers":false,"scrollRestoration":false,"externalDir":false,"devMemoryThresholdRestart":true,"disableOptimizedLoading":false,"gzipSize":true,"craCompat":false,"esmExternals":true,"fullySpecified":false,"swcTraceProfiling":false,"forceSwcTransforms":false,"requestInsights":false,"largePageDataBytes":128000,"typedEnv":false,"parallelServerCompiles":false,"parallelServerBuildTraces":false,"ppr":false,"authInterrupts":false,"webpackMemoryOptimizations":false,"optimizeServerReact":true,"strictRouteTypes":false,"useTypeScriptCli":true,"removeUncaughtErrorAndRejectionListeners":false,"validateRSCRequestHeaders":true,"staleTimes":{"dynamic":0,"static":300},"reactDebugChannel":true,"serverComponentsHmrCache":true,"serverComponentsHmrCancellation":false,"staticGenerationMaxConcurrency":8,"staticGenerationMinPagesPerWorker":25,"transitionIndicator":false,"gestureTransition":false,"inlineCss":false,"useCache":false,"globalNotFound":false,"browserDebugInfoInTerminal":"warn","lockDistDir":true,"proxyClientMaxBodySize":10485760,"hideLogsAfterAbort":false,"mcpServer":true,"turbopackFileSystemCacheForDev":true,"turbopackFileSystemCacheForBuild":true,"turbopackInferModuleSideEffects":true,"turbopackPluginRuntimeStrategy":"childProcesses","turbopackMemoryEvictionMode":"auto","optimizePackageImports":["lucide-react","date-fns","lodash-es","ramda","antd","react-bootstrap","ahooks","@ant-design/icons","@headlessui/react","@headlessui-float/react","@heroicons/react/20/solid","@heroicons/react/24/solid","@heroicons/react/24/outline","@visx/visx","@tremor/react","rxjs","@mui/material","@mui/icons-material","recharts","react-use","effect","@effect/schema","@effect/platform","@effect/platform-node","@effect/platform-browser","@effect/platform-bun","@effect/sql","@effect/sql-mssql","@effect/sql-mysql2","@effect/sql-pg","@effect/sql-sqlite-node","@effect/sql-sqlite-bun","@effect/sql-sqlite-wasm","@effect/sql-sqlite-react-native","@effect/rpc","@effect/rpc-http","@effect/typeclass","@effect/experimental","@effect/opentelemetry","@material-ui/core","@material-ui/icons","@tabler/icons-react","mui-core","react-icons/ai","react-icons/bi","react-icons/bs","react-icons/cg","react-icons/ci","react-icons/di","react-icons/fa","react-icons/fa6","react-icons/fc","react-icons/fi","react-icons/gi","react-icons/go","react-icons/gr","react-icons/hi","react-icons/hi2","react-icons/im","react-icons/io","react-icons/io5","react-icons/lia","react-icons/lib","react-icons/lu","react-icons/md","react-icons/pi","react-icons/ri","react-icons/rx","react-icons/si","react-icons/sl","react-icons/tb","react-icons/tfi","react-icons/ti","react-icons/vsc","react-icons/wi"],"useCacheTimeout":54,"instantInsights":{"validationLevel":"warning"},"trustHostHeader":false,"isExperimentalCompile":false},"htmlLimitedBots":"[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight","bundlePagesRouterDependencies":false,"configFileName":"next.config.mjs","outputFileTracingExcludes":{"*":["dist/*.app/**","dist/*.dmg","dist/staging/**","dist/electron-stage/**","docs/**","tests/**",".git",".git/**","*.log","scripts/dmg/**","scripts/pin-python-packages.mjs","desktop/**","node_modules/electron/**","node_modules/@electron/**","site/**"]},"serverExternalPackages":["node:sqlite"],"repoRoot":"/home/runner/work/six-degrees/six-degrees","turbopack":{"root":"/home/runner/work/six-degrees/six-degrees"},"distDirRoot":".next"}
+const nextConfig = {"env":{},"webpack":null,"typescript":{"ignoreBuildErrors":false},"typedRoutes":false,"distDir":"./.next","cleanDistDir":true,"assetPrefix":"","cacheMaxMemorySize":52428800,"configOrigin":"next.config.mjs","useFileSystemPublicRoutes":true,"generateEtags":true,"pageExtensions":["tsx","ts","jsx","js"],"instrumentationClientInject":[],"poweredByHeader":true,"compress":true,"images":{"deviceSizes":[640,750,828,1080,1200,1920,2048,3840],"imageSizes":[32,48,64,96,128,256,384],"path":"/_next/image","loader":"default","loaderFile":"","domains":[],"disableStaticImages":false,"minimumCacheTTL":14400,"formats":["image/webp"],"maximumRedirects":3,"maximumResponseBody":50000000,"dangerouslyAllowLocalIP":false,"dangerouslyAllowSVG":false,"contentSecurityPolicy":"script-src 'none'; frame-src 'none'; sandbox;","contentDispositionType":"attachment","localPatterns":[{"pathname":"**","search":""}],"remotePatterns":[],"qualities":[75],"unoptimized":false,"customCacheHandler":false},"devIndicators":{"position":"bottom-left"},"onDemandEntries":{"maxInactiveAge":60000,"pagesBufferLength":5},"basePath":"","sassOptions":{},"trailingSlash":false,"i18n":null,"productionBrowserSourceMaps":false,"excludeDefaultMomentLocales":true,"reactProductionProfiling":false,"reactStrictMode":null,"reactMaxHeadersLength":6000,"httpAgentOptions":{"keepAlive":true},"logging":{"serverFunctions":true,"browserToTerminal":"warn"},"compiler":{},"expireTime":31536000,"staticPageGenerationTimeout":60,"output":"standalone","modularizeImports":{"@mui/icons-material":{"transform":"@mui/icons-material/{{member}}"},"lodash":{"transform":"lodash/{{member}}"}},"outputFileTracingRoot":"/home/runner/work/six-degrees/six-degrees","enablePrerenderSourceMaps":true,"cacheComponents":false,"cacheLife":{"default":{"stale":300,"revalidate":900,"expire":4294967294},"seconds":{"stale":30,"revalidate":1,"expire":60},"minutes":{"stale":300,"revalidate":60,"expire":3600},"hours":{"stale":300,"revalidate":3600,"expire":86400},"days":{"stale":300,"revalidate":86400,"expire":604800},"weeks":{"stale":300,"revalidate":604800,"expire":2592000},"max":{"stale":300,"revalidate":2592000,"expire":31536000}},"cacheHandlers":{},"experimental":{"appNewScrollHandler":true,"coldCacheBadge":false,"devValidationWorker":true,"useSkewCookie":false,"cssChunking":true,"multiZoneDraftMode":false,"appNavFailHandling":false,"prerenderEarlyExit":true,"serverMinification":true,"linkNoTouchStart":false,"caseSensitiveRoutes":false,"cachedNavigations":false,"dynamicOnHover":false,"useOffline":false,"varyParams":true,"optimisticRouting":true,"instrumentationClientRouterTransitionEvents":false,"prefetchInlining":{"maxSize":2048,"maxBundleSize":10240},"preloadEntriesOnStart":true,"clientRouterFilter":true,"clientRouterFilterRedirects":false,"fetchCacheKeyPrefix":"","proxyPrefetch":"flexible","optimisticClientCache":true,"manualClientBasePath":false,"cpus":3,"memoryBasedWorkersCount":false,"imgOptConcurrency":null,"imgOptOperationCache":null,"imgOptTimeoutInSeconds":7,"imgOptMaxInputPixels":268402689,"imgOptSequentialRead":null,"isrFlushToDisk":true,"workerThreads":false,"optimizeCss":false,"nextScriptWorkers":false,"scrollRestoration":false,"externalDir":false,"devMemoryThresholdRestart":true,"disableOptimizedLoading":false,"gzipSize":true,"craCompat":false,"esmExternals":true,"fullySpecified":false,"swcTraceProfiling":false,"forceSwcTransforms":false,"requestInsights":false,"largePageDataBytes":128000,"typedEnv":false,"parallelServerCompiles":false,"parallelServerBuildTraces":false,"ppr":false,"authInterrupts":false,"webpackMemoryOptimizations":false,"optimizeServerReact":true,"strictRouteTypes":false,"useTypeScriptCli":true,"removeUncaughtErrorAndRejectionListeners":false,"validateRSCRequestHeaders":true,"staleTimes":{"dynamic":0,"static":300},"reactDebugChannel":true,"serverComponentsHmrCache":true,"serverComponentsHmrCancellation":false,"staticGenerationMaxConcurrency":8,"staticGenerationMinPagesPerWorker":25,"transitionIndicator":false,"gestureTransition":false,"inlineCss":false,"useCache":false,"globalNotFound":false,"browserDebugInfoInTerminal":"warn","lockDistDir":true,"proxyClientMaxBodySize":10485760,"hideLogsAfterAbort":false,"mcpServer":true,"turbopackFileSystemCacheForDev":true,"turbopackFileSystemCacheForBuild":true,"turbopackInferModuleSideEffects":true,"turbopackPluginRuntimeStrategy":"childProcesses","turbopackMemoryEvictionMode":"auto","optimizePackageImports":["lucide-react","date-fns","lodash-es","ramda","antd","react-bootstrap","ahooks","@ant-design/icons","@headlessui/react","@headlessui-float/react","@heroicons/react/20/solid","@heroicons/react/24/solid","@heroicons/react/24/outline","@visx/visx","@tremor/react","rxjs","@mui/material","@mui/icons-material","recharts","react-use","effect","@effect/schema","@effect/platform","@effect/platform-node","@effect/platform-browser","@effect/platform-bun","@effect/sql","@effect/sql-mssql","@effect/sql-mysql2","@effect/sql-pg","@effect/sql-sqlite-node","@effect/sql-sqlite-bun","@effect/sql-sqlite-wasm","@effect/sql-sqlite-react-native","@effect/rpc","@effect/rpc-http","@effect/typeclass","@effect/experimental","@effect/opentelemetry","@material-ui/core","@material-ui/icons","@tabler/icons-react","mui-core","react-icons/ai","react-icons/bi","react-icons/bs","react-icons/cg","react-icons/ci","react-icons/di","react-icons/fa","react-icons/fa6","react-icons/fc","react-icons/fi","react-icons/gi","react-icons/go","react-icons/gr","react-icons/hi","react-icons/hi2","react-icons/im","react-icons/io","react-icons/io5","react-icons/lia","react-icons/lib","react-icons/lu","react-icons/md","react-icons/pi","react-icons/ri","react-icons/rx","react-icons/si","react-icons/sl","react-icons/tb","react-icons/tfi","react-icons/ti","react-icons/vsc","react-icons/wi"],"useCacheTimeout":54,"instantInsights":{"validationLevel":"warning"},"trustHostHeader":false,"isExperimentalCompile":false},"htmlLimitedBots":"[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight","bundlePagesRouterDependencies":false,"configFileName":"next.config.mjs","outputFileTracingExcludes":{"*":["dist/*.app/**","dist/*.dmg","dist/staging/**","dist/electron-stage/**","docs/**","tests/**",".git",".git/**","*.log","scripts/dmg/**","scripts/pin-python-packages.mjs","scripts/check-site-version.mjs","desktop/**","node_modules/electron/**","node_modules/@electron/**","site/**","*.csv","*.sqlite*","*.sixdegrees","chrome-profile/**"]},"serverExternalPackages":["node:sqlite"],"repoRoot":"/home/runner/work/six-degrees/six-degrees","turbopack":{"root":"/home/runner/work/six-degrees/six-degrees"},"distDirRoot":".next"}
 
 process.env.__NEXT_PRIVATE_STANDALONE_CONFIG = JSON.stringify(nextConfig)