six-degrees 0.4.7 → 0.4.9

package/.next/standalone/scripts/scrape.py

@@ -59,7 +59,7 @@ _ENV_LOCAL = _load_env_local()
 # a server they don't control.
 APP_URL = os.getenv("APP_URL", "http://localhost:3000")
 MESSAGING_URL = "https://www.linkedin.com/messaging/"
-MESSAGE_SCROLLS = 6               # scrolls down the conversation list, a few seconds apart
+MESSAGE_SCROLLS = 15
 
 CONNECTIONS_URL = "https://www.linkedin.com/mynetwork/invite-connect/connections/"
 
@@ -330,7 +330,7 @@ def ensure_logged_in(page, timeout_s=LOGIN_WAIT_SECONDS, log_fn=None, stop_on_ch
     if stop_on_checkpoint and signed_out:
         reason = "a security check" if wall == "checkpoint" else "being signed out"
         _keep_pushback_evidence(page, reason)
-        set_cooldown(seconds=DAY_SECONDS, reason=f"LinkedIn pushed back: {reason}")
+        set_cooldown(seconds=AUTO_PUSHBACK_REST if EXPERIMENT["on"] else DAY_SECONDS, reason=f"LinkedIn pushed back: {reason}")
         raise LinkedInPushedBack(0, reason)
 
     if _has_session_cookie(context) and not _looks_logged_out(page) and not wall:
@@ -1282,10 +1282,17 @@ LEGACY_PAGES_READ = 10            # how far every read before 0.1.6 went, at mos
 # A sitting of SESSION_PAGES searches, then a long rest; searches only in the
 # daytime on this computer's clock; and at the daily budget it waits for the
 # budget to free up instead of stopping. Pages are saved one at a time.
+#
+# Its own ceilings sit under whatever budget is set on the Scan page: the account
+# restricted on 2026-09-28 did 373 searches in 24 hours with the budget raised to
+# 500. Auto scan stays far below that however high the budget is set.
 EXPERIMENT = {"on": False, "pages": 0, "wire": 0}
-SESSION_PAGES = 10                # searches in one sitting
-SESSION_REST = 45 * 60            # the rest after each sitting
-DRIP_HOURS = (9, 19)              # searches only from 09:00 to 19:00, local time
+SESSION_PAGES = 8                 # searches in one sitting
+SESSION_REST = 60 * 60            # the rest after each sitting
+DRIP_HOURS = (9, 18)              # searches only from 09:00 to 18:00, local time
+AUTO_DAY_CAP = 40                 # at most this many searches in any 24 hours
+AUTO_WEEK_CAP = 200               # and this many in any 7 days
+AUTO_PUSHBACK_REST = 2 * 24 * 3600   # after any LinkedIn check, nothing for two days
 WIRE_SAMPLES = 5                  # raw LinkedIn responses kept per run, for research
 
 
@@ -1899,6 +1906,18 @@ def voyager_total(payload):
     return None
 
 
+_WORDS = {"body", "text", "subject", "attributedBody", "snippet", "messages", "headline", "firstName", "lastName", "rootUrl", "artifacts", "sponsoredPreviewText", "customPronoun", "advertiserLabel"}
+
+
+def _redact_words(x):
+    """A copy of a LinkedIn response with every piece of writing and every name removed."""
+    if isinstance(x, dict):
+        return {k: ("<removed>" if k in _WORDS else _redact_words(v)) for k, v in x.items()}
+    if isinstance(x, list):
+        return [_redact_words(v) for v in x]
+    return x
+
+
 def _wire_sample(data, prefix="search"):
     if EXPERIMENT["wire"] >= WIRE_SAMPLES:
         return
@@ -1908,6 +1927,10 @@ def _wire_sample(data, prefix="search"):
             return
         folder = _home() / "wire-samples"
         folder.mkdir(parents=True, exist_ok=True)
+        if prefix == "messages":
+            # Messaging responses carry what people wrote, and their names: a sample
+            # keeps only the shape (keys, links, dates, counts), never the words.
+            raw = json.dumps(_redact_words(data))
         (folder / f"{prefix}-{int(time.time() * 1000)}.json").write_text(raw)
         EXPERIMENT["wire"] += 1
     except Exception:
@@ -1948,14 +1971,51 @@ def _wire_merge(page, profile_url, page_results):
     return page_results
 
 
-_PROFILE_LINK = re.compile(r"https://www\\.linkedin\\.com/in/[^/?#\\s\"']+")
+_PROFILE_LINK = re.compile(r"https://www\.linkedin\.com/in/[^/?#\s\"']+")
+
+
+def _dig(x, *keys):
+    """x[k1][k2]..., or None as soon as a step isn't there."""
+    for k in keys:
+        if not isinstance(x, dict):
+            return None
+        x = x.get(k)
+    return x
+
+
+def _link_key(link):
+    """A profile link as the messages sync compares them: no query, one closing slash."""
+    return link.split("?")[0].rstrip("/") + "/" if isinstance(link, str) and "/in/" in link else None
 
 
 def voyager_conversations(payload):
     """Conversations in a LinkedIn messaging response (experimental): per conversation,
-    everyone in it (profile links), when it was last active, and unread messages.
-    Anything with a lastActivityAt and participants, wherever it sits. Never the text."""
+    everyone in it (profile links), when it was last active, unread messages, its link,
+    whether it's a group, and the newest message LinkedIn sent along with it (when,
+    from whom, and its words). Anything with a lastActivityAt and participants,
+    wherever it sits. The words are handed to the app only while Keep my messages is
+    on there (sync_messages); a wire sample never has them (_redact_words)."""
     out = []
+    names = {}
+
+    def text_of(v):
+        return v.get("text", "") if isinstance(v, dict) else (v if isinstance(v, str) else "")
+
+    def latest_of(x):
+        # conversation.messages.elements[]: body.text, deliveredAt, and who sent it
+        # (actor.participantType.member.profileUrl, the same /in/ACoA... link the
+        # participants have, so it tells you from them).
+        msgs = x.get("messages")
+        elements = msgs.get("elements") if isinstance(msgs, dict) else msgs
+        best = None
+        for m in elements if isinstance(elements, list) else []:
+            t = _dig(m, "deliveredAt")
+            if not isinstance(t, (int, float)) or t < 1e12 or (best and t <= best["t"]):
+                continue
+            text = text_of(_dig(m, "body")).strip()
+            best = {"t": int(t), "from": _link_key(_dig(m, "actor", "participantType", "member", "profileUrl")),
+                    "text": text or None}
+        return best
 
     def people(x):
         found = set()
@@ -1966,6 +2026,14 @@ def voyager_conversations(payload):
                 pid = v.get("publicIdentifier")
                 if isinstance(pid, str) and pid and pid != "UNKNOWN":
                     found.add(f"https://www.linkedin.com/in/{pid}/")
+                # Messaging gives a member's link as /in/ACoA… (their member id), not
+                # the /in/name link your connections list has, so their name comes
+                # along to match them up (the app keeps the match, not the name).
+                key = _link_key(v.get("profileUrl"))
+                if key:
+                    name = f"{text_of(v.get('firstName'))} {text_of(v.get('lastName'))}".strip()
+                    if name:
+                        names[key] = name
                 stack.extend(v.values())
             elif isinstance(v, list):
                 stack.extend(v)
@@ -1984,7 +2052,13 @@ def voyager_conversations(payload):
                     who |= people(v)
                 if who:
                     unread = x.get("unreadCount")
-                    out.append({"people": sorted(who), "at": int(at), "unread": int(unread) if isinstance(unread, (int, float)) else None})
+                    link = x.get("conversationUrl")
+                    group = x.get("groupChat")
+                    out.append({"people": sorted(who), "at": int(at), "unread": int(unread) if isinstance(unread, (int, float)) else None,
+                                "names": {u: names[u] for u in who if u in names},
+                                "url": link if isinstance(link, str) and link.startswith("https://www.linkedin.com/messaging/") else None,
+                                "group": group if isinstance(group, bool) else None,
+                                "latest": latest_of(x)})
             for v in x.values():
                 walk(v)
         elif isinstance(x, list):
@@ -1997,8 +2071,9 @@ def voyager_conversations(payload):
 def sync_messages(headless=False):
     """Experimental: read your LinkedIn messages list once, for the Social tab. Opens
     Messaging in your own Chrome, scrolls the list a few times, and keeps from LinkedIn's
-    own data only who each 1:1 conversation is with, when it was last active and whether
-    it has unread messages. No search budget; no message text is read or kept."""
+    own data who each 1:1 conversation is with, when it was last active, whether it has
+    unread messages, its link and who wrote last. The newest message's words go to the
+    app only while Keep my messages is on there. No search budget."""
     from playwright.sync_api import sync_playwright
     kept = []
     with sync_playwright() as p:
@@ -2021,7 +2096,12 @@ def sync_messages(headless=False):
             if stop_requested() or _window_closed(page):
                 break
             try:
-                page.evaluate("""() => { const l = document.querySelector('.msg-conversations-container__conversations-list, [class*="conversations-list"]');
+                # The list scrolls inside its own panel, not the window: find the panel
+                # that holds the conversation links and scroll that to its end.
+                page.evaluate("""() => {
+                  let l = document.querySelector('.msg-conversations-container__conversations-list, [class*="conversations-list"]');
+                  const a = document.querySelector('a[href*="/messaging/thread/"], li[class*="conversation-listitem"]');
+                  for (let e = a; !l && e; e = e.parentElement) if (e.scrollHeight > e.clientHeight + 20) l = e;
                   if (l) l.scrollTop = l.scrollHeight; }""")
             except Exception:
                 pass
@@ -2036,7 +2116,35 @@ def sync_messages(headless=False):
             _wire_sample(data, prefix="messages")
             convos.extend(voyager_conversations(data))
         browser.close()
-    # You are the one in the most conversations; keep the 1:1 ones.
+    _me, people = live_people(convos)
+    print(f"  Found {len(convos)} conversations in LinkedIn's data, {len(people)} with one person.")
+    # The newest message's words go to the app only if Keep my messages is on
+    # there. The app checks too; this way they don't even leave the scanner.
+    keep = False
+    try:
+        r = requests.get(f"{APP_URL}/api/social", headers=app_headers(json_body=False), timeout=15)
+        keep = r.ok and ((r.json() or {}).get("social") or {}).get("keepMessages") is True
+    except Exception:
+        pass
+    if not keep:
+        for v in people.values():
+            if v.get("preview"):
+                v["preview"]["text"] = None
+    try:
+        r = requests.put(f"{APP_URL}/api/social", json={"live": people}, headers=app_headers(), timeout=30)
+        got = r.json() if r.ok else {}
+        matched, unmatched = got.get("people", 0), got.get("unmatched", 0)
+        print(f"  Saved for the Social tab: {matched} matched to your connections, {unmatched} not.")
+    except Exception as e:
+        print(f"  Couldn't save them: {e}")
+
+
+def live_people(convos):
+    """The messages sync's conversations, per person: you are the one in the most
+    conversations, and only the 1:1 ones are kept. For each person, their newest
+    conversation's last activity, unread count, name (for the app to match them to
+    a connection; it keeps the match, not the name), link, who wrote last and the
+    newest message. Returns (you, {profile link: ...})."""
     counts = {}
     for c in convos:
         for u in c["people"]:
@@ -2045,17 +2153,23 @@ def sync_messages(headless=False):
     people = {}
     for c in convos:
         others = [u for u in c["people"] if u != me]
-        if len(others) != 1:
+        if len(others) != 1 or c.get("group") is True:
             continue
         cur = people.get(others[0])
-        if not cur or c["at"] > cur["last"]:
-            people[others[0]] = {"last": c["at"], "unread": c["unread"]}
-    print(f"  Found {len(convos)} conversations in LinkedIn's data, {len(people)} with one person.")
-    try:
-        requests.put(f"{APP_URL}/api/social", json={"live": people}, headers=app_headers(), timeout=30)
-        print("  Saved for the Social tab.")
-    except Exception as e:
-        print(f"  Couldn't save them: {e}")
+        if cur and c["at"] <= cur["last"]:
+            continue
+        entry = {"last": c["at"], "unread": c["unread"], "name": c.get("names", {}).get(others[0]),
+                 "threadUrl": c.get("url"), "lastFromThem": None, "preview": None}
+        latest = c.get("latest")
+        # Who sent it is known only when their link is one of the two in the
+        # conversation; a link in some other form says nothing either way.
+        sender = latest.get("from") if latest else None
+        if sender in (me, others[0]):
+            from_me = sender == me
+            entry["lastFromThem"] = not from_me
+            entry["preview"] = {"t": latest["t"], "fromMe": from_me, "text": latest.get("text")}
+        people[others[0]] = entry
+    return me, people
 
 
 def _drip_wait(label, seconds):
@@ -2095,9 +2209,28 @@ def _drip_before_search():
             if not _drip_wait("Today's searches are used", _seconds_until_search_frees()):
                 return False
             continue
+        wait = _auto_ceiling_wait()
+        if wait:
+            label, seconds = wait
+            if not _drip_wait(label, seconds):
+                return False
+            continue
         return True
 
 
+def _auto_ceiling_wait(now=None):
+    """Auto scan's own ceilings (AUTO_DAY_CAP in 24 hours, AUTO_WEEK_CAP in 7 days):
+    None to go ahead, or (why, seconds until the oldest counted search drops out)."""
+    now = now if now is not None else time.time()
+    searches = sorted(t for t in _read_activity()["searches"] if now - 7 * DAY_SECONDS < t <= now)
+    day = [t for t in searches if now - DAY_SECONDS < t]
+    if len(day) >= AUTO_DAY_CAP:
+        return (f"Auto scan's {AUTO_DAY_CAP} searches for today are used", day[-AUTO_DAY_CAP] + DAY_SECONDS - now + 5)
+    if len(searches) >= AUTO_WEEK_CAP:
+        return (f"Auto scan's {AUTO_WEEK_CAP} searches for this week are used", searches[-AUTO_WEEK_CAP] + 7 * DAY_SECONDS - now + 5)
+    return None
+
+
 def interruptible_sleep(seconds, on_tick=None, step=5):
     """Sleep, but notice a stop request. Returns False if we were interrupted."""
     waited = 0
@@ -4073,7 +4206,7 @@ Examples:
     parser.add_argument("--rescrape", type=str, help="Delete + re-scrape a bridge's cluster from scratch")
     parser.add_argument("--company", type=str, help="Scan everyone the app can see at one company")
     parser.add_argument("--messages", action="store_true",
-                        help="Experimental: read your messages list once for the Social tab (who, when, unread; never text)")
+                        help="Experimental: read your messages list once for the Social tab (who, when, unread; the newest message's words only if Keep my messages is on)")
     parser.add_argument("--experimental", action="store_true",
                         help="Auto-Bridge, experimental: all-day pacing (sittings, rests, daytime only, "
                              "waits for the daily budget) and reading LinkedIn's own data beside the page text")

package/.next/standalone/scripts/site-markdown.mjs

@@ -0,0 +1,188 @@
+// A small Markdown renderer for the website's blog posts, pages and release notes.
+// No dependencies, on purpose: the site builds with nothing but Node. It covers
+// what the posts and CHANGELOG.md use, and nothing more:
+//
+//   # to ####### headings (with an id from their text), paragraphs, blank-line breaks
+//   - and 1. lists, nested by two spaces, with continuation lines
+//   > block quotes, ``` fenced code, --- rules, | tables |
+//   **bold**, *italic*, `code`, [links](url), ![images](src "caption")
+//
+// Raw HTML is escaped, never passed through: a post can't inject markup or script.
+
+export const escapeHtml = (s) => String(s)
+  .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
+
+/** "What it does, for you!" → "what-it-does-for-you" */
+export function slugify(text) {
+  return String(text).toLowerCase()
+    .replace(/<[^>]+>/g, '')
+    .replace(/&[a-z]+;/g, '')
+    .replace(/[^a-z0-9\s-]/g, '')
+    .trim().replace(/\s+/g, '-').replace(/-+/g, '-');
+}
+
+const safeUrl = (url) => (/^(https?:|mailto:|\/|#|\.\.?\/)/i.test(url) || !/^[a-z]+:/i.test(url) ? url : '#');
+
+/** Inline marks, on text that is escaped first. */
+export function inline(md) {
+  let s = escapeHtml(md);
+  const codes = [];
+  s = s.replace(/`([^`]+)`/g, (_, c) => { codes.push(c); return `\u0000${codes.length - 1}\u0000`; });
+  s = s.replace(/!\[([^\]]*)\]\(([^)\s]+)(?:\s+&quot;([^&]*)&quot;)?\)/g,
+    (_, alt, src) => `<img src="${safeUrl(src)}" alt="${alt}" loading="lazy">`);
+  s = s.replace(/\[([^\]]+)\]\(([^)\s]+)\)/g, (_, text, url) => `<a href="${safeUrl(url)}">${text}</a>`);
+  s = s.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>');
+  s = s.replace(/(^|[^\w*])\*(?!\s)(.+?)\*(?!\w)/g, '$1<em>$2</em>');
+  s = s.replace(/\u0000(\d+)\u0000/g, (_, i) => `<code>${codes[Number(i)]}</code>`);
+  return s;
+}
+
+/**
+ * Front matter between --- lines: `key: value`, with [a, b] lists. Returns
+ * { data, body }. A file without it is all body.
+ */
+export function frontMatter(text) {
+  const m = text.match(/^---\n([\s\S]*?)\n---\n?/);
+  if (!m) return { data: {}, body: text };
+  const data = {};
+  for (const line of m[1].split('\n')) {
+    const kv = line.match(/^([A-Za-z][\w-]*):\s*(.*)$/);
+    if (!kv) continue;
+    let v = kv[2].trim();
+    if (/^\[.*\]$/.test(v)) v = v.slice(1, -1).split(',').map((x) => x.trim().replace(/^["']|["']$/g, '')).filter(Boolean);
+    else v = v.replace(/^["'](.*)["']$/, '$1');
+    data[kv[1]] = v;
+  }
+  return { data, body: text.slice(m[0].length) };
+}
+
+/** The same, for an HTML page whose first thing is a <!-- key: value --> comment. */
+export function htmlFrontMatter(text) {
+  const m = text.match(/^<!--\n([\s\S]*?)\n-->\n?/);
+  if (!m) return { data: {}, body: text };
+  return { data: frontMatter(`---\n${m[1]}\n---\n`).data, body: text.slice(m[0].length) };
+}
+
+function table(lines) {
+  const cells = (l) => l.trim().replace(/^\||\|$/g, '').split('|').map((c) => c.trim());
+  const head = cells(lines[0]);
+  const rows = lines.slice(2).map(cells);
+  return `<div class="table-wrap prose-table"><table>\n<thead><tr>${head.map((h) => `<th scope="col">${inline(h)}</th>`).join('')}</tr></thead>\n<tbody>\n${
+    rows.map((r) => `<tr>${r.map((c, i) => (i === 0 ? `<th scope="row">${inline(c)}</th>` : `<td>${inline(c)}</td>`)).join('')}</tr>`).join('\n')
+  }\n</tbody></table></div>`;
+}
+
+/**
+ * Lists: items start with "- " or "1. " at an indent; deeper indents nest. An
+ * item's text, its sub-lists and any later paragraphs stay in their order.
+ */
+function list(lines, start) {
+  const indentOf = (l) => l.match(/^(\s*)/)[1].length;
+  const base = indentOf(lines[start]);
+  const ordered = /^\s*\d+\.\s/.test(lines[start]);
+  const items = [];
+  let i = start;
+  let gap = false; // a blank line: the next text inside this item is a new paragraph
+  while (i < lines.length) {
+    const l = lines[i];
+    if (!l.trim()) {
+      // A blank line inside a list continues it only if an indented line follows.
+      const next = lines[i + 1];
+      if (next && indentOf(next) > base && next.trim()) { gap = true; i += 1; continue; }
+      break;
+    }
+    const ind = indentOf(l);
+    const bullet = l.match(/^\s*(?:[-*]|\d+\.)\s+(.*)$/);
+    if (ind === base && bullet) { items.push({ parts: [bullet[1]] }); gap = false; i += 1; continue; }
+    if (ind < base || (!bullet && ind === base) || !items.length) break;
+    const parts = items.at(-1).parts;
+    if (bullet && ind > base) {
+      const [html, next] = list(lines, i);
+      parts.push({ html });
+      gap = false;
+      i = next;
+      continue;
+    }
+    if (gap || typeof parts.at(-1) !== 'string') parts.push(l.trim());
+    else parts[parts.length - 1] += ` ${l.trim()}`;
+    gap = false;
+    i += 1;
+  }
+  const tag = ordered ? 'ol' : 'ul';
+  const li = (it) => it.parts.map((p, n) => (typeof p !== 'string' ? p.html : n === 0 ? inline(p) : `<p>${inline(p)}</p>`)).join('');
+  const html = `<${tag}>\n${items.map((it) => `<li>${li(it)}</li>`).join('\n')}\n</${tag}>`;
+  return [html, i];
+}
+
+/**
+ * Markdown → HTML. `headingOffset` shifts heading levels (1 makes # an <h2>),
+ * `ids` adds an id to each heading.
+ */
+export function markdown(md, { headingOffset = 0, ids = true } = {}) {
+  const lines = md.replace(/\r\n/g, '\n').split('\n');
+  const out = [];
+  let i = 0;
+  while (i < lines.length) {
+    const l = lines[i];
+    if (!l.trim()) { i += 1; continue; }
+    const fence = l.match(/^```(\w*)/);
+    if (fence) {
+      const code = [];
+      i += 1;
+      while (i < lines.length && !/^```/.test(lines[i])) code.push(lines[i++]);
+      i += 1;
+      out.push(`<pre><code${fence[1] ? ` class="lang-${fence[1]}"` : ''}>${escapeHtml(code.join('\n'))}</code></pre>`);
+      continue;
+    }
+    const h = l.match(/^(#{1,6})\s+(.*)$/);
+    if (h) {
+      const level = Math.min(6, h[1].length + headingOffset);
+      const html = inline(h[2]);
+      out.push(`<h${level}${ids ? ` id="${slugify(h[2])}"` : ''}>${html}</h${level}>`);
+      i += 1;
+      continue;
+    }
+    if (/^(-{3,}|\*{3,})\s*$/.test(l)) { out.push('<hr>'); i += 1; continue; }
+    if (/^\s*(?:[-*]|\d+\.)\s+/.test(l)) {
+      const [html, next] = list(lines, i);
+      out.push(html);
+      i = next;
+      continue;
+    }
+    if (/^>\s?/.test(l)) {
+      const q = [];
+      while (i < lines.length && /^>\s?/.test(lines[i])) q.push(lines[i++].replace(/^>\s?/, ''));
+      out.push(`<blockquote>${markdown(q.join('\n'), { headingOffset, ids: false })}</blockquote>`);
+      continue;
+    }
+    if (/^\|.*\|\s*$/.test(l) && /^\|?\s*:?-{2,}/.test(lines[i + 1] || '')) {
+      const t = [];
+      while (i < lines.length && /^\|.*\|\s*$/.test(lines[i])) t.push(lines[i++]);
+      out.push(table(t));
+      continue;
+    }
+    const img = l.match(/^!\[([^\]]*)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)\s*$/);
+    if (img) {
+      out.push(`<figure><img src="${escapeHtml(safeUrl(img[2]))}" alt="${escapeHtml(img[1])}" loading="lazy">${img[3] ? `<figcaption>${inline(img[3])}</figcaption>` : ''}</figure>`);
+      i += 1;
+      continue;
+    }
+    const para = [];
+    while (i < lines.length && lines[i].trim() && !/^(#{1,6}\s|```|>\s?|\s*(?:[-*]|\d+\.)\s+|\|)/.test(lines[i])) para.push(lines[i++].trim());
+    if (!para.length) { para.push(lines[i++].trim()); }
+    out.push(`<p>${inline(para.join(' '))}</p>`);
+  }
+  return out.join('\n');
+}
+
+/** Plain words from Markdown, for descriptions and feeds. */
+export function plainText(md) {
+  return md
+    .replace(/!\[[^\]]*\]\([^)]+\)/g, '')
+    .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+    .replace(/\*\*(.+?)\*\*/g, '$1')
+    .replace(/(^|[^\w*])\*(?!\s)(.+?)\*(?!\w)/g, '$1$2')
+    .replace(/`([^`]+)`/g, '$1')
+    .replace(/\s+/g, ' ')
+    .trim();
+}