six-degrees 0.2.0

package/scripts/audit-avatars.mjs

@@ -0,0 +1,132 @@
+#!/usr/bin/env node
+// Find people wearing somebody else's face.
+//
+// Two things had to be true before this worked:
+//
+//  1. The bridge scraper matched people by the TEXT of their link, and every
+//     out-of-network person shows as "LinkedIn Member" — so they all inherited
+//     the first one's profile and photograph.
+//  2. Avatars are saved as sha1(profile_url).webp — named after the PERSON, not
+//     the picture. One photo handed to fifty people therefore becomes fifty
+//     separate files with identical bytes, and LinkedIn signs every image URL
+//     with a unique token, so comparing URLs finds nothing at all.
+//
+// So compare the pictures. Group by the hash of the file contents.
+//
+// Read-only by default. --fix clears the database references so those people
+// fall back to their initials; --prune also deletes the duplicate image files.
+// Nothing is re-scraped and no person is deleted. A wrong face is worse than
+// no face, and this needs no LinkedIn access — which is the point when the
+// account is rate-limited.
+
+import { DatabaseSync } from 'node:sqlite';
+import { createHash } from 'node:crypto';
+import { homedir } from 'node:os';
+import { existsSync, readFileSync, unlinkSync } from 'node:fs';
+import path from 'node:path';
+
+const apply = process.argv.includes('--fix') || process.argv.includes('--prune');
+const prune = process.argv.includes('--prune');
+
+const dir = process.env.SIX_DEGREES_HOME || path.join(homedir(), '.six-degrees');
+const file = process.env.SIX_DEGREES_DB || path.join(dir, 'six-degrees.sqlite');
+const avatarDir = path.join(dir, 'avatars');
+
+if (!existsSync(file)) {
+  console.error(`No database at ${file}`);
+  process.exit(1);
+}
+
+const db = new DatabaseSync(file);
+const rows = db.prepare(`
+  SELECT id, name, degree, profile_image_url AS img
+  FROM linkedin_connections
+  WHERE profile_image_url IS NOT NULL AND profile_image_url <> ''
+`).all();
+
+// Group by what the picture actually is, not by what it is called.
+const byContent = new Map();
+const unreadable = [];
+for (const r of rows) {
+  let key;
+  if (r.img.startsWith('/avatars/')) {
+    const f = path.join(avatarDir, path.basename(r.img));
+    if (!existsSync(f)) { unreadable.push(r); continue; }
+    try {
+      key = `sha:${createHash('sha256').update(readFileSync(f)).digest('hex')}`;
+    } catch { unreadable.push(r); continue; }
+  } else {
+    // Not captured locally yet. Signed URLs differ per fetch, so this only
+    // catches the exact-duplicate case — the local files are the real signal.
+    key = `url:${r.img}`;
+  }
+  if (!byContent.has(key)) byContent.set(key, []);
+  byContent.get(key).push(r);
+}
+
+const dupes = [...byContent.entries()]
+  .filter(([, people]) => people.length > 1)
+  .sort((a, b) => b[1].length - a[1].length);
+
+const affected = dupes.reduce((s, [, p]) => s + p.length, 0);
+
+console.log(`\n  Database:  ${file}`);
+console.log(`  Avatars:   ${avatarDir}`);
+const local = rows.filter((r) => r.img.startsWith('/avatars/')).length;
+const remote = rows.length - local;
+const byDegree = {};
+for (const r of rows) byDegree[r.degree] = (byDegree[r.degree] || 0) + 1;
+console.log(`  People with a photo: ${rows.length}` +
+  `  (${local} captured locally, ${remote} still remote)`);
+console.log(`  By degree: ` + Object.entries(byDegree).map(([d, n]) => `d${d}=${n}`).join('  '));
+if (remote > 0) {
+  // Remote LinkedIn URLs carry a per-request signature, so two copies of the
+  // same picture never look alike. Only captured files can be compared.
+  console.log(`  Note: the ${remote} remote ones cannot be compared by content —`);
+  console.log('        only exact URL matches show up for those.');
+}
+if (unreadable.length) console.log(`  Photos missing from disk: ${unreadable.length}`);
+console.log(`  Distinct pictures shared by more than one person: ${dupes.length}`);
+console.log(`  People wearing a shared picture: ${affected}\n`);
+
+if (dupes.length === 0) {
+  console.log('  Every picture belongs to exactly one person.\n');
+  process.exit(0);
+}
+
+for (const [, people] of dupes.slice(0, 12)) {
+  const names = people.slice(0, 4).map((p) => `${p.name} (d${p.degree})`).join(', ');
+  console.log(`  ${String(people.length).padStart(4)} people share one picture`);
+  console.log(`       ${names}${people.length > 4 ? ` … +${people.length - 4} more` : ''}`);
+}
+if (dupes.length > 12) console.log(`\n  …and ${dupes.length - 12} more shared pictures`);
+
+if (!apply) {
+  console.log(`\n  Read-only. --fix clears these ${affected} photos so those people show`);
+  console.log('  initials. --prune also deletes the duplicate image files.');
+  console.log('  No person is deleted and nothing is re-scraped.\n');
+  process.exit(0);
+}
+
+// We cannot tell which of the group the picture truly belongs to, so none of
+// them keeps it. Guessing would leave one person confidently mislabelled.
+const clear = db.prepare('UPDATE linkedin_connections SET profile_image_url = NULL WHERE id = ?');
+let cleared = 0;
+let deleted = 0;
+const seenFiles = new Set();
+for (const [, people] of dupes) {
+  for (const p of people) {
+    cleared += clear.run(p.id).changes;
+    if (prune && p.img.startsWith('/avatars/')) {
+      const f = path.join(avatarDir, path.basename(p.img));
+      if (!seenFiles.has(f) && existsSync(f)) {
+        try { unlinkSync(f); deleted += 1; } catch { /* leave it */ }
+        seenFiles.add(f);
+      }
+    }
+  }
+}
+
+console.log(`\n  Cleared ${cleared} photos — those people now show initials.`);
+if (prune) console.log(`  Deleted ${deleted} duplicate image files.`);
+console.log('  Re-run without flags to confirm.\n');

package/scripts/check-node.mjs

@@ -0,0 +1,28 @@
+// Runs before `npm run dev` and `npm run build`.
+//
+// The database is Node's built-in node:sqlite, which arrived in 22.13. On an
+// older Node the app installs and starts perfectly and then throws
+// "Cannot find module 'node:sqlite'" on the first query — a runtime error that
+// reads like a broken app rather than a missing requirement. `engines` in
+// package.json is only a warning by default, so it does not catch this either.
+
+const NEED = [22, 13];
+const [major, minor] = process.versions.node.split('.').map(Number);
+
+if (major < NEED[0] || (major === NEED[0] && minor < NEED[1])) {
+  const need = NEED.join('.');
+  console.error(`
+  ┌──────────────────────────────────────────────────────────────┐
+  │  6 Degrees needs Node ${need} or newer.                          │
+  └──────────────────────────────────────────────────────────────┘
+
+  You have Node ${process.versions.node}.
+
+  The database uses node:sqlite, which is built into Node from ${need}.
+  On older versions the app starts and then fails on its first query.
+
+  Install the current LTS from https://nodejs.org and run this again.
+  With nvm:  nvm install --lts && nvm use --lts
+`);
+  process.exit(1);
+}

package/scripts/check-stale-server.mjs

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+// Runs before `npm run dev`.
+//
+// A dev server started before a `git pull` keeps serving what it had at launch,
+// and when you start a second one Next quietly takes the next free port. Your
+// browser is still pointed at the first, so the app looks like the pull never
+// happened — the exact confusion that cost an evening: correct commit, correct
+// files, old screen.
+//
+// This does not kill anything. It tells you what is already listening and gives
+// you the line to run.
+
+import { execFileSync } from 'node:child_process';
+
+const PORTS = [3000, 3001, 3002, 3210];
+
+function listeners() {
+  try {
+    const out = execFileSync('lsof', ['-nP', '-iTCP', '-sTCP:LISTEN'], {
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).toString();
+    return out
+      .split('\n')
+      .filter((l) => /^node\b/.test(l))
+      .map((l) => {
+        const pid = l.split(/\s+/)[1];
+        const port = (l.match(/:(\d+)\s+\(LISTEN\)/) || [])[1];
+        return port ? { pid, port: Number(port) } : null;
+      })
+      .filter(Boolean)
+      .filter((p) => PORTS.includes(p.port));
+  } catch {
+    return [];   // no lsof, or not permitted — never block the dev server
+  }
+}
+
+const running = listeners();
+if (running.length > 0) {
+  const list = running.map((r) => `port ${r.port} (pid ${r.pid})`).join(', ');
+  console.log(`
+  ┌──────────────────────────────────────────────────────────────┐
+  │  Something is already serving on a port this app uses.       │
+  └──────────────────────────────────────────────────────────────┘
+
+  Already listening: ${list}
+
+  If that is an older copy of this app, it is still serving the code it had
+  when it started — including from before your last git pull. Starting another
+  one now takes the NEXT free port, so your browser keeps showing the old one.
+
+  To replace it:
+
+      pkill -f "next dev"; pkill -f "next-server"; npm run dev
+
+  Continuing anyway — watch the port this prints, it may not be ${running[0].port}.
+`);
+}

package/scripts/image_store.py

@@ -0,0 +1,125 @@
+"""Permanent avatar capture.
+
+LinkedIn CDN image URLs (media.licdn.com) are signed and expire in ~3 weeks, so
+storing the raw URL means every avatar breaks on that cycle. Instead, while the
+signed URL is still fresh (right after scraping), we download the bytes, crop to a
+small square, and re-encode as a tiny WebP saved under the user's data directory
+(~/.six-degrees/avatars, or $SIX_DEGREES_HOME/avatars). The record stores the local
+path /avatars/<slug>.webp, which never expires and is served by app/avatars/[file].
+
+Avatars live outside the repo on purpose: they are photographs of real people and
+must never be committed, and an installed copy of the app must not write into its
+own package directory.
+
+Typical output: 96x96 WebP ~2-4 KB per person (vs ~15-40 KB for the original).
+"""
+
+import io
+import hashlib
+import pathlib
+
+import requests
+from PIL import Image, ImageOps
+
+import os
+
+def _data_dir():
+    return pathlib.Path(os.environ.get("SIX_DEGREES_HOME") or (pathlib.Path.home() / ".six-degrees"))
+
+AVATAR_DIR = _data_dir() / "avatars"
+
+SIZE = 96        # nodes render small; 96px stays crisp on retina
+QUALITY = 72     # WebP quality — good balance for face icons
+TIMEOUT = 15
+
+
+def _slug(key):
+    """Stable filename derived from a person's profile URL (or any stable key)."""
+    return hashlib.sha1(key.strip().encode("utf-8")).hexdigest()[:16]
+
+
+# Pictures already claimed by somebody, keyed by the hash of the downloaded
+# bytes. Filled as a run proceeds and consulted before anything is written.
+_claimed = {}
+
+
+def reset_claims():
+    """Start a fresh run. Only useful for tests and long-lived processes."""
+    _claimed.clear()
+
+
+def store_avatar(image_url, key, overwrite=True):
+    """Download image_url, compress to a small square WebP, save under the data dir.
+
+    Returns the local web path ("/avatars/<slug>.webp") or None if the fetch/decode
+    fails (e.g. the signed URL already expired -> 403), or if this exact picture
+    already belongs to somebody else.
+
+    That last case matters more than it sounds. Files are named after the PERSON —
+    sha1(profile_url) — so one photograph handed to fifty people used to become fifty
+    separate files holding identical bytes, and nothing downstream could tell. On one
+    real database that was 2,821 of 3,486 photographs: mostly LinkedIn's placeholder
+    silhouette for people who have no picture, plus genuine mis-attributions from a
+    scraper that matched people by the text of their link. Both look the same from
+    here, and both are wrong to save. Initials are honest; somebody else's face is not.
+    """
+    if not image_url or not key:
+        return None
+    slug = _slug(key)
+    out = AVATAR_DIR / f"{slug}.webp"
+    if out.exists() and not overwrite:
+        return "/avatars/%s.webp" % slug
+    try:
+        resp = requests.get(image_url, timeout=TIMEOUT)
+        if resp.status_code != 200 or not resp.content:
+            return None
+
+        # Hash what actually arrived, before any re-encoding, so two fetches of
+        # the same picture under different signed URLs still collide.
+        digest = hashlib.sha256(resp.content).hexdigest()
+        owner = _claimed.get(digest)
+        if owner is not None and owner != key:
+            return None
+        _claimed[digest] = key
+
+        img = Image.open(io.BytesIO(resp.content))
+        img = ImageOps.exif_transpose(img).convert("RGB")
+        img = ImageOps.fit(img, (SIZE, SIZE), Image.LANCZOS)  # center-crop to square
+        AVATAR_DIR.mkdir(parents=True, exist_ok=True)
+        img.save(out, "WEBP", quality=QUALITY, method=6)
+        return "/avatars/%s.webp" % slug
+    except Exception as e:  # network hiccup, non-image body, decode error
+        print("  [image_store] skip %s: %s" % (key[:40], e))
+        return None
+
+
+def localize_images(images):
+    """Turn scraper image records that point at expiring licdn URLs into permanent
+    local ones. Input/output: list of {"profileUrl", "imageUrl"}. Entries whose
+    download or compression fails are dropped (kept as no-image rather than a dead URL)."""
+    out = []
+    ok = 0
+    for rec in images:
+        purl = rec.get("profileUrl")
+        iurl = rec.get("imageUrl")
+        local = store_avatar(iurl, purl) if (purl and iurl) else None
+        if local:
+            out.append({"profileUrl": purl, "imageUrl": local})
+            ok += 1
+    shared = len(images) - ok
+    print("  [image_store] captured %d/%d avatars -> %s" % (ok, len(images), AVATAR_DIR))
+    if shared > 0:
+        print("  [image_store] %d skipped: no picture, or the same picture as someone else" % shared)
+    return out
+
+
+if __name__ == "__main__":
+    # Quick self-test: python image_store.py <image_url> <key>
+    import sys
+    if len(sys.argv) >= 3:
+        path = store_avatar(sys.argv[1], sys.argv[2])
+        if path:
+            f = REPO / "public" / path.lstrip("/")
+            print("saved %s (%d bytes)" % (path, f.stat().st_size))
+        else:
+            print("failed (URL likely expired or not an image)")

package/scripts/requirements.txt

@@ -0,0 +1,4 @@
+playwright>=1.40.0
+requests>=2.31.0
+Pillow>=10.0.0
+tzdata>=2024.1; sys_platform == "win32"

package/scripts/score_new_connections.sql

@@ -0,0 +1,77 @@
+-- RETIRED: the hosted-era scoring model, kept for history. The app scores with
+-- lib/scoring.js (see its header for the model and why it changed).
+-- CANONICAL SCORER — the scoring model this project's tiers are derived from.
+--
+-- This is the reference implementation: any other scorer in the codebase
+-- (lib/csv.js for CSV imports, the inline scorer in app/api/ingest/route.js)
+-- should be diffed against it. Written for Postgres; the local build ports the
+-- same CASE ladders to SQLite.
+--
+-- Note it differs from the inline JS scorer in two ways the JS omits entirely:
+-- the headline influence bonus (+2 / +1.5) and the 30-day recency bonus (+0.5).
+--
+-- Scores only rows WHERE tier IS NULL, so it is safe to re-run.
+
+CREATE OR REPLACE FUNCTION public.score_new_connections()
+ RETURNS void
+ LANGUAGE plpgsql
+AS $function$
+BEGIN
+  -- Seniority score
+  UPDATE linkedin_connections SET seniority_score = CASE
+    WHEN role ~* '(CEO|Chief|Founder|President|Owner)' THEN 10
+    WHEN role ~* '(VP|Vice President|SVP|EVP|Managing Director|Global Head|Country Director)' THEN 9
+    WHEN role ~* '(Senior Director|Director|Head of|Country Lead)' THEN 8
+    WHEN role ~* '(Senior Manager|Manager,|Engineering Manager|Group Product)' THEN 7
+    WHEN role ~* '(Lead|Principal|Staff|Senior.*Engineer|Senior.*Designer|Senior.*Manager)' THEN 6
+    WHEN role ~* '(Partner|Client Partner|Account Manager|Strategist)' THEN 5
+    WHEN role ~* '(Engineer|Developer|Designer|Analyst|Coordinator)' THEN 4
+    WHEN role ~* '(Intern|Student|Undergraduate|Junior|Entry)' THEN 2
+    WHEN headline ~* '(Student|UCF|University)' AND role !~* '(Manager|Lead|Director|VP|CEO)' THEN 1
+    ELSE 3
+  END
+  WHERE tier IS NULL;
+
+  -- Company prestige score
+  UPDATE linkedin_connections SET company_prestige_score = CASE
+    WHEN company ~* '(^Google|^Meta |^Apple$|^Microsoft|^Amazon|^NVIDIA|^Tesla$)' THEN 10
+    WHEN company ~* '(Snap|Snapchat)' THEN 9
+    WHEN company ~* '(Goldman Sachs|BlackRock|Merrill Lynch|McKinsey|BCG|^Deloitte|Heidrick|Boies Schiller)' THEN 9
+    WHEN company ~* '(Polymarket|Anduril|CoreWeave|^Stripe$|Palantir)' THEN 9
+    WHEN company ~* '(Coca-Cola|The Coca-Cola)' THEN 9
+    WHEN company ~* '(Pinterest|TikTok|Roku|Netflix|Uber|Spotify|Databricks|Datadog|OpenAI|Anthropic)' THEN 8
+    WHEN company ~* '(SpaceX|Lockheed Martin|Northrop|Siemens|Boeing|Raytheon|Disney)' THEN 8
+    WHEN company ~* '(Nike|LVMH|Unilever|Danone|Beiersdorf|Starbucks|PepsiCo|Pepsico)' THEN 8
+    WHEN company ~* '(Whatnot|BNY|Bank of New York|ServiceNow|Salesforce|Visa|Oracle|IBM)' THEN 8
+    WHEN company ~* '(World Bank|Goodyear|Kellanova|Kellogg|Campbell)' THEN 8
+    WHEN company ~* '(U\.S\. Space Force|Space Force)' THEN 8
+    WHEN company ~* '(Mercury|Credit Karma|Capital One|PayPal|Plaid|Coinbase|Scale AI|Chick-fil-A)' THEN 7
+    WHEN company ~* '(WPP|Intel|AMD|Figma|Notion|Vercel)' THEN 7
+    WHEN company ~* '(Hard Rock Digital|Beam|Later|Vanta|Kaseya)' THEN 6
+    WHEN company = '' OR company IS NULL THEN 2
+    ELSE 4
+  END
+  WHERE tier IS NULL;
+
+  -- Power score
+  UPDATE linkedin_connections SET power_score =
+    (seniority_score * 0.5) + (company_prestige_score * 0.3) +
+    CASE
+      WHEN headline ~* '(billion|million|M\+|B\+|Forbes|YC|a16z|venture|investor|Wharton|MIT|Stanford)' THEN 2
+      WHEN headline ~* '(award|patent|speaker|author|TEDx|Board Member|Board Director)' THEN 1.5
+      ELSE 0
+    END +
+    CASE WHEN degree = 1 AND connected_date >= CURRENT_DATE - INTERVAL '30 days' THEN 0.5 ELSE 0 END
+  WHERE tier IS NULL;
+
+  -- Tier assignment
+  UPDATE linkedin_connections SET tier = CASE
+    WHEN power_score >= 7 THEN 'S'
+    WHEN power_score >= 5.5 THEN 'A'
+    WHEN power_score >= 4 THEN 'B'
+    WHEN power_score >= 2.5 THEN 'C'
+    ELSE 'D'
+  END
+  WHERE tier IS NULL;
+END;
+$function$;