six-degrees 0.2.1 → 0.3.0

package/lib/linkedin-limits.js

@@ -6,8 +6,9 @@
 //
 // They belong to the LinkedIn account, not to a profile in the app. TRAPS §35.
 
-import { readFileSync, writeFileSync, renameSync, unlinkSync, existsSync } from 'node:fs';
+import { readFileSync, writeFileSync, renameSync, unlinkSync, existsSync, lstatSync } from 'node:fs';
 import path from 'node:path';
+import { durable } from './durable.js';
 
 export const DEFAULT_LIMITS = { daily: 50, monthly: 250 };
 export const DAILY_CHOICES = [25, 50, 100, 200, 500];
@@ -116,3 +117,125 @@ export function linkedinState(dir, now = Date.now()) {
     cooldown: readCooldown(dir, now),
   };
 }
+
+// ── another computer's budget (Settings → Your data → Import) ───────────────
+// An import carries these three files from the other computer. They describe
+// the same LinkedIn account, so they are merged with the ones here, never put
+// in their place: replacing them would forget the searches made here (and lift
+// a cooldown nobody chose to lift), and the account would be asked for more
+// than its budget (TRAPS §35).
+
+const ACTIVITY = 'linkedin-activity.json';
+const COOLDOWN = 'linkedin-cooldown.json';
+const LIMITS = 'scan-limits.json';
+
+const isObject = (v) => Boolean(v) && typeof v === 'object' && !Array.isArray(v);
+const isTimeList = (v) => v === undefined
+  || (Array.isArray(v) && v.every((t) => typeof t === 'number' && Number.isFinite(t) && t >= 0));
+const isTime = (v) => typeof v === 'number' && Number.isFinite(v) && v >= 0;
+
+/**
+ * Why one of these files from another computer can't be used here, in words
+ * for the page, or null when it can. The limits follow writeLimits: only the
+ * menu's own choices. A value the app never offers is refused rather than
+ * passed to the scanner, which reads a daily limit of 0 as "no limit at all".
+ */
+export function budgetFileProblem(name, data) {
+  if (!isObject(data)) return 'it is not a JSON object';
+  if (name === ACTIVITY) {
+    if (!isTimeList(data.searches) || !isTimeList(data.profiles)) return 'its record of searches is not a list of times';
+  } else if (name === COOLDOWN) {
+    if (!isTime(data.until)) return 'its pause has no end time';
+    if (data.reason !== undefined && typeof data.reason !== 'string') return 'its pause has a reason that is not text';
+    if (data.set_at !== undefined && !isTime(data.set_at)) return 'its pause has a start time that is not a time';
+  } else if (name === LIMITS) {
+    if (data.daily !== undefined && !DAILY_CHOICES.includes(data.daily)) {
+      return `its daily limit (${String(data.daily).slice(0, 20)}) is not one Six Degrees offers`;
+    }
+    if (data.monthly !== undefined && !MONTHLY_CHOICES.includes(data.monthly)) {
+      return `its monthly limit (${String(data.monthly).slice(0, 20)}) is not one Six Degrees offers`;
+    }
+  }
+  return null;
+}
+
+/**
+ * Both lists of times, each time kept as many times as the list that has it
+ * most often. Not a plain union, and not a de-duplication: the scanner writes
+ * one time n times when a single click cost n searches (charge_linkedin), and
+ * those are n searches; and a history both computers already share (one was
+ * imported from the other before) must still count once, not twice.
+ */
+export function mergeTimes(mine = [], theirs = []) {
+  const counts = (list) => list.reduce((m, t) => m.set(t, (m.get(t) || 0) + 1), new Map());
+  const merged = counts(mine);
+  for (const [t, n] of counts(theirs)) if (n > (merged.get(t) || 0)) merged.set(t, n);
+  return [...merged].sort((a, b) => a[0] - b[0]).flatMap(([t, n]) => Array(n).fill(t));
+}
+
+/** The scanner's own reading of a record: every entry a number (float() in scrape.py). */
+function readableTimes(v) {
+  if (v === undefined) return [];
+  if (!Array.isArray(v)) return null;
+  const out = v.map(Number);
+  return out.every(Number.isFinite) ? out : null;
+}
+
+const hasFile = (p) => {
+  try { lstatSync(p); return true; } catch { return false; }
+};
+
+/**
+ * Merge the budget files in `from` (an import's, already checked with
+ * budgetFileProblem) into the data folder `dir`:
+ *   linkedin-activity.json  both computers' searches and profile views (mergeTimes)
+ *   linkedin-cooldown.json  whichever pause ends later
+ *   scan-limits.json        this computer's, if it has one; the copy's otherwise.
+ *                           The limits are a choice someone made here, and an
+ *                           import never quietly changes it.
+ * A record here that can't be read is left as it is: the scanner reads it as
+ * "today's searches are used up" (scrape.py _read_activity), which is the safe
+ * way round. Running it twice gives the same files as running it once, so a
+ * start that stops part-way can simply do it again.
+ */
+export function mergeBudgetFiles({ dir, from }) {
+  const here = (name) => path.join(dir, name);
+  const theirs = (name) => {
+    const data = readJson(path.join(from, name));
+    return isObject(data) && !budgetFileProblem(name, data) ? data : null;
+  };
+
+  const activity = theirs(ACTIVITY);
+  if (activity) {
+    const mine = hasFile(here(ACTIVITY)) ? readJson(here(ACTIVITY)) : {};
+    const searches = isObject(mine) ? readableTimes(mine.searches) : null;
+    const profiles = isObject(mine) ? readableTimes(mine.profiles) : null;
+    if (searches && profiles) {
+      durable.writeJsonDurably(here(ACTIVITY), {
+        searches: mergeTimes(searches, activity.searches),
+        profiles: mergeTimes(profiles, activity.profiles),
+      });
+    }
+  }
+
+  const cooldown = theirs(COOLDOWN);
+  if (cooldown) {
+    const mine = readJson(here(COOLDOWN));
+    const mineEnds = Number.isFinite(Number(mine?.until)) ? Number(mine.until) : -Infinity;
+    if (cooldown.until > mineEnds) {
+      durable.writeJsonDurably(here(COOLDOWN), {
+        until: cooldown.until,
+        reason: cooldown.reason ?? 'LinkedIn pushed back',
+        set_at: cooldown.set_at ?? null,
+      });
+    }
+  }
+
+  const limits = theirs(LIMITS);
+  if (limits && !hasFile(here(LIMITS))) {
+    durable.writeJsonDurably(here(LIMITS), {
+      daily: limits.daily ?? DEFAULT_LIMITS.daily,
+      monthly: limits.monthly ?? DEFAULT_LIMITS.monthly,
+    });
+  }
+}

package/lib/rpc.js

@@ -1,52 +1,160 @@
-import { getDb, nowIso } from './db-client.js';
+import { createHash } from 'node:crypto';
+import { getDb, newId, nowIso } from './db-client.js';
 
 // Replacements for the stored procedures the hosted database used to provide.
 // Scoring lives in lib/scoring.js; this file only reads rows and writes the
-// results back. (scripts/*.sql are the retired hosted-era model.)
+// results back. rpc('score_new_connections') is that procedure's name, kept
+// for the routes that call it; it rescores with lib/scoring.js like the rest.
 
-import { scoreNetwork, explainScore, SCORING_VERSION } from './scoring.js';
-import { industryOf } from './companies.js';
+import { readNetwork, scoreNetwork, explainScore, SCORING_VERSION, KNOWN_COMPANIES, RULE_TABLES } from './scoring.js';
+import { industryKeyOf, INDUSTRIES } from './companies.js';
+import { readSettings } from './settings.js';
+import { SECTOR_FOCUS_SETTING, focusFingerprint, previewSectorFocus } from './sector-focus.js';
+import { LEGACY_OFFER_KEY } from './legacy-scores.js';
+import { sectorMatcher, sectorGroup, sectorLabel } from './sector-directory.js';
+
+// The word lists and rules the stored scores were computed with: a fingerprint
+// of every entry on the curated company list (name, score, alias, industry),
+// of the broad industries' words and labels (lib/companies.js INDUSTRIES), and
+// of every rule table scoring reads headlines and names with (lib/scoring.js
+// RULE_TABLES: titles, students and clubs, what isn't a company, the reach
+// bonuses), stamped beside them in app_meta 'scoring_list'. The industries'
+// words decide a company's one industry, which decides whether it is a school
+// (no estimate) and whether a broad pick in Settings leans it; their labels
+// are in the stored working ("4 + 1 your sector: Healthcare & Biotech").
+// Editing any of them makes stored scores stale, as a new SCORING_VERSION
+// does, with no number to remember to bump. Copies from before the list was
+// made neutral stamped none, which is also how rescoreAll() knows to offer
+// keeping their old scores (lib/legacy-offer.js).
+export function scoringStamp({ known = KNOWN_COMPANIES, industries = INDUSTRIES, rules = RULE_TABLES } = {}) {
+  // JSON has no patterns or sets: a pattern counts by its source and flags, a set by its members.
+  const plain = (key, value) => (value instanceof RegExp ? String(value) : value instanceof Set ? [...value] : value);
+  return createHash('sha256')
+    .update(JSON.stringify([
+      known.map(([name, score, alias, industry]) => [name, score, String(alias), industry]),
+      industries.map(({ key, label, words }) => [key, label, String(words)]),
+      rules,
+    ], plain))
+    .digest('hex').slice(0, 16);
+}
+
+export const KNOWN_LIST_STAMP = scoringStamp();
 
 export function companyOverrides(db = getDb()) {
   return new Map(db.prepare('SELECT name, score FROM company_scores').all().map((r) => [r.name, r.score]));
 }
 
+/**
+ * Set company scores as your own ([[name, score], …]), the way Paths → Scores
+ * does: one row per name, replacing any you had. Rescoring is the caller's
+ * job, so a batch rescores once.
+ */
+export function setCompanyScores(db, entries) {
+  const upsert = db.prepare(`INSERT INTO company_scores (id, name, score, updated_at) VALUES (?, ?, ?, ?)
+    ON CONFLICT(name) DO UPDATE SET score = excluded.score, updated_at = excluded.updated_at`);
+  const now = nowIso();
+  for (const [name, score] of entries) upsert.run(newId(), name, score, now);
+}
+
+/** The sector focus saved in Settings (lib/sector-focus.js), or none. */
+export function sectorFocusOf(db = getDb()) {
+  return readSettings(db, { sectorFocus: SECTOR_FOCUS_SETTING }).sectorFocus;
+}
+
+/**
+ * What rescoring reads: every row, as scoring needs it. `withPeople` adds each
+ * row's name, for naming who a change would move.
+ */
+export function scoringRows(db = getDb(), { withPeople = false } = {}) {
+  return db.prepare(`SELECT id, degree, headline, role, company, scanned_company, profile_url, source_connection_id${withPeople ? ', name' : ''}
+    FROM linkedin_connections`).all();
+}
+
+/**
+ * Every headline read once, the way scoring reads it: each company's
+ * headcount, its one industry, the sector directory's sectors it matches and
+ * the industries those sit under. Rescoring, Settings' preview and
+ * suggestions, and Paths → Scores all read the rows through this, so they
+ * agree on which company is in which sector.
+ */
+export function readForScoring(rows) {
+  return readNetwork(rows, { industryOf: industryKeyOf, sectorsOf: sectorMatcher(), groupOf: sectorGroup });
+}
+
 /**
  * Rescore everyone. The score depends on the whole network (how many of your
  * people work at a company, how strong a bridge's circle is), so after any
  * ingest or company-score change every row is recomputed — ~0.1s for 4,000.
+ * Your company scores and your sector focus are read here, not passed in, so
+ * every path that rescores (an import, a company score, Settings, a stale
+ * model) applies both.
+ *
+ * `compareWith` is another sector focus: Settings' save passes the one it
+ * replaced. The same rows are then also scored with it, in memory, and the
+ * answer says who changed tier between the two, counted by the preview's own
+ * function (lib/sector-focus.js), so a save reports what its preview said.
+ * That compares two fresh scorings rather than the stored tiers, which can be
+ * stale (a save whose rescore failed, a database from another computer).
+ * @returns { scored } rows written, plus { people, up, down } when comparing.
  */
-export function rescoreAll() {
+export function rescoreAll({ compareWith } = {}) {
   const db = getDb();
-  const rows = db.prepare('SELECT id, degree, headline, role, company, scanned_company, profile_url, source_connection_id FROM linkedin_connections').all();
-  const { scores } = scoreNetwork(rows, { overrides: companyOverrides(db), industryOf: (c, h) => industryOf(c, h).key });
+  const rows = scoringRows(db);
+  const overrides = companyOverrides(db);
+  const focus = sectorFocusOf(db);
+  // Every headline is read once, however many ways the rows are then scored.
+  const read = readForScoring(rows);
+  const { scores } = scoreNetwork(rows, { overrides, focus, read });
+  // Worked out before anything is written, so a failure leaves nothing half done.
+  const change = compareWith === undefined ? null
+    : previewSectorFocus(rows, { overrides, read, from: compareWith, to: focus, examples: 0 });
   const update = db.prepare(`UPDATE linkedin_connections SET seniority_score = ?, company_prestige_score = ?, power_score = ?, tier = ?,
     score_why = ?, circle_power = ?, circle_s_count = ?, circle_a_count = ?, circle_elite_pct = ?, is_catalyst = ?, catalyst_score = ?, updated_at = ?
     WHERE id = ?`);
   const now = nowIso();
   db.exec('BEGIN');
   try {
+    // Stored scores with no list stamp came from a copy whose curated list
+    // held one person's picks. The first time they are replaced, Paths →
+    // Scores offers once to keep the old ones as your own (lib/legacy-offer.js).
+    // A database nothing has scored yet never had them, so it is never offered.
+    const listed = db.prepare("SELECT 1 FROM app_meta WHERE key = 'scoring_list'").get();
+    if (!listed && db.prepare('SELECT 1 FROM linkedin_connections WHERE tier IS NOT NULL LIMIT 1').get()) {
+      db.prepare("INSERT OR IGNORE INTO app_meta (key, value) VALUES (?, 'open')").run(LEGACY_OFFER_KEY);
+    }
     for (const r of rows) {
       const s = scores.get(r.id);
       const c = s.circle;
-      update.run(s.title.points, s.companyScore, s.power, s.tier, explainScore(s, s.boost),
+      update.run(s.title.points, s.companyScore, s.power, s.tier, explainScore(s, s.boost, { sectorLabel }),
         c ? Math.round(c.circlePower * 100) / 100 : 0, c?.sCount || 0, c?.aCount || 0, c ? Math.round(c.elitePct * 100) / 100 : 0,
         c?.isCatalyst ? 1 : 0, c?.isCatalyst ? c.sCount + c.aCount : 0, now, r.id);
     }
-    db.prepare('INSERT INTO app_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run('scoring_version', String(SCORING_VERSION));
+    const meta = db.prepare('INSERT INTO app_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');
+    meta.run('scoring_version', String(SCORING_VERSION));
+    // Which curated list and sector focus these scores were computed with (see rescoreIfStale).
+    meta.run('scoring_list', KNOWN_LIST_STAMP);
+    meta.run('scoring_focus', focusFingerprint(focus));
     db.exec('COMMIT');
   } catch (err) {
     db.exec('ROLLBACK');
     throw err;
   }
-  return { scored: rows.length };
+  return change ? { scored: rows.length, people: change.people, up: change.up, down: change.down } : { scored: rows.length };
 }
 
-/** Rescore once if the stored scores came from an older scoring model. */
+/**
+ * Rescore once if the stored scores came from an older scoring model or
+ * curated list, or from a sector focus other than the one saved now: an
+ * update, a database restored or brought from another computer, or a Settings
+ * save whose rescore never finished.
+ */
 export function rescoreIfStale() {
   const db = getDb();
-  const v = db.prepare("SELECT value FROM app_meta WHERE key = 'scoring_version'").get()?.value;
-  return v === String(SCORING_VERSION) ? { scored: 0 } : rescoreAll();
+  const meta = (key) => db.prepare('SELECT value FROM app_meta WHERE key = ?').get(key)?.value;
+  const fresh = meta('scoring_version') === String(SCORING_VERSION)
+    && meta('scoring_list') === KNOWN_LIST_STAMP
+    && meta('scoring_focus') === focusFingerprint(sectorFocusOf(db));
+  return fresh ? { scored: 0 } : rescoreAll();
 }
 
 function incrementXp({ amount = 0 } = {}) {

package/lib/sample-stats.js

@@ -0,0 +1,16 @@
+// The sample network's own numbers, for the launch page (app/launch/page.js),
+// which shows them rather than anyone's real network. Counted from
+// public/demo-data.json, where every person is invented
+// (scripts/gen-synthetic.mjs); tests/sample-stats.test.mjs counts them again,
+// so regenerating the sample can't leave them stale. They're written here
+// rather than read from the file so the page doesn't download the whole sample
+// to show a few numbers.
+export const SAMPLE_STATS = {
+  connections: 873,       // rows: the 1st degree and the circles of the bridges
+  degree1: 150,
+  degree2: 723,
+  people: 748,            // each person once, however many circles they're in
+  bridges: 14,            // 1st-degree people whose circle is mapped
+  companies: 19,          // where people work now, as Paths groups them
+  recommendations: 474,   // the Queue's list: 2nd-degree S, A and B you aren't connected to
+};

package/lib/scan-progress.js

@@ -9,8 +9,13 @@
 //                "  350 / 817 collected"            (every fifty people)
 //                "  Collected all 817 connections."  (the end)
 //   2nd degree:  "[3/10] Some Name (A-tier, score 6.2)"  (as each person starts)
+//
+// And one from the app itself (app/api/scraper/route.js setupPlan), while
+// "Set up the scanner" downloads its Python:
+//                "Downloading Python 3.12.14: 12.0 of 24.0 MB"
 
 const REPORTED = /LinkedIn reports (\d+) connections/;
+const DOWNLOADING = /^Downloading Python [\d.]+: ([\d.]+) of ([\d.]+) MB$/;
 const COLLECTED = /(\d+)\s*\/\s*(\d+)\s+collected/;
 const COLLECTED_ALL = /Collected all (\d+) connections/;
 const BRIDGE_STEP = /\[(\d+)\/(\d+)\]/;
@@ -26,6 +31,14 @@ const SAVING = /^(Collected \d+ connections|Pushing \d+ connections|\s*Sent \d+
 export function scanProgress(log, action) {
   if (!Array.isArray(log) || log.length === 0) return null;
 
+  // Only while the download is the step under way: once the log has moved on
+  // (unpacking, installing), a bar left at 100% would look stuck.
+  if (action === 'setup') {
+    const m = String(log[log.length - 1]).match(DOWNLOADING);
+    if (!m || !Number(m[2])) return null;
+    return { done: Math.min(Number(m[1]), Number(m[2])), total: Number(m[2]), kind: 'download' };
+  }
+
   if (action === 'full' || action === 'refresh') {
     for (let i = log.length - 1; i >= 0; i--) {
       const line = String(log[i]);

package/lib/scan-state.js

@@ -0,0 +1,53 @@
+// Whether the scanner is running, for routes other than the one that runs it.
+//
+// app/api/scraper/route.js owns the job and its state. It registers that state
+// here, on globalThis, so a route that must not act mid-scan (an import that
+// replaces the database, a restart) can ask without an HTTP call to its own
+// server, and gets the same answer however the bundler splits route modules.
+//
+// The scanner's route runs more than scans: Install, Set up the scanner and
+// the LinkedIn sign-in window take the same slot. A refusal says which one is
+// running (runningNow): "A scan is running" while Install ran sent people
+// looking for a scan that wasn't there.
+
+const KEY = Symbol.for('six-degrees.scan-state');
+
+/** The scanner's state object: the one already registered in this server, or `initial`. */
+export function registerScanState(initial) {
+  if (!globalThis[KEY]) globalThis[KEY] = initial;
+  return globalThis[KEY];
+}
+
+/** What the scanner's route is running: its action ('full', 'install', 'setup', 'login', …), or null. */
+export function scannerJob() {
+  const state = globalThis[KEY];
+  if (state?.running !== true) return null;
+  return typeof state.action === 'string' && state.action ? state.action : 'scan';
+}
+
+/** Is a scan, or the scanner's setup, running in this server? */
+export function scanIsRunning() {
+  return scannerJob() !== null;
+}
+
+/**
+ * What is running, in words that start a refusal: "A scan is running", "The
+ * scanner is being set up", … `job` is scannerJob()'s answer; `true` (a caller
+ * that only knows something runs) reads as a scan.
+ */
+export function runningNow(job) {
+  switch (job) {
+    case 'install': return 'The scanner\'s packages are being installed';
+    case 'setup': return 'The scanner is being set up';
+    case 'login': return 'The LinkedIn sign-in window is open';
+    default: return 'A scan is running';
+  }
+}
+
+/**
+ * A refusal while the scanner's route runs something: "<what runs>. <then>",
+ * or null when nothing runs.
+ */
+export function busyRefusal(then, job = scannerJob()) {
+  return job ? `${runningNow(job)}. ${then}` : null;
+}