six-degrees 0.3.0 → 0.4.0

package/.next/standalone/lib/scraper-client.js

@@ -7,6 +7,8 @@
 //
 // Now they all call this, which talks to /api/scraper on the app itself.
 
+import { runningNow } from './scan-state.js';
+
 export async function scraperStatus() {
   const r = await fetch('/api/scraper');
   if (!r.ok) throw new Error('Could not reach the app.');
@@ -25,24 +27,41 @@ export async function startScrape(action, name, extra = {}) {
 }
 
 /**
- * Run one scrape and report progress until it ends.
- * onLog receives the whole log each tick; returns the final status.
+ * Where Resume would carry on with one of your connections, for their profile
+ * card: { nextPage, pagesRead, legacy }, or null when there is nothing left to
+ * read (never mapped, read to the end, hidden).
  */
-export function runScrape(action, { name, onLog, ...extra } = {}) {
-  return new Promise((resolve, reject) => {
-    startScrape(action, name, extra).then(() => {
-      const poll = setInterval(async () => {
-        try {
-          const s = await scraperStatus();
-          onLog?.(s.log || []);
-          if (!s.running) {
-            clearInterval(poll);
-            resolve(s);
-          }
-        } catch { /* transient; keep polling */ }
-      }, 1500);
-    }).catch(reject);
-  });
+export async function resumePoint(id) {
+  const r = await fetch(`/api/scraper?resume=${encodeURIComponent(id)}`);
+  if (!r.ok) return null;
+  return (await r.json()).resume ?? null;
+}
+
+/** Nothing noted: before the first answer, and for the sample or a CSV. */
+export const NO_SCAN_NOTES = Object.freeze({ skips: [], read: [] });
+
+/**
+ * What the scanner noted, for lib/reach.js: `skips`, the hidden lists
+ * ([{ profileUrl, … }]), and `read`, the profile URLs of every list it has
+ * read. Never throws: with no answer (the app restarting, say) nothing is
+ * noted, so a hidden list shows as ready rather than anyone going missing.
+ */
+export async function loadScanNotes() {
+  try {
+    const r = await fetch('/api/scraper?reach=1');
+    if (!r.ok) return NO_SCAN_NOTES;
+    const d = await r.json();
+    return { skips: Array.isArray(d.skips) ? d.skips : [], read: Array.isArray(d.read) ? d.read : [] };
+  } catch {
+    return NO_SCAN_NOTES;
+  }
+}
+
+/** One of your connections for the Scan page's "Scan one circle" box: { person, circle }. */
+export async function pickedPerson(id) {
+  const r = await fetch(`/api/scraper?person=${encodeURIComponent(id)}`);
+  if (!r.ok) return { person: null };
+  return r.json();
 }
 
 /** What to tell someone when the scanner is not usable yet. */
@@ -67,3 +86,202 @@ export async function stopScrape() {
     body: JSON.stringify({ action: 'cancel' }),
   });
 }
+
+// One scan at a time, and every button knows it.
+//
+// The scanner runs one job and refuses a second. The buttons that start one
+// didn't know that: each kept its own idea of whether a scan was running, so a
+// scan started on one profile card left every other card's button live, and
+// pressing it said "Scan failed — connections may be private" about a scan
+// that had never started. Now they all read the scanner's own answer, asked
+// here once for the whole page however many buttons are showing: every 1.5 s
+// while a job runs, every 5 s while none does (so one started on the Scan page
+// or in another window greys them out too), and not at all while no button is
+// on screen. The idle look is skipped while the page is hidden.
+
+const BUSY_MS = 1500;
+const QUIET_MS = 5000;
+
+/** Before the first answer, and on the server, where nothing is asked. */
+export const SCANNER_UNKNOWN = Object.freeze({
+  known: false, running: false, pending: false, action: null, target: null,
+  startedAt: null, progress: null, log: [], finished: [],
+});
+
+const listeners = new Set();
+let now = SCANNER_UNKNOWN;
+let said = JSON.stringify(now);
+let timer = null;
+let asking = null;
+let epoch = 0;              // moves when this page starts a job: answers asked before then are stale
+let starting = false;       // this page's start request is on its way
+const followed = new Set(); // jobs (by startedAt) this page has seen running, or started
+
+function publish(next) {
+  const text = JSON.stringify(next);
+  if (text === said) return;
+  said = text;
+  now = Object.freeze(next);
+  for (const listener of [...listeners]) listener();
+}
+
+/**
+ * An answer from GET /api/scraper?job=1. `finished` is how the jobs this page
+ * watched ended, newest first, so a card can say its scan is done even if it
+ * was closed while the scan ran, and never shows the end of a scan from before
+ * the page opened.
+ */
+function take(s) {
+  if (!s || typeof s !== 'object') return;
+  const running = s.running === true;
+  if (running && s.startedAt != null) followed.add(s.startedAt);
+  const finished = (Array.isArray(s.recent) ? s.recent : [])
+    .filter((j) => followed.has(j.startedAt))
+    .map((j) => ({ ...j, log: !running && j.startedAt === s.startedAt && Array.isArray(s.log) ? s.log : [] }));
+  // While our own start request is on its way, the server may not have begun
+  // the job yet and says nothing runs: the buttons stay grey regardless.
+  publish({
+    known: true,
+    running: running || starting,
+    pending: starting && !running,
+    action: running ? s.action ?? null : starting ? now.action : null,
+    target: running ? s.target ?? null : starting ? now.target : null,
+    startedAt: running ? s.startedAt ?? null : null,
+    progress: running ? s.progress ?? null : null,
+    log: running && Array.isArray(s.log) ? s.log : [],
+    finished,
+  });
+}
+
+function ask() {
+  if (asking && asking.epoch === epoch) return asking.promise;
+  const asked = epoch;
+  const promise = fetch('/api/scraper?job=1')
+    .then((r) => (r.ok ? r.json() : null))
+    .then((s) => { if (asked === epoch) take(s); })
+    .catch(() => { /* the app restarting, say: keep the last answer */ })
+    .finally(() => { if (asking?.promise === promise) asking = null; });
+  asking = { epoch: asked, promise };
+  return promise;
+}
+
+function schedule() {
+  clearTimeout(timer);
+  timer = listeners.size ? setTimeout(tick, now.running ? BUSY_MS : QUIET_MS) : null;
+}
+
+async function tick() {
+  const hidden = typeof document !== 'undefined' && document.hidden;
+  if (!hidden || now.running) await ask();
+  schedule();
+}
+
+/** Call `listener` whenever the answer changes. Returns the way to stop. */
+export function watchScanner(listener) {
+  listeners.add(listener);
+  if (listeners.size === 1) tick();
+  return () => {
+    listeners.delete(listener);
+    if (!listeners.size) {
+      clearTimeout(timer);
+      timer = null;
+    }
+  };
+}
+
+/** The latest answer: { known, running, pending, action, target, startedAt, progress, log, finished }. */
+export function scannerNow() {
+  return now;
+}
+
+/**
+ * Start a job. Every Scan button greys out at once, not a poll later.
+ * Resolves once the scanner has taken it, with `ended`: a promise of how it
+ * ends ({ exitCode, failure, log, … }). onLog gets the whole log as it grows.
+ * `id` is who a scan of a person is of, so their card can find it again.
+ */
+export async function beginScrape(action, { name, id, onLog, ...extra } = {}) {
+  epoch++;
+  starting = true;
+  publish({
+    ...now, known: true, running: true, pending: true, action,
+    target: name || id ? { id: id ?? null, name: name ?? null } : null,
+    startedAt: null, progress: null, log: [],
+  });
+  let started;
+  try {
+    started = await startScrape(action, name, { ...(id ? { id } : {}), ...extra });
+  } catch (err) {
+    // Refused, often because something else runs: ask what, straight away.
+    // Until the answer comes, only our own claim is taken back.
+    starting = false;
+    epoch++;
+    if (now.pending) publish({ ...now, running: false, pending: false, action: null, target: null, log: [] });
+    tick();
+    throw err;
+  }
+  starting = false;
+  epoch++;
+  const mine = started?.startedAt ?? null;
+  if (mine != null) followed.add(mine);
+  const ended = new Promise((resolve) => {
+    const stop = watchScanner(() => {
+      if (now.running && now.startedAt === mine) onLog?.(now.log);
+      // A server from before jobs had a start time (one left running while
+      // the app updated): the first answer with nothing running is the end.
+      const end = mine == null
+        ? (!now.running ? { exitCode: null, failure: null, log: [] } : null)
+        : now.finished.find((j) => j.startedAt === mine);
+      if (!end) return;
+      stop();
+      if (end.log.length) onLog?.(end.log);
+      resolve(end);
+    });
+    tick();
+  });
+  return { startedAt: mine, ended };
+}
+
+/** Start a job and follow it to its end: resolves with how it ended. */
+export function runScrape(action, opts = {}) {
+  return beginScrape(action, opts).then((s) => s.ended);
+}
+
+/** Is this job, running or finished, a read of one person's circle? */
+export function isCircleScan(job) {
+  return Boolean(job?.target) && ['bridge', 'rescrape', 'resume'].includes(job.action);
+}
+
+/**
+ * Is this job, running or finished, a read of this person's circle? By id when
+ * both have one: two connections can share a name. Else by name.
+ */
+export function scansCircleOf(job, person) {
+  if (!isCircleScan(job) || !person) return false;
+  const { id, name } = job.target;
+  if (id && person.id) return id === person.id;
+  return Boolean(name) && name === person.name;
+}
+
+/** Why the Scan buttons are greyed out, in a few words; null when nothing runs. */
+export function busyReason(job = now) {
+  if (!job?.running) return null;
+  const who = job.target?.name;
+  switch (job.action) {
+    case 'bridge':
+    case 'rescrape':
+    case 'resume':
+      return who ? `${who}’s circle is being scanned` : 'A circle is being scanned';
+    case 'company':
+      return who ? `${who} is being scanned` : 'A company is being scanned';
+    case 'auto-bridge':
+    case 'auto-bridge-retry':
+    case 'resume-all':
+      return 'Your bridges are being mapped, one by one';
+    case 'full':
+    case 'refresh':
+      return 'Your own connections are being scanned';
+    default:
+      return runningNow(job.action);
+  }
+}

package/.next/standalone/lib/sector-directory.js

@@ -1460,7 +1460,7 @@ export function sectorMatcher() {
 export { sectorByKey, sectorLabel } from './sector-labels.js';
 
 /**
- * Everything Settings → Your sector can pick, in the order a choice is stored:
+ * Everything Scores → Your sector can pick, in the order a choice is stored:
  * each broad industry, then its sectors.
  */
 export const SECTOR_KEYS = Object.freeze(INDUSTRIES.flatMap((g) => [g.key, ...DIRECTORY.filter((s) => s.group === g.key).map((s) => s.key)]));

package/.next/standalone/lib/sector-focus.js

@@ -48,6 +48,19 @@ export function parseSectorFocus(value) {
 /** How Settings declares it (lib/settings.js). */
 export const SECTOR_FOCUS_SETTING = { default: NO_FOCUS, parse: parseSectorFocus };
 
+/**
+ * Whether the Scan page has asked for your field, before the first scan
+ * (app/components/FieldStep.js): true once it was answered or skipped, so it
+ * is asked once. Kept with the data, like the choice itself.
+ */
+export const FIELD_ASKED_SETTING = {
+  default: false,
+  parse(value) {
+    if (typeof value !== 'boolean') throw new Error('Whether your field was asked is true or false.');
+    return value;
+  },
+};
+
 /**
  * A short, stable string for a focus, stamped beside the stored scores
  * (app_meta 'scoring_focus'). If the saved focus stops matching it (a
@@ -128,10 +141,11 @@ export function tierMoves(rows, tierBefore, tierAfter, { examples = 5, companyOf
  */
 export function previewSectorFocus(rows = [], {
   overrides = new Map(), industryOf, sectorsOf = sectorMatcher(), groupOf = sectorGroup, from = NO_FOCUS, to = NO_FOCUS, examples = 5,
-  read = readNetwork(rows, { industryOf, sectorsOf, groupOf }),
+  tierScale, read = readNetwork(rows, { industryOf, sectorsOf, groupOf }),
 } = {}) {
-  const before = scoreNetwork(rows, { overrides, focus: from, read });
-  const after = scoreNetwork(rows, { overrides, focus: to, read });
+  // Both on the tier scale saved now (Scores → Tiers), as rescoring grades them.
+  const before = scoreNetwork(rows, { overrides, focus: from, tierScale, read });
+  const after = scoreNetwork(rows, { overrides, focus: to, tierScale, read });
 
   // Companies whose score moves, biggest first (by your people there now).
   // Every company anyone names counts, a former employer too: its score moves

package/.next/standalone/lib/separation.js

@@ -10,6 +10,8 @@
 // This is the ONE merge: the Separation view and the Sidebar's path box both
 // read it, so the list and the panel can never disagree about who reaches whom.
 
+import { localPhoto } from './photos.js';
+
 const TIER_RANK = { S: 0, A: 1, B: 2, C: 3, D: 4 };
 export const TIERS = ['S', 'A', 'B', 'C', 'D'];
 
@@ -62,11 +64,11 @@ function compareRoutes(a, b) {
   return byId(a.id, b.id);
 }
 
-/** The better copy of a person: higher shown score, then one with a photo, then the lower id. */
+/** The better copy of a person: higher shown score, then one with a saved photo, then the lower id. */
 function betterRow(a, b) {
   const d = score1(a) - score1(b);
   if (d) return d > 0;
-  const pa = !!a.profile_image_url, pb = !!b.profile_image_url;
+  const pa = !!localPhoto(a.profile_image_url), pb = !!localPhoto(b.profile_image_url);
   if (pa !== pb) return pa;
   return byId(a.id, b.id) < 0;
 }

package/.next/standalone/lib/settings-client.js

@@ -0,0 +1,21 @@
+// Saving settings from the browser: POST /api/settings with only the settings
+// that change (lib/settings.js keeps the rest as stored). Scores → Your sector
+// and Tiers save through it, and so does the Scan page's question about your
+// field before the first scan (app/components/FieldStep.js).
+
+/**
+ * @returns { settings, effects, error }: `settings` as saved, also when the
+ *   save landed but the work it set in motion failed (lib/settings-effects.js);
+ *   `effects` what that work reported; `error` a message fit to show, or null.
+ */
+export async function saveSettings(patch) {
+  try {
+    const r = await fetch('/api/settings', {
+      method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ settings: patch }),
+    });
+    const d = await r.json();
+    return { settings: d.settings || null, effects: d.effects || null, error: r.ok ? null : d.error || 'Could not save.' };
+  } catch {
+    return { settings: null, effects: null, error: 'Could not reach the app. Reload this page to see what is saved.' };
+  }
+}

package/.next/standalone/lib/settings-effects.js

@@ -4,12 +4,30 @@
 // circle). A setting with no entry here is simply saved.
 
 import { rescoreAll } from './rpc.js';
-import { sameFocus, NO_FOCUS } from './sector-focus.js';
+import { sameFocus, NO_FOCUS, tierMoves } from './sector-focus.js';
 
 // Each entry: `changed(before, after)` says whether a change matters (by
 // default, any change to the stored value), and `run(db, after, before)` does
 // the work and returns what to tell the page.
 const EFFECTS = {
+  tierScale: {
+    // Tiers are graded when everyone is scored, so a new scale rescores
+    // everyone. The answer counts who changed tier, from the stored tiers
+    // before and after, with the sector focus's own counter (tierMoves): a
+    // person reachable through two bridges is two rows but one person, counted
+    // once at their closest degree.
+    run(db) {
+      const rows = () => db.prepare('SELECT id, degree, profile_url, tier FROM linkedin_connections').all();
+      try {
+        const before = new Map(rows().map((r) => [r.id, r.tier]));
+        const { scored } = rescoreAll();
+        const { people, up, down } = tierMoves(rows(), (r) => before.get(r.id), (r) => r.tier, { examples: 0 });
+        return { scored, people, moved: up + down, up, down };
+      } catch (err) {
+        throw new Error(`Saved, but rescoring your network failed (${err.message}). It will try again the next time the map loads.`);
+      }
+    },
+  },
   sectorFocus: {
     // Only a change to what scoring uses: a strength with no sectors picked
     // scores exactly like nothing picked, so it saves without a rescore.

package/.next/standalone/lib/settings.js

@@ -10,11 +10,16 @@
 // error. A key that isn't declared is refused, so a stale page or a stray
 // request can't plant values nothing reads.
 
-import { SECTOR_FOCUS_SETTING } from './sector-focus.js';
+import { SECTOR_FOCUS_SETTING, FIELD_ASKED_SETTING } from './sector-focus.js';
+import { TIER_SCALE_SETTING } from './scoring.js';
 
 export const SETTINGS = {
-  // Settings → Your sector: industries whose companies count for more (lib/sector-focus.js).
+  // Scores → Your sector: industries whose companies count for more (lib/sector-focus.js).
   sectorFocus: SECTOR_FOCUS_SETTING,
+  // The Scan page's question before the first scan, your field: answered or skipped (lib/sector-focus.js).
+  fieldAsked: FIELD_ASKED_SETTING,
+  // Scores → Tiers: graded on your network's curve, or on the fixed scale (lib/scoring.js).
+  tierScale: TIER_SCALE_SETTING,
 };
 
 const KEY = 'settings';

package/.next/standalone/next.config.mjs

@@ -21,14 +21,22 @@ const nextConfig = {
   // server, never inside it: Electron alone is ~250 MB.
   //
   // scripts/pin-python-packages.mjs is a developer's tool that asks PyPI
-  // about packages; nothing the app runs uses it, so it doesn't ship.
+  // about packages, and scripts/check-site-version.mjs a release check of the
+  // website; nothing the app runs uses either, so they don't ship.
   outputFileTracingExcludes: {
     '*': ['dist/*.app/**', 'dist/*.dmg', 'dist/staging/**', 'dist/electron-stage/**', 'docs/**', 'tests/**',
-          '.git', '.git/**', '*.log', 'scripts/dmg/**', 'scripts/pin-python-packages.mjs', 'desktop/**',
+          '.git', '.git/**', '*.log', 'scripts/dmg/**', 'scripts/pin-python-packages.mjs', 'scripts/check-site-version.mjs', 'desktop/**',
           'node_modules/electron/**', 'node_modules/@electron/**',
           // The download website (site/, published by pages.yml from the repo) is
           // never read by the app; tracing swept its 17 MB of videos in.
-          'site/**'],
+          'site/**',
+          // A contributor's own data: a LinkedIn export, a database and its
+          // -wal/-shm, a saved copy of a network, the scanner's signed-in browser
+          // profile. .gitignore keeps them out of a commit; the trace never reads
+          // it. In contains mode these match at any depth, node_modules included,
+          // and nothing the app depends on has such a file (checked when added).
+          // A dependency that ships one would lose it, so smoke-test every route.
+          '*.csv', '*.sqlite*', '*.sixdegrees', 'chrome-profile/**'],
   },
   // Emits .next/standalone with a server and only the dependencies actually
   // reached, so the published package can run without node_modules being
@@ -39,6 +47,20 @@ const nextConfig = {
   // to be required at runtime.
   serverExternalPackages: ['node:sqlite'],
 
+  // Pictures come only from this app: saved photos through /avatars, and its
+  // own files. Views never put LinkedIn's image links in a page
+  // (lib/photos.js), and this has the browser refuse one if a view ever does,
+  // so browsing can't go online for a picture. The same pages are what the Mac
+  // app's window shows and what a browser gets from npx or a checkout, so this
+  // one header covers all three. img-src only: scripts, styles and fonts stay
+  // as Next needs them.
+  async headers() {
+    return [{
+      source: '/(.*)',
+      headers: [{ key: 'Content-Security-Policy', value: "img-src 'self' data: blob:" }],
+    }];
+  },
+
   // experimental.proxyClientMaxBodySize stays at Next's 10 MB on purpose. Next
   // copies the body of every request middleware.js sees into memory, up to that
   // size, before middleware decides anything, so raising it for the one big

package/.next/standalone/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "six-degrees",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "six-degrees",
-      "version": "0.3.0",
+      "version": "0.4.0",
       "license": "MIT",
       "os": [
         "darwin",

package/.next/standalone/package.json

@@ -1,6 +1,6 @@
 {
   "name": "six-degrees",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Map your professional network as a galaxy — leverage tiers, bridges into new circles, and the shortest path to anyone. Runs locally on your own data.",
   "license": "MIT",
   "author": "Blake Burford",
@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/blakeb056/six-degrees/issues"
   },
-  "homepage": "https://blakeb056.github.io/six-degrees/",
+  "homepage": "https://sixdegreesapp.com/",
   "keywords": [
     "linkedin",
     "network",