six-degrees 0.4.9 → 0.4.11

package/lib/linkedin-export.js

@@ -12,7 +12,14 @@
 //              (app/api/social/messages).
 //   chapters   your jobs (Positions.csv) and the connections made in each
 //   posts      your posts (Shares.csv) and the connections that came the week after
-//   invites    requests to and from you (Invitations.csv), when the export has them
+//   invites    requests to and from you (Invitations.csv), when the export has
+//              them: counts, and one record per request (who, which way, when).
+//              The note sent with a request only with Keep my messages.
+//   names      the names the export gives people, so someone who isn't a
+//              connection (and so isn't in the app's network) can be listed
+//
+// Every conversation is kept, whoever it's with and whatever folder it's in;
+// Sponsored Messages and InMails are marked, not dropped (buildConversations).
 //
 // Plain functions, no React; tests/linkedin-export.test.mjs.
 
@@ -111,29 +118,86 @@ export function messageStats(rows = []) {
   return { self, asOf: asOf || null, people };
 }
 
+/** A person with no profile link in the export, by their name: `name:ada quill`. */
+export const nameOnlyKey = (name) => {
+  const n = nameKey(name);
+  return n ? `name:${n}` : null;
+};
+
+const SPONSORED = /sponsored/i;
+const INMAIL = /in\s?mail/i;
+// A company's or a showcase page's link as the sender: an advert, not a person.
+const PAGE_SENDER = /linkedin\.com\/(company|showcase|school)\//i;
+const yes = (v) => /^(yes|true|1)$/i.test(String(v || '').trim());
+
+/**
+ * What LinkedIn's rows say a conversation is: 'sponsored' (a Sponsored
+ * Message, an advert), 'inmail' (an InMail), or null for an ordinary one. The
+ * export marks neither in a column of its own, so it's read from the folder,
+ * the conversation's title, and a sender that's a company page, not a person.
+ */
+function kindOfRow(r) {
+  const folder = String(r.folder || '');
+  const title = String(r['conversation title'] || '');
+  if (SPONSORED.test(folder) || SPONSORED.test(title) || PAGE_SENDER.test(String(r['sender profile url'] || ''))) return 'sponsored';
+  if (INMAIL.test(folder) || INMAIL.test(title)) return 'inmail';
+  return null;
+}
+
 /**
- * Every conversation in messages.csv you're part of, one record each:
- * { id, title, people, group, messages: [{ t, fromMe, text }] }, messages oldest
- * first. `people` is everyone but you, by profile key; more than one makes it a
- * group. You are found the way messageStats finds you. The words (the subject,
+ * Every conversation in messages.csv, one record each:
+ * { id, title, people, group, kind, folder, messages: [{ t, fromMe, text }] },
+ * messages oldest first. `people` is everyone but you, by profile key, or by
+ * name (nameOnlyKey) when the export has no link for them; more than one makes
+ * it a group, none (only you wrote, and to whom isn't said) leaves it empty.
+ * Nothing is left out for being in a folder: INBOX, SENT, ARCHIVE, SPAM and
+ * anything else LinkedIn adds all count; `folder` is the newest message's, in
+ * lower case. `kind` is 'sponsored', 'inmail' or null (kindOfRow). Drafts
+ * weren't sent, so they aren't messages; they're counted in `drafts`, and rows
+ * with no conversation or no readable date in `skipped`.
+ *
+ * You are found the way messageStats finds you, and by your name on rows with
+ * no link. `names` is everyone's name as the export spells it, by key, so a
+ * person who isn't a connection can still be shown. The words (the subject,
  * when there is one, then the message) are read only with keepText: without it
  * a message is a date and who sent it, and the title (a group's name) is left
  * out too.
  */
 export function buildConversations(rows = [], { keepText = false } = {}) {
   const self = selfOf(rows);
+  // Your name as the export spells it, for rows that have no link for you.
+  const selfNames = new Map();
+  for (const r of rows) {
+    if (self && urlKey(r['sender profile url']) === self && r.from) selfNames.set(r.from, (selfNames.get(r.from) || 0) + 1);
+  }
+  const selfName = nameKey([...selfNames].sort((a, b) => b[1] - a[1])[0]?.[0] || '');
+  const isMe = (k) => !!k && (k === self || (!!selfName && k === `name:${selfName}`));
+  const names = {};
   const convos = new Map();
+  let drafts = 0;
+  let skipped = 0;
   for (const r of rows) {
-    const from = urlKey(r['sender profile url']);
     const id = r['conversation id'];
-    if (!id || !from) continue;
     const t = exportDate(r.date);
-    if (t == null) continue;
-    if (!convos.has(id)) convos.set(id, { id, title: '', everyone: new Set(), messages: [] });
+    if (!id || t == null) { skipped += 1; continue; }
+    if (yes(r['is message draft'])) { drafts += 1; continue; }
+    const fromUrl = urlKey(r['sender profile url']);
+    const from = fromUrl || nameOnlyKey(r.from);
+    if (from && r.from && !names[from]) names[from] = String(r.from).trim();
+    const toUrls = String(r['recipient profile urls'] || '').split(/[,;\s]+/).filter(Boolean).map(urlKey);
+    const toNames = String(r.to || '').split(',').map((s) => s.trim()).filter(Boolean);
+    // Names line up with links only when there are as many of each (a name can
+    // hold a comma: "Ben Ode, MBA"); without links, the names are all there is.
+    const to = toUrls.length ? toUrls : toNames.map(nameOnlyKey).filter(Boolean);
+    if (toUrls.length && toUrls.length === toNames.length) toUrls.forEach((u, i) => { if (!names[u]) names[u] = toNames[i]; });
+    if (!toUrls.length) toNames.forEach((n) => { const k = nameOnlyKey(n); if (k && !names[k]) names[k] = n; });
+    if (!convos.has(id)) convos.set(id, { id, title: '', everyone: new Set(), messages: [], kind: null, folder: '', newest: -Infinity });
     const c = convos.get(id);
-    c.everyone.add(from);
-    String(r['recipient profile urls'] || '').split(/[,;\s]+/).filter(Boolean).forEach((u) => c.everyone.add(urlKey(u)));
-    const m = { t, fromMe: from === self };
+    if (from) c.everyone.add(from);
+    to.forEach((p) => c.everyone.add(p));
+    c.kind = c.kind || kindOfRow(r);
+    if (t >= c.newest) { c.newest = t; c.folder = String(r.folder || '').trim().toLowerCase().slice(0, 30); }
+    const m = { t, fromMe: isMe(from) };
     if (keepText) {
       m.text = [r.subject, r.content].map((x) => String(x || '').trim()).filter(Boolean).join('\n\n');
       if (!c.title && r['conversation title']) c.title = String(r['conversation title']).trim();
@@ -142,37 +206,62 @@ export function buildConversations(rows = [], { keepText = false } = {}) {
   }
   const conversations = [];
   for (const c of convos.values()) {
-    if (!c.everyone.has(self)) continue;
-    const people = [...c.everyone].filter((p) => p && p !== self).sort();
-    if (!people.length) continue;
+    const people = [...c.everyone].filter((p) => p && !isMe(p)).sort();
     c.messages.sort((a, b) => a.t - b.t);
-    const out = { id: c.id, people, group: people.length > 1, messages: c.messages };
+    const out = { id: c.id, people, group: people.length > 1, kind: c.kind, folder: c.folder, messages: c.messages };
     if (keepText) out.title = c.title;
     conversations.push(out);
   }
-  return { self, conversations };
+  for (const k of Object.keys(names)) if (isMe(k)) delete names[k];
+  return { self, conversations, names, drafts, skipped };
 }
 
-/** What the Conversations list keeps of one conversation, never any words: { id, people, group, last, lastFromThem, count }. */
+/**
+ * What the Conversations list keeps of one conversation, never any words:
+ * { id, people, group, kind, folder, last, lastFromThem, count, mine }. `mine`
+ * is how many of the messages you wrote; all of them means no one has answered.
+ */
 export function conversationSummary(c) {
   const last = c.messages[c.messages.length - 1];
   return {
     id: c.id,
     people: c.people,
     group: c.group,
+    kind: c.kind || null,
+    folder: c.folder || '',
     last: last ? last.t : null,
     lastFromThem: last ? !last.fromMe : null,
     count: c.messages.length,
+    mine: c.messages.filter((m) => m.fromMe).length,
   };
 }
 
+/** The same people, by their names: a set's signature, or null when a name is missing. */
+function nameSignature(keys, nameOf) {
+  const out = [];
+  for (const k of keys) {
+    const n = nameKey(nameOf(k));
+    if (!n) return null;
+    out.push(n);
+  }
+  return out.sort().join('|');
+}
+
 /**
  * The Conversations list: the export's conversations with what the live sync
  * adds (unread, the thread's link, and a newer last message), plus a
- * conversation for each person the sync found and the export doesn't have yet
- * (id `live:<key>`, count unknown).
+ * conversation for each person or group the sync found and the export doesn't
+ * have yet (id `live:<key>`, count unknown).
+ *
+ *   live    your connections, by their key (matchLive)
+ *   others  people who aren't your connections, by the link messaging gives
+ *           them, with their name: matched to the export's conversation with
+ *           someone of that name when only one has it
+ *   groups  group conversations: { threadUrl, people, names, last, unread },
+ *           matched to the export's group with the same people by name
+ *   names   the export's names, by key
  */
-export function conversationList(list = [], live = {}) {
+export function conversationList(list = [], live = {}, { others = {}, groups = [], names = {} } = {}) {
   const out = (Array.isArray(list) ? list : []).map((c) => ({ ...c, unread: null, threadUrl: null }));
   const oneToOne = new Map();
   for (const c of out) {
@@ -180,20 +269,59 @@ export function conversationList(list = [], live = {}) {
     const had = oneToOne.get(c.people[0]);
     if (!had || (c.last || 0) > (had.last || 0)) oneToOne.set(c.people[0], c);
   }
+  const addLive = (c, v) => {
+    if (v.last && (!c.last || v.last > c.last)) {
+      c.last = v.last;
+      if (typeof v.lastFromThem === 'boolean') c.lastFromThem = v.lastFromThem;
+    }
+    c.unread = Number.isFinite(v.unread) ? v.unread : c.unread;
+    c.threadUrl = v.threadUrl || c.threadUrl;
+  };
+  const fresh = (id, people, group) => ({ id, people, group, kind: null, folder: '', last: null, lastFromThem: null, count: null, mine: null, unread: null, threadUrl: null });
   for (const [key, v] of Object.entries(live || {})) {
     if (!v || typeof v !== 'object') continue;
     let c = oneToOne.get(key);
     if (!c) {
-      c = { id: `live:${key}`, people: [key], group: false, last: null, lastFromThem: null, count: null, unread: null, threadUrl: null };
+      c = fresh(`live:${key}`, [key], false);
       out.push(c);
       oneToOne.set(key, c);
     }
-    if (v.last && (!c.last || v.last > c.last)) {
-      c.last = v.last;
-      if (typeof v.lastFromThem === 'boolean') c.lastFromThem = v.lastFromThem;
+    addLive(c, v);
+  }
+  // Someone who isn't a connection: the export knows them by their /in/name
+  // link, messaging by /in/ACoA…, so only their name can join the two.
+  const byName = new Map();
+  for (const [key, c] of oneToOne) {
+    const n = nameKey(names[key]);
+    if (n) byName.set(n, byName.has(n) ? null : c);
+  }
+  for (const [link, v] of Object.entries(others || {})) {
+    if (!v || typeof v !== 'object') continue;
+    let c = oneToOne.get(link) || byName.get(nameKey(v.name)) || null;
+    if (!c) {
+      c = fresh(`live:${link}`, [link], false);
+      out.push(c);
+      oneToOne.set(link, c);
     }
-    c.unread = Number.isFinite(v.unread) ? v.unread : c.unread;
-    c.threadUrl = v.threadUrl || c.threadUrl;
+    c.liveKey = link;
+    addLive(c, v);
+  }
+  const groupsByNames = new Map();
+  for (const c of out) {
+    if (!c.group) continue;
+    const sig = nameSignature(c.people, (k) => names[k]);
+    if (sig) groupsByNames.set(sig, groupsByNames.has(sig) ? null : c);
+  }
+  for (const g of Array.isArray(groups) ? groups : []) {
+    if (!g || !Array.isArray(g.people) || !g.people.length) continue;
+    const sig = nameSignature(g.people, (k) => g.names?.[k] || names[k]);
+    let c = (sig && groupsByNames.get(sig)) || null;
+    if (!c) {
+      c = fresh(`live:${g.threadUrl || g.people.join(',')}`, g.people, true);
+      out.push(c);
+      if (sig) groupsByNames.set(sig, c);
+    }
+    addLive(c, g);
   }
   return out;
 }
@@ -268,7 +396,32 @@ export function invitationSplit(rows = []) {
   return { incoming, outgoing, total: incoming + outgoing };
 }
 
-const nameKey = (s) => String(s || '').split(',')[0].normalize('NFD').replace(/[̀-ͯ]/g, '')
+/**
+ * Every request in Invitations.csv, one record each, newest first:
+ * { id, dir: 'in' | 'out', key, name, t, note? }. `key` is the other person's
+ * profile link (inviterProfileUrl or inviteeProfileUrl, in newer exports), or
+ * their name (nameOnlyKey) when there's no link. The note sent with a request
+ * is written by a person, like a message, so it's read only with keepText.
+ */
+export function buildInvitations(rows = [], { keepText = false } = {}) {
+  const out = [];
+  for (const r of rows) {
+    const d = String(r.direction || '').toUpperCase();
+    if (d !== 'INCOMING' && d !== 'OUTGOING') continue;
+    const dir = d === 'INCOMING' ? 'in' : 'out';
+    const name = String((dir === 'in' ? r.from : r.to) || '').trim();
+    const url = urlKey(dir === 'in' ? r.inviterprofileurl : r.inviteeprofileurl);
+    const key = (url && /\/in\//.test(url) ? url : null) || nameOnlyKey(name);
+    if (!key) continue;
+    const t = exportDate(r['sent at']);
+    const inv = { id: `${dir}|${key}|${t ?? ''}`, dir, key, name: name.slice(0, 120), t };
+    if (keepText && String(r.message || '').trim()) inv.note = String(r.message).trim().slice(0, 2000);
+    out.push(inv);
+  }
+  return out.sort((a, b) => (b.t || 0) - (a.t || 0));
+}
+
+export const nameKey = (s) => String(s || '').split(',')[0].normalize('NFD').replace(/[̀-ͯ]/g, '')
   .toLowerCase().replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim();
 
 const THREAD_LINK = /^https:\/\/www\.linkedin\.com\/messaging\/[\w\-/%=.]*$/;
@@ -288,16 +441,8 @@ function previewOf(p) {
   return { t, fromMe: p.fromMe === true, text };
 }
 
-/**
- * The live messages sync's people, matched to your connections. LinkedIn's
- * messaging gives a member link (/in/ACoA…) your connections list doesn't have,
- * so a person is matched by their link when it's the same, otherwise by their
- * name when exactly one connection has it. Only the match is kept, never the
- * name. Returns { live: { connectionKey: { last, unread, threadUrl,
- * lastFromThem, preview } }, unmatched }; the caller decides whether the
- * preview's words are kept.
- */
-export function matchLive(incoming = {}, connections = []) {
+/** (link, name) → your connection's key: by link when it's the same, else by a name only one connection has; null otherwise. */
+function connectionMatcher(connections = []) {
   const byUrl = new Map();
   const byName = new Map();
   for (const c of connections) {
@@ -306,21 +451,97 @@ export function matchLive(incoming = {}, connections = []) {
     const n = nameKey(c.name);
     if (n) byName.set(n, byName.has(n) ? null : key);   // null: two people share it
   }
+  return (url, name) => byUrl.get(keyFor({ profile_url: url })) || byName.get(nameKey(name)) || null;
+}
+
+const PROFILE = /^https:\/\/www\.linkedin\.com\/in\/[^/?#\s]+\/?$/;
+const cleanName = (s) => (typeof s === 'string' ? s.replace(/[\u0000-\u001f]/g, ' ').trim().slice(0, 120) : '');
+
+/**
+ * The live messages sync's people, matched to your connections. LinkedIn's
+ * messaging gives a member link (/in/ACoA…) your connections list doesn't have,
+ * so a person is matched by their link when it's the same, otherwise by their
+ * name when exactly one connection has it. For a connection only the match is
+ * kept, never the name: the app has it already. Returns
+ *
+ *   live       { connectionKey: { last, unread, threadUrl, lastFromThem, preview } }
+ *   others     { link: { name, ...the same } }: people who aren't your
+ *              connections. Their name is kept (and nothing else says who they
+ *              are), so the Social tab can list them.
+ *   unmatched  how many are in `others`
+ *
+ * The caller decides whether the previews' words are kept.
+ */
+export function matchLive(incoming = {}, connections = []) {
+  const match = connectionMatcher(connections);
   const live = {};
-  let unmatched = 0;
+  const others = {};
   for (const [url, v] of Object.entries(incoming || {})) {
     if (!v || typeof v !== 'object') continue;
-    const key = byUrl.get(keyFor({ profile_url: url })) || byName.get(nameKey(v.name)) || null;
-    if (!key) { unmatched += 1; continue; }
-    const last = Number(v.last) || null;
-    if (live[key] && (live[key].last || 0) >= (last || 0)) continue;
-    live[key] = {
-      last,
+    const key = match(url, v.name);
+    const entry = {
+      last: Number(v.last) || null,
       unread: Number.isFinite(Number(v.unread)) && v.unread !== null ? Number(v.unread) : null,
       threadUrl: threadLink(v.threadUrl),
       lastFromThem: typeof v.lastFromThem === 'boolean' ? v.lastFromThem : null,
       preview: previewOf(v.preview),
     };
+    if (!key) {
+      const link = keyFor({ profile_url: url });
+      if (PROFILE.test(url) && !others[link]) others[link] = { name: cleanName(v.name), ...entry };
+      continue;
+    }
+    if (live[key] && (live[key].last || 0) >= (entry.last || 0)) continue;
+    live[key] = entry;
+  }
+  return { live, others, unmatched: Object.keys(others).length };
+}
+
+/**
+ * The live sync's group conversations, each member matched to your connections
+ * as matchLive does: { threadUrl, people, names, last, unread }. `people` are
+ * connection keys where matched, else the link messaging gave; `names` keeps
+ * the names of the ones who aren't connections only. No words: a group's
+ * newest message isn't read.
+ */
+export function matchLiveGroups(groups = [], connections = []) {
+  const match = connectionMatcher(connections);
+  const out = [];
+  for (const g of Array.isArray(groups) ? groups.slice(0, 5000) : []) {
+    if (!g || typeof g !== 'object' || !Array.isArray(g.people)) continue;
+    const people = [];
+    const names = {};
+    for (const url of g.people.slice(0, 200)) {
+      if (typeof url !== 'string' || !PROFILE.test(url)) continue;
+      const name = cleanName(g.names?.[url]);
+      const key = match(url, name);
+      if (key) { people.push(key); continue; }
+      const link = keyFor({ profile_url: url });
+      people.push(link);
+      if (name) names[link] = name;
+    }
+    if (people.length < 2) continue;
+    out.push({
+      threadUrl: threadLink(g.threadUrl),
+      people: [...new Set(people)].sort(),
+      names,
+      last: Number(g.last) || null,
+      unread: Number.isFinite(Number(g.unread)) && g.unread !== null ? Number(g.unread) : null,
+      lastFromThem: typeof g.lastFromThem === 'boolean' ? g.lastFromThem : null,
+    });
+  }
+  return out;
+}
+
+/**
+ * A new live sync over what earlier ones found: per person (or group), the
+ * newer of the two. A quick sync reads only the top of the list, so what a
+ * full-history sync found further down stays until something newer replaces it.
+ */
+export function mergeLive(before = {}, after = {}) {
+  const out = { ...(before && typeof before === 'object' ? before : {}) };
+  for (const [k, v] of Object.entries(after || {})) {
+    if (!out[k] || (v.last || 0) >= (out[k].last || 0)) out[k] = v;
   }
-  return { live, unmatched };
+  return out;
 }