six-degrees 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.next/standalone/.next/BUILD_ID +1 -1
- package/.next/standalone/.next/build-manifest.json +3 -3
- package/.next/standalone/.next/prerender-manifest.json +13 -13
- package/.next/standalone/.next/server/app/_global-error.html +1 -1
- package/.next/standalone/.next/server/app/_global-error.rsc +1 -1
- package/.next/standalone/.next/server/app/_global-error.segments/__PAGE__.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_global-error.segments/_full.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_global-error.segments/_tree.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_not-found/page.js +1 -1
- package/.next/standalone/.next/server/app/_not-found/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/_not-found/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/_not-found.html +1 -1
- package/.next/standalone/.next/server/app/_not-found.rsc +18 -17
- package/.next/standalone/.next/server/app/_not-found.segments/_full.segment.rsc +18 -17
- package/.next/standalone/.next/server/app/_not-found.segments/_not-found/__PAGE__.segment.rsc +14 -13
- package/.next/standalone/.next/server/app/_not-found.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/api/admin-delete/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/admin-update/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/bridges/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/bulk-import/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/company-scores/legacy/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/company-scores/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/connections/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/data/export/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/data/import/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/data/restart/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/data/reveal/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/data/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/delete-cluster/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/ingest/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/network/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/notifications/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/outreach/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/queue/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/scraper/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/settings/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/settings/sector-preview/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/settings/sector-suggestions/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/setup-profile/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/unlock/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/update/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/update-images/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/users/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/avatars/[file]/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/import/page.js +1 -1
- package/.next/standalone/.next/server/app/import/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/import/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/import.html +1 -1
- package/.next/standalone/.next/server/app/import.rsc +22 -21
- package/.next/standalone/.next/server/app/import.segments/_full.segment.rsc +22 -21
- package/.next/standalone/.next/server/app/import.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/import.segments/import/__PAGE__.segment.rsc +16 -15
- package/.next/standalone/.next/server/app/index.html +1 -1
- package/.next/standalone/.next/server/app/index.rsc +20 -19
- package/.next/standalone/.next/server/app/index.segments/__PAGE__.segment.rsc +16 -15
- package/.next/standalone/.next/server/app/index.segments/_full.segment.rsc +20 -19
- package/.next/standalone/.next/server/app/index.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/launch/page.js +1 -1
- package/.next/standalone/.next/server/app/launch/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/launch/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/launch.html +2 -2
- package/.next/standalone/.next/server/app/launch.rsc +22 -21
- package/.next/standalone/.next/server/app/launch.segments/_full.segment.rsc +22 -21
- package/.next/standalone/.next/server/app/launch.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/launch.segments/launch/__PAGE__.segment.rsc +16 -15
- package/.next/standalone/.next/server/app/page.js +1 -1
- package/.next/standalone/.next/server/app/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/paths/page.js +1 -1
- package/.next/standalone/.next/server/app/paths/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/paths/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/paths.html +1 -1
- package/.next/standalone/.next/server/app/paths.rsc +22 -21
- package/.next/standalone/.next/server/app/paths.segments/_full.segment.rsc +22 -21
- package/.next/standalone/.next/server/app/paths.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/paths.segments/paths/__PAGE__.segment.rsc +16 -15
- package/.next/standalone/.next/server/app/profile/page.js +1 -1
- package/.next/standalone/.next/server/app/profile/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/profile/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/profile.html +1 -1
- package/.next/standalone/.next/server/app/profile.rsc +22 -21
- package/.next/standalone/.next/server/app/profile.segments/_full.segment.rsc +22 -21
- package/.next/standalone/.next/server/app/profile.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/profile.segments/profile/__PAGE__.segment.rsc +16 -15
- package/.next/standalone/.next/server/app/queue/page.js +1 -1
- package/.next/standalone/.next/server/app/queue/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/queue/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/queue.html +1 -1
- package/.next/standalone/.next/server/app/queue.rsc +22 -21
- package/.next/standalone/.next/server/app/queue.segments/_full.segment.rsc +22 -21
- package/.next/standalone/.next/server/app/queue.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/queue.segments/queue/__PAGE__.segment.rsc +16 -15
- package/.next/standalone/.next/server/app/scores/page.js +1 -1
- package/.next/standalone/.next/server/app/scores/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/scores/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/scores.html +1 -1
- package/.next/standalone/.next/server/app/scores.rsc +22 -21
- package/.next/standalone/.next/server/app/scores.segments/_full.segment.rsc +22 -21
- package/.next/standalone/.next/server/app/scores.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/scores.segments/scores/__PAGE__.segment.rsc +16 -15
- package/.next/standalone/.next/server/app/settings/page.js +1 -1
- package/.next/standalone/.next/server/app/settings/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/settings/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/settings.html +1 -1
- package/.next/standalone/.next/server/app/settings.rsc +22 -21
- package/.next/standalone/.next/server/app/settings.segments/_full.segment.rsc +22 -21
- package/.next/standalone/.next/server/app/settings.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/settings.segments/settings/__PAGE__.segment.rsc +16 -15
- package/.next/standalone/.next/server/app/setup/page.js +1 -1
- package/.next/standalone/.next/server/app/setup/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/setup/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/setup.html +1 -1
- package/.next/standalone/.next/server/app/setup.rsc +22 -21
- package/.next/standalone/.next/server/app/setup.segments/_full.segment.rsc +22 -21
- package/.next/standalone/.next/server/app/setup.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/setup.segments/setup/__PAGE__.segment.rsc +16 -15
- package/.next/standalone/.next/server/chunks/_1_fynia._.js +5 -5
- package/.next/standalone/.next/server/chunks/_1itd1o_._.js +1 -1
- package/.next/standalone/.next/server/chunks/lib_db-client_15le7cw.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0d3vjsq._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0m4vk0b._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/{_15xuwkv._.js → _012mop3._.js} +2 -2
- package/.next/standalone/.next/server/chunks/ssr/_1163x49._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/_1av11ws._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/_1ig7em7._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/app_page_1fazqbf.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/app_setup_page_1-282zx.js +1 -1
- package/.next/standalone/.next/server/middleware-build-manifest.js +3 -3
- package/.next/standalone/.next/server/middleware-manifest.json +5 -5
- package/.next/standalone/.next/server/pages/404.html +1 -1
- package/.next/standalone/.next/server/pages/500.html +1 -1
- package/.next/standalone/.next/server/server-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/static/chunks/{049vavjsmqgmn.js → 0_3ubivay59bq.js} +4 -4
- package/.next/standalone/.next/static/chunks/{2p6o90z0vgcwr.css → 0bo1dxn5gwzpk.css} +1 -1
- package/.next/standalone/.next/static/chunks/0f4ugd6v3ph0-.js +1 -0
- package/.next/standalone/.next/static/chunks/10ogurbcuf3nq.js +1 -0
- package/.next/standalone/.next/static/chunks/1489t7jrjojk_.js +1 -0
- package/.next/standalone/.next/static/chunks/{0ax_9zzboyv6d.js → 17-rhp8sg4m1b.js} +2 -2
- package/.next/standalone/.next/static/chunks/1zunq4tk-u3ng.js +1 -0
- package/.next/standalone/.next/static/chunks/1zwjy1mmpkdrg.js +6 -0
- package/.next/standalone/.next/static/chunks/2_wcobfqry2uj.js +1 -0
- package/.next/standalone/.next/static/chunks/2dwtoszeq91us.js +1 -0
- package/.next/standalone/.next/static/chunks/{0ljxa-wk_iu__.js → 2pyo4j84prgp6.js} +3 -3
- package/.next/{static/chunks/22x2_r768-u5i.js → standalone/.next/static/chunks/2wwmljnf7yn_5.js} +1 -1
- package/.next/{static/chunks/2xjho0lroiwlq.js → standalone/.next/static/chunks/3kllnkb3hof1s.js} +1 -1
- package/.next/standalone/CHANGELOG.md +27 -0
- package/.next/standalone/app/api/scraper/route.js +17 -0
- package/.next/standalone/app/components/FilterPanel.js +37 -5
- package/.next/standalone/app/components/OrbitGraph.js +17 -4
- package/.next/standalone/app/components/ScanStatusBar.js +82 -0
- package/.next/standalone/app/globals.css +4 -0
- package/.next/standalone/app/layout.js +3 -0
- package/.next/standalone/app/page.js +8 -4
- package/.next/standalone/app/setup/page.js +13 -1
- package/.next/standalone/lib/scraper-client.js +1 -0
- package/.next/standalone/package-lock.json +2 -2
- package/.next/standalone/package.json +1 -1
- package/.next/standalone/scripts/scrape.py +219 -5
- package/.next/static/chunks/{049vavjsmqgmn.js → 0_3ubivay59bq.js} +4 -4
- package/.next/static/chunks/{2p6o90z0vgcwr.css → 0bo1dxn5gwzpk.css} +1 -1
- package/.next/static/chunks/0f4ugd6v3ph0-.js +1 -0
- package/.next/static/chunks/10ogurbcuf3nq.js +1 -0
- package/.next/static/chunks/1489t7jrjojk_.js +1 -0
- package/.next/static/chunks/{0ax_9zzboyv6d.js → 17-rhp8sg4m1b.js} +2 -2
- package/.next/static/chunks/1zunq4tk-u3ng.js +1 -0
- package/.next/static/chunks/1zwjy1mmpkdrg.js +6 -0
- package/.next/static/chunks/2_wcobfqry2uj.js +1 -0
- package/.next/static/chunks/2dwtoszeq91us.js +1 -0
- package/.next/static/chunks/{0ljxa-wk_iu__.js → 2pyo4j84prgp6.js} +3 -3
- package/.next/{standalone/.next/static/chunks/22x2_r768-u5i.js → static/chunks/2wwmljnf7yn_5.js} +1 -1
- package/.next/{standalone/.next/static/chunks/2xjho0lroiwlq.js → static/chunks/3kllnkb3hof1s.js} +1 -1
- package/CHANGELOG.md +27 -0
- package/lib/scraper-client.js +1 -0
- package/package.json +1 -1
- package/scripts/scrape.py +219 -5
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0h1_f_2._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__19ip1qq._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/_06w_gl4._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/_13kzrwv._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/_1hgd5dp._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/node_modules_1f8t_rp._.js +0 -23
- package/.next/standalone/.next/server/chunks/ssr/node_modules_next_dist_client_components_server-async-storage_0t1dato.js +0 -3
- package/.next/standalone/.next/static/chunks/0mp9b72fm-jp6.js +0 -6
- package/.next/standalone/.next/static/chunks/1pre1cotntk6-.js +0 -1
- package/.next/standalone/.next/static/chunks/28bu26ausqzmn.js +0 -1
- package/.next/standalone/.next/static/chunks/2jz311xl1pp_8.js +0 -1
- package/.next/standalone/.next/static/chunks/2ot61qsv6_0eq.js +0 -1
- package/.next/standalone/.next/static/chunks/3zs_6-nn0dapb.js +0 -1
- package/.next/static/chunks/0mp9b72fm-jp6.js +0 -6
- package/.next/static/chunks/1pre1cotntk6-.js +0 -1
- package/.next/static/chunks/28bu26ausqzmn.js +0 -1
- package/.next/static/chunks/2jz311xl1pp_8.js +0 -1
- package/.next/static/chunks/2ot61qsv6_0eq.js +0 -1
- package/.next/static/chunks/3zs_6-nn0dapb.js +0 -1
- /package/.next/standalone/.next/static/{wGVrXBkFLmEB4snfhtOcO → 814ehZBeFO2rUKle7mzX-}/_buildManifest.js +0 -0
- /package/.next/standalone/.next/static/{wGVrXBkFLmEB4snfhtOcO → 814ehZBeFO2rUKle7mzX-}/_clientMiddlewareManifest.js +0 -0
- /package/.next/standalone/.next/static/{wGVrXBkFLmEB4snfhtOcO → 814ehZBeFO2rUKle7mzX-}/_ssgManifest.js +0 -0
- /package/.next/static/{wGVrXBkFLmEB4snfhtOcO → 814ehZBeFO2rUKle7mzX-}/_buildManifest.js +0 -0
- /package/.next/static/{wGVrXBkFLmEB4snfhtOcO → 814ehZBeFO2rUKle7mzX-}/_clientMiddlewareManifest.js +0 -0
- /package/.next/static/{wGVrXBkFLmEB4snfhtOcO → 814ehZBeFO2rUKle7mzX-}/_ssgManifest.js +0 -0
|
@@ -295,11 +295,24 @@ const OrbitGraph = forwardRef(function OrbitGraph(
|
|
|
295
295
|
placed += row.length;
|
|
296
296
|
r += step;
|
|
297
297
|
}
|
|
298
|
+
// Their people go behind this fan, centred on it (not on wherever the
|
|
299
|
+
// person they came through sits), side by side in the order those people
|
|
300
|
+
// sit, so no fan spills into the next one's space; a line runs back to
|
|
301
|
+
// each. Wider than the wedge, they close up to fit it.
|
|
298
302
|
let edge = r;
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
const
|
|
302
|
-
|
|
303
|
+
const next = subs.filter(({ d }) => depth < 6 && reachOf(d)).sort((x, y) => x.a - y.a);
|
|
304
|
+
if (next.length) {
|
|
305
|
+
const from = r + 24;
|
|
306
|
+
const gap = 10 / from;
|
|
307
|
+
const want = next.map(({ d }) => Math.max(0.12, (Math.min(reachOf(d), 60) * 8) / from));
|
|
308
|
+
const need = want.reduce((sum, w) => sum + w, 0) + gap * (next.length - 1);
|
|
309
|
+
const scale = need > wedge ? wedge / need : 1;
|
|
310
|
+
let at = angle - (need * scale) / 2;
|
|
311
|
+
next.forEach(({ d }, k) => {
|
|
312
|
+
const w = want[k] * scale;
|
|
313
|
+
edge = Math.max(edge, placeFan(d, at + w / 2, w, from, depth + 1));
|
|
314
|
+
at += w + gap * scale;
|
|
315
|
+
});
|
|
303
316
|
}
|
|
304
317
|
return edge;
|
|
305
318
|
};
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
'use client';
|
|
2
|
+
|
|
3
|
+
// The scan status bar: while a scan runs, every page shows what it's doing,
|
|
4
|
+
// for whom, how far it's got, and today's LinkedIn budget, with Stop. Stop
|
|
5
|
+
// saves what was read, and the Scan page's Resume carries on from that page.
|
|
6
|
+
// It sits just under the header's tabs (Network Circle, Degrees…), on every
|
|
7
|
+
// page, the Scan page too (Blake, 2026-09-29). It only reports: the
|
|
8
|
+
// pacing and the caps live in the scanner (scripts/scrape.py, lib/linkedin-limits.js).
|
|
9
|
+
|
|
10
|
+
import { useEffect, useState } from 'react';
|
|
11
|
+
import Link from 'next/link';
|
|
12
|
+
import { watchScanner, scannerNow, stopScrape } from '../../lib/scraper-client';
|
|
13
|
+
|
|
14
|
+
const WHAT = {
|
|
15
|
+
full: 'Scanning your network',
|
|
16
|
+
refresh: 'Checking for new connections',
|
|
17
|
+
bridge: 'Scanning a circle',
|
|
18
|
+
resume: 'Carrying on with a circle',
|
|
19
|
+
'resume-all': 'Carrying on with circles',
|
|
20
|
+
'auto-bridge': 'Auto-Bridge',
|
|
21
|
+
'auto-bridge-retry': 'Auto-Bridge',
|
|
22
|
+
rescrape: 'Re-scanning a circle',
|
|
23
|
+
login: 'Signing in to LinkedIn',
|
|
24
|
+
setup: 'Setting up the scanner',
|
|
25
|
+
company: 'Scanning a company',
|
|
26
|
+
photos: 'Saving photos',
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
// Green while there's plenty left, amber past 60%, red past 90%.
|
|
30
|
+
const tone = (used, cap) => (!cap ? '#8b9a9a' : used / cap >= 0.9 ? '#ff6b6b' : used / cap >= 0.6 ? '#FFD700' : '#00ff88');
|
|
31
|
+
|
|
32
|
+
export default function ScanStatusBar() {
|
|
33
|
+
const [job, setJob] = useState(() => scannerNow());
|
|
34
|
+
const [stopping, setStopping] = useState(false);
|
|
35
|
+
useEffect(() => watchScanner(() => {
|
|
36
|
+
const now = scannerNow();
|
|
37
|
+
setJob(now);
|
|
38
|
+
if (!now.running) setStopping(false);
|
|
39
|
+
}), []);
|
|
40
|
+
if (!job?.running) return null;
|
|
41
|
+
|
|
42
|
+
const p = job.progress;
|
|
43
|
+
const step = p?.kind === 'batch' && p.total ? `${p.current || p.done || 0} of ${p.total}`
|
|
44
|
+
: p?.kind === 'walk' && p.total ? `${(p.done || 0).toLocaleString()} of ${p.total.toLocaleString()} read`
|
|
45
|
+
: p?.kind === 'saving' ? 'saving' : null;
|
|
46
|
+
const b = job.budget;
|
|
47
|
+
const last = [...(job.log || [])].reverse().find((l) => l && l.trim())?.trim();
|
|
48
|
+
|
|
49
|
+
return (
|
|
50
|
+
<div role="status" aria-live="polite" style={{
|
|
51
|
+
position: 'fixed', left: '50%', top: 'var(--scan-bar-top, 94px)', transform: 'translateX(-50%)', zIndex: 60,
|
|
52
|
+
maxWidth: 'calc(100vw - 32px)', display: 'flex', alignItems: 'center', gap: 12, flexWrap: 'nowrap', whiteSpace: 'nowrap',
|
|
53
|
+
padding: '8px 12px', borderRadius: 12, fontSize: 12, color: '#cfd8d8',
|
|
54
|
+
background: 'rgba(14,16,32,0.94)', border: '1px solid rgba(0,255,136,0.25)',
|
|
55
|
+
}}>
|
|
56
|
+
<span style={{ width: 8, height: 8, borderRadius: '50%', background: '#00ff88', flexShrink: 0 }} />
|
|
57
|
+
<b style={{ color: '#fff' }}>{WHAT[job.action] || 'Scanning'}</b>
|
|
58
|
+
{job.target?.name && <span>{job.target.name}</span>}
|
|
59
|
+
{step && <span style={{ color: '#8b9a9a' }}>{step}</span>}
|
|
60
|
+
{b && (
|
|
61
|
+
<span style={{ color: tone(b.searches, b.cap) }} title="Searches on your LinkedIn account in the last 24 hours, and your daily budget">
|
|
62
|
+
{b.searches}{b.cap ? ` of ${b.cap}` : ''} searches today
|
|
63
|
+
</span>
|
|
64
|
+
)}
|
|
65
|
+
{last && (
|
|
66
|
+
<span style={{ color: '#667', flex: '0 1 240px', minWidth: 0, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }} title={last}>
|
|
67
|
+
{last}
|
|
68
|
+
</span>
|
|
69
|
+
)}
|
|
70
|
+
<Link href="/setup" style={{ color: '#3498DB', textDecoration: 'none' }}>Details</Link>
|
|
71
|
+
<button
|
|
72
|
+
onClick={async () => { setStopping(true); try { await stopScrape(); } catch { setStopping(false); } }}
|
|
73
|
+
disabled={stopping}
|
|
74
|
+
title="Stops after saving what's been read; Resume on the Scan page carries on from the same page"
|
|
75
|
+
style={{
|
|
76
|
+
padding: '4px 10px', borderRadius: 8, fontSize: 12, cursor: stopping ? 'default' : 'pointer',
|
|
77
|
+
background: 'rgba(255,107,107,0.12)', color: '#ff9b9b', border: '1px solid rgba(255,107,107,0.35)',
|
|
78
|
+
}}
|
|
79
|
+
>{stopping ? 'Stopping…' : 'Stop'}</button>
|
|
80
|
+
</div>
|
|
81
|
+
);
|
|
82
|
+
}
|
|
@@ -38,3 +38,7 @@ html:has([data-field-question]) {
|
|
|
38
38
|
background: rgba(255, 255, 255, 0.15);
|
|
39
39
|
border-radius: 3px;
|
|
40
40
|
}
|
|
41
|
+
|
|
42
|
+
/* Where the scan status bar sits: just under the header (app/components/ScanStatusBar.js). */
|
|
43
|
+
:root { --scan-bar-top: 94px; }
|
|
44
|
+
@media (max-width: 767px) { :root { --scan-bar-top: 66px; } }
|
|
@@ -19,6 +19,7 @@ export const metadata = {
|
|
|
19
19
|
|
|
20
20
|
import UserProvider from './components/UserProvider';
|
|
21
21
|
import StaleServerBanner from './components/StaleServerBanner';
|
|
22
|
+
import ScanStatusBar from './components/ScanStatusBar';
|
|
22
23
|
|
|
23
24
|
export default function RootLayout({ children }) {
|
|
24
25
|
return (
|
|
@@ -30,6 +31,8 @@ export default function RootLayout({ children }) {
|
|
|
30
31
|
is running. Every page, because the answer to "did my update
|
|
31
32
|
work?" should not require finding the right screen. */}
|
|
32
33
|
<StaleServerBanner />
|
|
34
|
+
{/* While a scan runs: what it's doing and today's budget, with Stop. */}
|
|
35
|
+
<ScanStatusBar />
|
|
33
36
|
</UserProvider>
|
|
34
37
|
</body>
|
|
35
38
|
</html>
|
|
@@ -73,6 +73,8 @@ function HomeInner() {
|
|
|
73
73
|
const [degree3, setDegree3] = useState([]);
|
|
74
74
|
// Which degrees Network Circle draws: your connections alone until you pick more.
|
|
75
75
|
const [degrees, setDegrees] = useState([1]);
|
|
76
|
+
// Tiers switched off in Network Circle's Filter panel (C and D make a lot of noise).
|
|
77
|
+
const [hiddenTiers, setHiddenTiers] = useState([]);
|
|
76
78
|
const [selected, setSelected] = useState(null);
|
|
77
79
|
const focusNodeRef = useRef(null);
|
|
78
80
|
const [filter, setFilter] = useState('all');
|
|
@@ -170,10 +172,10 @@ function HomeInner() {
|
|
|
170
172
|
() => (isDegreesMode ? connections : degrees.flatMap((d) => byDegree[d] || [])),
|
|
171
173
|
[isDegreesMode, connections, degrees, byDegree],
|
|
172
174
|
);
|
|
173
|
-
const filtered = useMemo(
|
|
174
|
-
()
|
|
175
|
-
|
|
176
|
-
);
|
|
175
|
+
const filtered = useMemo(() => {
|
|
176
|
+
if (!isDegreesMode) return hiddenTiers.length ? networkRows.filter(c => !hiddenTiers.includes(c.tier)) : networkRows;
|
|
177
|
+
return filter === 'all' ? networkRows : networkRows.filter(c => c.tier === filter);
|
|
178
|
+
}, [isDegreesMode, networkRows, filter, hiddenTiers]);
|
|
177
179
|
// Lookups built once per load. Every click re-renders this component, and the
|
|
178
180
|
// three places below used to scan one list inside another — degree1.find per
|
|
179
181
|
// 2nd-degree row, degree2.some per connection — about 16M comparisons, twice
|
|
@@ -438,6 +440,8 @@ function HomeInner() {
|
|
|
438
440
|
tierCounts={isDegreesMode ? stats?.tiers || {} : tierCountsOf(networkRows)}
|
|
439
441
|
degrees={degrees}
|
|
440
442
|
onDegreesChange={setDegrees}
|
|
443
|
+
hiddenTiers={hiddenTiers}
|
|
444
|
+
onHiddenTiersChange={setHiddenTiers}
|
|
441
445
|
degreeCounts={{ 1: byDegree[1].length, 2: byDegree[2].length, 3: byDegree[3].length }}
|
|
442
446
|
bridgeTierCounts={bridgeTierCounts}
|
|
443
447
|
/>
|
|
@@ -65,6 +65,8 @@ function SetupInner() {
|
|
|
65
65
|
const [pages, setPages] = useRemembered('six-degrees-bridge-pages-v2', 100);
|
|
66
66
|
// Also finish people mapped before, from the page each one's read stopped at.
|
|
67
67
|
const [finish, setFinish] = useRemembered('six-degrees-bridge-finish', true);
|
|
68
|
+
// Experimental Auto-Bridge: all-day pacing and LinkedIn's own data (scripts/scrape.py --experimental).
|
|
69
|
+
const [experimental, setExperimental] = useRemembered('six-degrees-experimental-auto', false);
|
|
68
70
|
const [error, setError] = useState(null);
|
|
69
71
|
const logRef = useRef(null);
|
|
70
72
|
// What's saved, for the question about your field: undefined while it
|
|
@@ -440,6 +442,16 @@ function SetupInner() {
|
|
|
440
442
|
<input type="checkbox" checked={finish} onChange={(e) => setFinish(e.target.checked)} disabled={running} />
|
|
441
443
|
Also finish people already mapped, from the page each one stopped at
|
|
442
444
|
</label>
|
|
445
|
+
<label style={{ display: 'flex', gap: 8, alignItems: 'flex-start', fontSize: 12.5, color: '#b8c4c4', cursor: running ? 'default' : 'pointer' }}>
|
|
446
|
+
<input type="checkbox" checked={experimental} disabled={running} style={{ marginTop: 3 }}
|
|
447
|
+
onChange={(e) => setExperimental(e.target.checked)} />
|
|
448
|
+
<span>
|
|
449
|
+
<b style={{ color: '#FFD700' }}>Experimental:</b> all-day pacing. Up to 10 pages in a sitting,
|
|
450
|
+
then a 45-minute rest; searches only from 9:00 to 19:00; at the daily budget it waits for it to
|
|
451
|
+
free up instead of stopping; and it saves every page. It also reads LinkedIn’s own data beside
|
|
452
|
+
the page text, to fill gaps and measure how the two compare. It keeps running while the app is open.
|
|
453
|
+
</span>
|
|
454
|
+
</label>
|
|
443
455
|
<div style={{ fontSize: 12, color: '#FFD700', lineHeight: 1.6 }}>
|
|
444
456
|
Every page is a LinkedIn search, so it rests 20 seconds before each one and a
|
|
445
457
|
minute after every 10, and a long list can take
|
|
@@ -449,7 +461,7 @@ function SetupInner() {
|
|
|
449
461
|
what it read and stops, and carries on from that page next time. Keep batches small.
|
|
450
462
|
</div>
|
|
451
463
|
<div style={{ display: 'flex', gap: 10, flexWrap: 'wrap', alignItems: 'center' }}>
|
|
452
|
-
<Btn onClick={() => run('auto-bridge', { maxBridges: batch, tiers: order === 'score' ? tiers : [], order, maxPages: pages, deeper: finish })} disabled={!canSearch || (order === 'score' && !tiers.length)} primary>
|
|
464
|
+
<Btn onClick={() => run('auto-bridge', { maxBridges: batch, tiers: order === 'score' ? tiers : [], order, maxPages: pages, deeper: finish, experimental })} disabled={!canSearch || (order === 'score' && !tiers.length)} primary>
|
|
453
465
|
{running && s.action === 'auto-bridge' ? 'Mapping…' : 'Map 2nd degree'}
|
|
454
466
|
</Btn>
|
|
455
467
|
<select
|
|
@@ -148,6 +148,7 @@ function take(s) {
|
|
|
148
148
|
target: running ? s.target ?? null : starting ? now.target : null,
|
|
149
149
|
startedAt: running ? s.startedAt ?? null : null,
|
|
150
150
|
progress: running ? s.progress ?? null : null,
|
|
151
|
+
budget: running ? s.budget ?? null : null,
|
|
151
152
|
log: running && Array.isArray(s.log) ? s.log : [],
|
|
152
153
|
finished,
|
|
153
154
|
});
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "six-degrees",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.3",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "six-degrees",
|
|
9
|
-
"version": "0.4.
|
|
9
|
+
"version": "0.4.3",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"os": [
|
|
12
12
|
"darwin",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "six-degrees",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.3",
|
|
4
4
|
"description": "Map your professional network as a galaxy — leverage tiers, bridges into new circles, and the shortest path to anyone. Runs locally on your own data.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Blake Burford",
|
|
@@ -27,11 +27,12 @@ in one session — LinkedIn detects automation and flags your account.
|
|
|
27
27
|
import argparse
|
|
28
28
|
import json
|
|
29
29
|
import os
|
|
30
|
+
import re
|
|
30
31
|
import signal
|
|
31
32
|
import sys
|
|
32
33
|
import time
|
|
33
34
|
import requests
|
|
34
|
-
from datetime import datetime
|
|
35
|
+
from datetime import datetime, timedelta
|
|
35
36
|
from pathlib import Path
|
|
36
37
|
from urllib.parse import parse_qs, urlparse
|
|
37
38
|
|
|
@@ -1274,6 +1275,17 @@ PAGE_PAUSE = 20 # seconds before each next page of results
|
|
|
1274
1275
|
CHUNK_COOLDOWN = 60 # extra seconds after every SAVE_EVERY_PAGES pages
|
|
1275
1276
|
LEGACY_PAGES_READ = 10 # how far every read before 0.1.6 went, at most
|
|
1276
1277
|
|
|
1278
|
+
# Experimental all-day pacing for Auto-Bridge (--experimental; item 44, Graph
|
|
1279
|
+
# Study §8). Openly slow, never disguised: fixed waits, no random "human" timing.
|
|
1280
|
+
# A sitting of SESSION_PAGES searches, then a long rest; searches only in the
|
|
1281
|
+
# daytime on this computer's clock; and at the daily budget it waits for the
|
|
1282
|
+
# budget to free up instead of stopping. Pages are saved one at a time.
|
|
1283
|
+
EXPERIMENT = {"on": False, "pages": 0, "wire": 0}
|
|
1284
|
+
SESSION_PAGES = 10 # searches in one sitting
|
|
1285
|
+
SESSION_REST = 45 * 60 # the rest after each sitting
|
|
1286
|
+
DRIP_HOURS = (9, 19) # searches only from 09:00 to 19:00, local time
|
|
1287
|
+
WIRE_SAMPLES = 5 # raw LinkedIn responses kept per run, for research
|
|
1288
|
+
|
|
1277
1289
|
|
|
1278
1290
|
# Someone whose connections are hidden can never produce a 2nd-degree row. The
|
|
1279
1291
|
# "who still needs bridging" query is "everyone with no 2nd-degree rows", so
|
|
@@ -1787,6 +1799,193 @@ def stop_requested():
|
|
|
1787
1799
|
return _stop_requested
|
|
1788
1800
|
|
|
1789
1801
|
|
|
1802
|
+
# LinkedIn's own data (experimental, with --experimental). While a circle is
|
|
1803
|
+
# read, the responses the browser already gets from LinkedIn's own API are kept
|
|
1804
|
+
# and read for the same people the page shows: the same traffic, no extra
|
|
1805
|
+
# requests. The page text stays in charge. This fills blanks for people both
|
|
1806
|
+
# found and gives the list's real total; people only in LinkedIn's data are
|
|
1807
|
+
# logged, not added, since a results page can carry other boxes ("People you
|
|
1808
|
+
# may know"). Each page logs how the two compare, and a few raw responses are
|
|
1809
|
+
# kept in the data folder (wire-samples/) to tune it from real ones.
|
|
1810
|
+
_WIRE = {"page": None, "responses": []}
|
|
1811
|
+
|
|
1812
|
+
|
|
1813
|
+
def _wire_tap(page):
|
|
1814
|
+
if _WIRE["page"] is page:
|
|
1815
|
+
return
|
|
1816
|
+
_WIRE["page"], _WIRE["responses"] = page, []
|
|
1817
|
+
|
|
1818
|
+
def keep(response):
|
|
1819
|
+
url = response.url
|
|
1820
|
+
if "/voyager/api/" in url and ("search" in url.lower() or "graphql" in url):
|
|
1821
|
+
_WIRE["responses"].append(response)
|
|
1822
|
+
page.on("response", keep)
|
|
1823
|
+
|
|
1824
|
+
|
|
1825
|
+
def mutual_count_text(text):
|
|
1826
|
+
"""LinkedIn's mutual line as a number, as the page reader's mutualCount reads it; None otherwise."""
|
|
1827
|
+
line = re.sub(r"\s+", " ", str(text or "")).strip()
|
|
1828
|
+
if not re.search(r"\bmutual connections?$", line, re.I) or len(line) > 300:
|
|
1829
|
+
return None
|
|
1830
|
+
num = lambda v: int(str(v).replace(",", ""))
|
|
1831
|
+
names = lambda v: len([p for p in re.split(r"\s*,\s*|\s+and\s+", v) if p])
|
|
1832
|
+
m = re.match(r"^([\d,]+) mutual connections?$", line, re.I)
|
|
1833
|
+
if m:
|
|
1834
|
+
return num(m.group(1))
|
|
1835
|
+
m = re.match(r"^(.*?)\s*\band ([\d,]+) others? mutual connections?$", line, re.I)
|
|
1836
|
+
if m:
|
|
1837
|
+
return max(1, names(m.group(1))) + num(m.group(2))
|
|
1838
|
+
if re.search(r"\bis a mutual connection$", line, re.I):
|
|
1839
|
+
return 1
|
|
1840
|
+
m = re.match(r"^(.+?)\s+are mutual connections$", line, re.I)
|
|
1841
|
+
return names(m.group(1)) if m else None
|
|
1842
|
+
|
|
1843
|
+
|
|
1844
|
+
def voyager_people(payload):
|
|
1845
|
+
"""Everyone in a LinkedIn API response who looks like a search result: anything with a
|
|
1846
|
+
title and a link to a profile, wherever it sits in the response."""
|
|
1847
|
+
out, seen = [], set()
|
|
1848
|
+
text = lambda v: (v.get("text") if isinstance(v, dict) else v if isinstance(v, str) else "") or ""
|
|
1849
|
+
|
|
1850
|
+
def photo(x):
|
|
1851
|
+
try:
|
|
1852
|
+
for a in (x.get("image") or {}).get("attributes") or []:
|
|
1853
|
+
vi = ((a.get("detailData") or {}).get("nonEntityProfilePicture") or {}).get("vectorImage") or a.get("vectorImage")
|
|
1854
|
+
if vi and vi.get("rootUrl") and vi.get("artifacts"):
|
|
1855
|
+
best = max(vi["artifacts"], key=lambda f: f.get("width") or 0)
|
|
1856
|
+
return vi["rootUrl"] + (best.get("fileIdentifyingUrlPathSegment") or "")
|
|
1857
|
+
except Exception:
|
|
1858
|
+
pass
|
|
1859
|
+
return ""
|
|
1860
|
+
|
|
1861
|
+
def walk(x):
|
|
1862
|
+
if isinstance(x, dict):
|
|
1863
|
+
nav = x.get("navigationUrl")
|
|
1864
|
+
if isinstance(nav, str) and "/in/" in nav and isinstance(x.get("title"), dict):
|
|
1865
|
+
url = nav.split("?")[0].split("#")[0].rstrip("/") + "/"
|
|
1866
|
+
name = text(x.get("title")).strip()
|
|
1867
|
+
if name and name != "LinkedIn Member" and url not in seen:
|
|
1868
|
+
seen.add(url)
|
|
1869
|
+
mutual = None
|
|
1870
|
+
for ins in x.get("insightsResolutionResults") or []:
|
|
1871
|
+
mutual = mutual_count_text(text(((ins or {}).get("simpleInsight") or {}).get("title")))
|
|
1872
|
+
if mutual:
|
|
1873
|
+
break
|
|
1874
|
+
out.append({"name": name, "headline": text(x.get("primarySubtitle")).strip(),
|
|
1875
|
+
"profileUrl": url, "imageUrl": photo(x), "mutualCount": mutual})
|
|
1876
|
+
for v in x.values():
|
|
1877
|
+
walk(v)
|
|
1878
|
+
elif isinstance(x, list):
|
|
1879
|
+
for v in x:
|
|
1880
|
+
walk(v)
|
|
1881
|
+
walk(payload)
|
|
1882
|
+
return out
|
|
1883
|
+
|
|
1884
|
+
|
|
1885
|
+
def voyager_total(payload):
|
|
1886
|
+
"""The list's length, when a response says it (totalResultCount); None otherwise."""
|
|
1887
|
+
stack = [payload]
|
|
1888
|
+
while stack:
|
|
1889
|
+
x = stack.pop()
|
|
1890
|
+
if isinstance(x, dict):
|
|
1891
|
+
v = x.get("totalResultCount")
|
|
1892
|
+
if isinstance(v, int) and v > 0:
|
|
1893
|
+
return v
|
|
1894
|
+
stack.extend(x.values())
|
|
1895
|
+
elif isinstance(x, list):
|
|
1896
|
+
stack.extend(x)
|
|
1897
|
+
return None
|
|
1898
|
+
|
|
1899
|
+
|
|
1900
|
+
def _wire_sample(data):
|
|
1901
|
+
if EXPERIMENT["wire"] >= WIRE_SAMPLES:
|
|
1902
|
+
return
|
|
1903
|
+
try:
|
|
1904
|
+
raw = json.dumps(data)
|
|
1905
|
+
if len(raw) > 500_000:
|
|
1906
|
+
return
|
|
1907
|
+
folder = _home() / "wire-samples"
|
|
1908
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
1909
|
+
(folder / f"{int(time.time() * 1000)}.json").write_text(raw)
|
|
1910
|
+
EXPERIMENT["wire"] += 1
|
|
1911
|
+
except Exception:
|
|
1912
|
+
pass
|
|
1913
|
+
|
|
1914
|
+
|
|
1915
|
+
def _wire_merge(page, profile_url, page_results):
|
|
1916
|
+
"""Fill blanks in this page's people from LinkedIn's own data, and log how the two compare."""
|
|
1917
|
+
got, _WIRE["responses"] = _WIRE["responses"], []
|
|
1918
|
+
people, total = [], None
|
|
1919
|
+
for response in got:
|
|
1920
|
+
try:
|
|
1921
|
+
data = response.json()
|
|
1922
|
+
except Exception:
|
|
1923
|
+
continue
|
|
1924
|
+
_wire_sample(data)
|
|
1925
|
+
people.extend(voyager_people(data))
|
|
1926
|
+
total = total or voyager_total(data)
|
|
1927
|
+
if total:
|
|
1928
|
+
_LIST_TOTALS[profile_url] = int(total)
|
|
1929
|
+
own = (profile_url or "").rstrip("/").split("/")[-1]
|
|
1930
|
+
by_url = {c.get("profileUrl"): c for c in page_results}
|
|
1931
|
+
only_wire = 0
|
|
1932
|
+
for w in people:
|
|
1933
|
+
if own and own in w["profileUrl"]:
|
|
1934
|
+
continue
|
|
1935
|
+
c = by_url.get(w["profileUrl"])
|
|
1936
|
+
if c is None:
|
|
1937
|
+
only_wire += 1
|
|
1938
|
+
continue
|
|
1939
|
+
for k in ("name", "headline", "imageUrl"):
|
|
1940
|
+
if not c.get(k) and w.get(k):
|
|
1941
|
+
c[k] = w[k]
|
|
1942
|
+
if c.get("mutualCount") is None and w.get("mutualCount") is not None:
|
|
1943
|
+
c["mutualCount"] = w["mutualCount"]
|
|
1944
|
+
print(f" LinkedIn's own data: {len(people)} people, {only_wire} not in the page text"
|
|
1945
|
+
+ (f", list of about {total:,}" if total else ""), flush=True)
|
|
1946
|
+
return page_results
|
|
1947
|
+
|
|
1948
|
+
|
|
1949
|
+
def _drip_wait(label, seconds):
|
|
1950
|
+
"""Wait `seconds` for an experimental rest, saying when it ends, a line a minute. False if stopped."""
|
|
1951
|
+
seconds = max(1, int(seconds))
|
|
1952
|
+
until = datetime.fromtimestamp(time.time() + seconds).strftime("%H:%M")
|
|
1953
|
+
print(f" {label}. Carrying on at {until}.", flush=True)
|
|
1954
|
+
return interruptible_sleep(seconds, on_tick=lambda left: print(f" {label}: {max(1, left // 60)} min to go", flush=True), step=60)
|
|
1955
|
+
|
|
1956
|
+
|
|
1957
|
+
def _seconds_until_search_frees(now=None):
|
|
1958
|
+
"""How long until the oldest search in the last 24 hours drops out of the daily budget."""
|
|
1959
|
+
now = now if now is not None else time.time()
|
|
1960
|
+
recent = sorted(t for t in _read_activity()["searches"] if now - DAY_SECONDS < t <= now)
|
|
1961
|
+
return (recent[0] + DAY_SECONDS - now + 5) if recent else 5
|
|
1962
|
+
|
|
1963
|
+
|
|
1964
|
+
def _drip_before_search():
|
|
1965
|
+
"""Experimental pacing, before a search: rest after a sitting, sleep through the
|
|
1966
|
+
night, and wait for a used daily budget to free up. True to go ahead, False if stopped."""
|
|
1967
|
+
while True:
|
|
1968
|
+
if EXPERIMENT["pages"] >= SESSION_PAGES:
|
|
1969
|
+
EXPERIMENT["pages"] = 0
|
|
1970
|
+
if not _drip_wait(f"Resting after {SESSION_PAGES} pages", SESSION_REST):
|
|
1971
|
+
return False
|
|
1972
|
+
continue
|
|
1973
|
+
now = datetime.fromtimestamp(time.time())
|
|
1974
|
+
if not (DRIP_HOURS[0] <= now.hour < DRIP_HOURS[1]):
|
|
1975
|
+
start = now.replace(hour=DRIP_HOURS[0], minute=0, second=0, microsecond=0)
|
|
1976
|
+
if now.hour >= DRIP_HOURS[1]:
|
|
1977
|
+
start += timedelta(days=1)
|
|
1978
|
+
if not _drip_wait("Resting overnight", (start - now).total_seconds()):
|
|
1979
|
+
return False
|
|
1980
|
+
continue
|
|
1981
|
+
left, kind = searches_left()
|
|
1982
|
+
if left <= 0 and kind == "daily":
|
|
1983
|
+
if not _drip_wait("Today's searches are used", _seconds_until_search_frees()):
|
|
1984
|
+
return False
|
|
1985
|
+
continue
|
|
1986
|
+
return True
|
|
1987
|
+
|
|
1988
|
+
|
|
1790
1989
|
def interruptible_sleep(seconds, on_tick=None, step=5):
|
|
1791
1990
|
"""Sleep, but notice a stop request. Returns False if we were interrupted."""
|
|
1792
1991
|
waited = 0
|
|
@@ -2402,6 +2601,12 @@ def _scrape_one_bridge(page, bridge_name, bridge_id, profile_url, max_pages=LINK
|
|
|
2402
2601
|
# 1st, 2nd and 3rd+ — all of them; the app drops your own connections when
|
|
2403
2602
|
# it saves. It used to be announced as a "3rd+ filter", which it never was.
|
|
2404
2603
|
search_url = f"https://www.linkedin.com/search/results/people/?network=%5B%22F%22%2C%22S%22%2C%22O%22%5D&connectionOf=%5B%22{urn}%22%5D"
|
|
2604
|
+
if EXPERIMENT["on"] and not _drip_before_search():
|
|
2605
|
+
reach["more"] = True
|
|
2606
|
+
print(" Stopped before their list opened; the next run carries on from there.")
|
|
2607
|
+
return [], "stopped", reach
|
|
2608
|
+
if EXPERIMENT["on"]:
|
|
2609
|
+
_wire_tap(page)
|
|
2405
2610
|
left, kind = searches_left()
|
|
2406
2611
|
if left <= 0:
|
|
2407
2612
|
reach["budget"] = kind
|
|
@@ -2547,6 +2752,9 @@ def _scrape_one_bridge(page, bridge_name, bridge_id, profile_url, max_pages=LINK
|
|
|
2547
2752
|
break
|
|
2548
2753
|
|
|
2549
2754
|
page_results = [c for c in page_results if slug not in c.get("profileUrl", "")]
|
|
2755
|
+
if EXPERIMENT["on"]:
|
|
2756
|
+
page_results = _wire_merge(page, profile_url, page_results)
|
|
2757
|
+
EXPERIMENT["pages"] += 1
|
|
2550
2758
|
chunk.extend(page_results)
|
|
2551
2759
|
chunk_pages += 1
|
|
2552
2760
|
reach["last"] = pg
|
|
@@ -2569,7 +2777,7 @@ def _scrape_one_bridge(page, bridge_name, bridge_id, profile_url, max_pages=LINK
|
|
|
2569
2777
|
# own path: through the page-limit branch, a missing Next button would
|
|
2570
2778
|
# have marked the list finished.
|
|
2571
2779
|
left, kind = searches_left()
|
|
2572
|
-
if left <= 0:
|
|
2780
|
+
if left <= 0 and not (EXPERIMENT["on"] and kind == "daily"):
|
|
2573
2781
|
# Looking for Next costs nothing. No Next on a live page of results
|
|
2574
2782
|
# means the list is finished — not paused at the next page forever.
|
|
2575
2783
|
if _find_next(page) is None and not _window_closed(page) and _result_links(page):
|
|
@@ -2582,10 +2790,10 @@ def _scrape_one_bridge(page, bridge_name, bridge_id, profile_url, max_pages=LINK
|
|
|
2582
2790
|
|
|
2583
2791
|
# Rest before the next search, and longer after every SAVE_EVERY_PAGES.
|
|
2584
2792
|
rest = PAGE_PAUSE
|
|
2585
|
-
if (pg - start_page + 1) % SAVE_EVERY_PAGES == 0:
|
|
2793
|
+
if not EXPERIMENT["on"] and (pg - start_page + 1) % SAVE_EVERY_PAGES == 0:
|
|
2586
2794
|
rest += CHUNK_COOLDOWN
|
|
2587
2795
|
print(f" {SAVE_EVERY_PAGES} pages read; resting {rest}s before the next search.")
|
|
2588
|
-
if not interruptible_sleep(rest) or _window_closed(page):
|
|
2796
|
+
if not interruptible_sleep(rest) or _window_closed(page) or (EXPERIMENT["on"] and not _drip_before_search()):
|
|
2589
2797
|
reach["more"] = True
|
|
2590
2798
|
print(f" Stopped before page {pg + 1}; the next run carries on from there.")
|
|
2591
2799
|
break
|
|
@@ -2610,7 +2818,7 @@ def _scrape_one_bridge(page, bridge_name, bridge_id, profile_url, max_pages=LINK
|
|
|
2610
2818
|
|
|
2611
2819
|
# Save as it goes, so a stop, a crash or the search limit costs at most
|
|
2612
2820
|
# the last few pages rather than the whole list.
|
|
2613
|
-
if on_save and chunk_pages >= SAVE_EVERY_PAGES:
|
|
2821
|
+
if on_save and chunk_pages >= (1 if EXPERIMENT["on"] else SAVE_EVERY_PAGES):
|
|
2614
2822
|
on_save(_dedupe(chunk), reach["last"], True, reach["urn"])
|
|
2615
2823
|
chunk, chunk_pages = [], 0
|
|
2616
2824
|
|
|
@@ -3752,6 +3960,9 @@ Examples:
|
|
|
3752
3960
|
parser.add_argument("--bridge", type=str, help="Name of one bridge person to scrape")
|
|
3753
3961
|
parser.add_argument("--rescrape", type=str, help="Delete + re-scrape a bridge's cluster from scratch")
|
|
3754
3962
|
parser.add_argument("--company", type=str, help="Scan everyone the app can see at one company")
|
|
3963
|
+
parser.add_argument("--experimental", action="store_true",
|
|
3964
|
+
help="Auto-Bridge, experimental: all-day pacing (sittings, rests, daytime only, "
|
|
3965
|
+
"waits for the daily budget) and reading LinkedIn's own data beside the page text")
|
|
3755
3966
|
parser.add_argument("--auto-bridge", action="store_true",
|
|
3756
3967
|
help="Map every bridge in turn, highest tier first")
|
|
3757
3968
|
parser.add_argument("--retry-private", action="store_true",
|
|
@@ -3783,6 +3994,9 @@ Examples:
|
|
|
3783
3994
|
"(every scan also does this at its end)")
|
|
3784
3995
|
parser.add_argument("--headless", action="store_true", help="Run browser in headless mode")
|
|
3785
3996
|
args = parser.parse_args()
|
|
3997
|
+
if getattr(args, "experimental", False):
|
|
3998
|
+
EXPERIMENT["on"] = True
|
|
3999
|
+
print("Experimental Auto-Bridge: all-day pacing, and LinkedIn's own data read beside the page text.")
|
|
3786
4000
|
args.max_pages = max(1, min(LINKEDIN_MAX_PAGES, args.max_pages))
|
|
3787
4001
|
|
|
3788
4002
|
# A failed save ends the run with its reason, not a traceback (TRAPS §32).
|