salesprompter-cli 0.1.42 → 0.1.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -0
- package/dist/cli.js +3267 -110
- package/dist/linkedin-session.js +103 -17
- package/dist/sales-navigator.js +42 -15
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { spawn } from "node:child_process";
|
|
3
3
|
import { access, appendFile, mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
|
4
|
+
import { createServer } from "node:http";
|
|
4
5
|
import { createRequire } from "node:module";
|
|
5
6
|
import os from "node:os";
|
|
6
7
|
import path from "node:path";
|
|
@@ -26,10 +27,10 @@ import { InstantlySyncProvider } from "./instantly.js";
|
|
|
26
27
|
import { backfillLinkedInCompanies } from "./linkedin-companies.js";
|
|
27
28
|
import { parseLinkedInCompanyPage } from "./linkedin-companies.js";
|
|
28
29
|
import { crawlLinkedInProductCategory } from "./linkedin-products.js";
|
|
29
|
-
import { claimValidatedSalesNavigatorSessionCookieForCli, createLinkedInSessionSupabaseClient, resolveConfiguredEnvValue } from "./linkedin-session.js";
|
|
30
|
+
import { claimLinkedInSessionCookieForCli, claimValidatedSalesNavigatorSessionCookieForCli, createLinkedInSessionSupabaseClient, recordLinkedInSessionCookieAudit, resolveConfiguredEnvValue } from "./linkedin-session.js";
|
|
30
31
|
import { buildLeadlistsFunnelQueries } from "./leadlists-funnel.js";
|
|
31
32
|
import { readJsonFile, splitCsv, writeJsonFile, writeTextFile } from "./io.js";
|
|
32
|
-
import { buildSalesNavigatorCrawlPreview, createSalesNavigatorCrawlSeed, DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS, DEFAULT_SALES_NAVIGATOR_CRAWL_DIMENSIONS, buildSalesNavigatorPeopleSearchUrl, buildSalesNavigatorPeopleSlice, deriveSalesNavigatorTitleQuerySeeds, executeSalesNavigatorAdaptiveCrawl, expandSalesNavigatorCrawlAttempt, SalesNavigatorCrawlStopError, SalesNavigatorSliceTooBroadError } from "./sales-navigator.js";
|
|
33
|
+
import { buildSalesNavigatorCrawlPreview, buildSalesNavigatorAccountQuery, createSalesNavigatorCrawlSeed, DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS, DEFAULT_SALES_NAVIGATOR_CRAWL_DIMENSIONS, buildSalesNavigatorPeopleSearchUrl, buildSalesNavigatorPeopleSlice, deriveSalesNavigatorTitleQuerySeeds, executeSalesNavigatorAdaptiveCrawl, expandSalesNavigatorCrawlAttempt, SalesNavigatorCrawlStopError, SalesNavigatorSliceTooBroadError } from "./sales-navigator.js";
|
|
33
34
|
import { buildSalesNavigatorHistoricalBackfillPlan, ensureSalesNavigatorPeopleCount, resolveSalesNavigatorHistoricalBackfillConfig, resolveSalesNavigatorHistoricalBackfillResumeState, resolveSalesNavigatorHistoricalBackfillOrgId, salesNavigatorHistoricalBackfillDefaults } from "./salesnav-backfill.js";
|
|
34
35
|
const require = createRequire(import.meta.url);
|
|
35
36
|
const { version: packageVersion } = require("../package.json");
|
|
@@ -123,6 +124,84 @@ const LinkedInCompanyBackfillStatusResponseSchema = z.object({
|
|
|
123
124
|
failureCode: z.string().nullable().optional(),
|
|
124
125
|
failureMessage: z.string().nullable().optional()
|
|
125
126
|
});
|
|
127
|
+
const AffiliateCampaignLaunchResponseSchema = z.object({
|
|
128
|
+
status: z.enum(["ok", "partial"]),
|
|
129
|
+
dryRun: z.boolean(),
|
|
130
|
+
product: z.object({
|
|
131
|
+
domain: z.string().min(1),
|
|
132
|
+
affiliateLink: z.string().url()
|
|
133
|
+
}),
|
|
134
|
+
audience: z.object({
|
|
135
|
+
sourceType: z.enum(["sales_navigator_people", "linkedin_content"]),
|
|
136
|
+
linkedInUrl: z.string().url(),
|
|
137
|
+
maxResults: z.number().int().positive(),
|
|
138
|
+
peoplePreview: z.array(z.record(z.string(), z.unknown())).optional(),
|
|
139
|
+
postsPreview: z.array(z.record(z.string(), z.unknown())).optional()
|
|
140
|
+
}),
|
|
141
|
+
extraction: z
|
|
142
|
+
.object({
|
|
143
|
+
provider: z.string().min(1),
|
|
144
|
+
monitorId: z.string().min(1).optional(),
|
|
145
|
+
agentId: z.string().min(1).optional(),
|
|
146
|
+
containerId: z.string().min(1).optional()
|
|
147
|
+
})
|
|
148
|
+
.passthrough()
|
|
149
|
+
.optional(),
|
|
150
|
+
previewUrl: z.string().nullable().optional(),
|
|
151
|
+
next: z.string().optional()
|
|
152
|
+
});
|
|
153
|
+
const AffiliateCampaignLocalResultSchema = z
|
|
154
|
+
.object({
|
|
155
|
+
profileUrl: z.string().min(1),
|
|
156
|
+
})
|
|
157
|
+
.passthrough();
|
|
158
|
+
const DatabaseTimestampSchema = z.string().min(1);
|
|
159
|
+
const AffiliateOutreachRunSchema = z.object({
|
|
160
|
+
id: z.string().uuid(),
|
|
161
|
+
audience_run_id: z.string().uuid(),
|
|
162
|
+
organization_id: z.string().min(1),
|
|
163
|
+
status: z.enum(["preparing", "ready", "active", "failed"]),
|
|
164
|
+
provider: z.literal("instantly"),
|
|
165
|
+
instantly_campaign_id: z.string().min(1).nullable(),
|
|
166
|
+
instantly_campaign_name: z.string().min(1).nullable(),
|
|
167
|
+
sequence: z.array(z.object({
|
|
168
|
+
step: z.number().int().positive(),
|
|
169
|
+
subject: z.string(),
|
|
170
|
+
body: z.string(),
|
|
171
|
+
rationale: z.string(),
|
|
172
|
+
delayDays: z.number().int().nonnegative()
|
|
173
|
+
})),
|
|
174
|
+
enriched_leads: z.array(z.object({
|
|
175
|
+
rowIndex: z.number().int().nonnegative(),
|
|
176
|
+
fullName: z.string().min(1),
|
|
177
|
+
firstName: z.string().min(1),
|
|
178
|
+
lastName: z.string().min(1),
|
|
179
|
+
jobTitle: z.string().nullable(),
|
|
180
|
+
companyName: z.string().min(1),
|
|
181
|
+
companyDomain: z.string().min(1),
|
|
182
|
+
location: z.string().nullable(),
|
|
183
|
+
profileUrl: z.string().nullable(),
|
|
184
|
+
email: z.string().email(),
|
|
185
|
+
emailScore: z.number().min(0).max(100),
|
|
186
|
+
acceptAll: z.boolean(),
|
|
187
|
+
emailSource: z.enum(["hunter", "phantombuster"]).optional()
|
|
188
|
+
})),
|
|
189
|
+
stats: z.record(z.string(), z.unknown()),
|
|
190
|
+
settings: z.record(z.string(), z.unknown()),
|
|
191
|
+
error_message: z.string().nullable(),
|
|
192
|
+
activated_at: DatabaseTimestampSchema.nullable(),
|
|
193
|
+
created_at: DatabaseTimestampSchema,
|
|
194
|
+
updated_at: DatabaseTimestampSchema
|
|
195
|
+
});
|
|
196
|
+
const AffiliateOutreachResponseSchema = z.object({
|
|
197
|
+
status: z.literal("ok"),
|
|
198
|
+
outreach: AffiliateOutreachRunSchema
|
|
199
|
+
});
|
|
200
|
+
const AffiliateOutreachEnrichStartResponseSchema = z.object({
|
|
201
|
+
status: z.literal("accepted"),
|
|
202
|
+
audienceRunId: z.string().uuid(),
|
|
203
|
+
workflowRunId: z.string().min(1)
|
|
204
|
+
});
|
|
126
205
|
const SalesNavigatorLocalImportResponseSchema = z.object({
|
|
127
206
|
status: z.literal("ok"),
|
|
128
207
|
runId: z.string().min(1),
|
|
@@ -398,7 +477,15 @@ const cliPacks = [
|
|
|
398
477
|
slug: "outreach",
|
|
399
478
|
title: "Outreach",
|
|
400
479
|
summary: "Prepare and sync qualified leads into downstream systems.",
|
|
401
|
-
commands: [
|
|
480
|
+
commands: [
|
|
481
|
+
"affiliate:run",
|
|
482
|
+
"affiliate:status",
|
|
483
|
+
"affiliate:enrich",
|
|
484
|
+
"affiliate:activate",
|
|
485
|
+
"affiliate:launch",
|
|
486
|
+
"sync:outreach",
|
|
487
|
+
"sync:crm"
|
|
488
|
+
],
|
|
402
489
|
installStatus: "included"
|
|
403
490
|
}
|
|
404
491
|
];
|
|
@@ -407,6 +494,8 @@ const helpAliasByCommandName = new Map([
|
|
|
407
494
|
["companies:find-linkedin-urls", "companies:resolve-linkedin-urls"],
|
|
408
495
|
["contacts:process-emails", "contacts:resolve-emails"],
|
|
409
496
|
["linkedin-companies:backfill", "companies:enrich"],
|
|
497
|
+
["linkedin-companies:scrape-local", "companies:scrape-linkedin"],
|
|
498
|
+
["dealroom-companies:scrape-local", "companies:scrape-dealroom"],
|
|
410
499
|
["linkedin-products:scrape", "market:scrape"],
|
|
411
500
|
["salesnav:from-product-category", "leads:discover"],
|
|
412
501
|
["salesnav:crawl", "search:run"],
|
|
@@ -441,9 +530,16 @@ const helpVisibleCommandNames = new Set([
|
|
|
441
530
|
"leads:enrich",
|
|
442
531
|
"leads:score",
|
|
443
532
|
"leads:pipeline",
|
|
533
|
+
"affiliate:run",
|
|
534
|
+
"affiliate:status",
|
|
535
|
+
"affiliate:enrich",
|
|
536
|
+
"affiliate:activate",
|
|
537
|
+
"affiliate:launch",
|
|
444
538
|
"sync:crm",
|
|
445
539
|
"sync:outreach",
|
|
446
540
|
"linkedin-companies:backfill",
|
|
541
|
+
"linkedin-companies:scrape-local",
|
|
542
|
+
"dealroom-companies:scrape-local",
|
|
447
543
|
"linkedin-products:scrape",
|
|
448
544
|
"salesnav:from-product-category",
|
|
449
545
|
"salesnav:crawl",
|
|
@@ -1296,9 +1392,74 @@ function normalizeLinkedInDirectLookupCookieHeader(cookie) {
|
|
|
1296
1392
|
}
|
|
1297
1393
|
return `li_at=${trimmed}`;
|
|
1298
1394
|
}
|
|
1395
|
+
function mergeLinkedInCookieHeader(cookieHeader, setCookieHeaders) {
|
|
1396
|
+
const cookies = new Map();
|
|
1397
|
+
for (const part of cookieHeader.split(";")) {
|
|
1398
|
+
const separatorIndex = part.indexOf("=");
|
|
1399
|
+
if (separatorIndex <= 0) {
|
|
1400
|
+
continue;
|
|
1401
|
+
}
|
|
1402
|
+
cookies.set(part.slice(0, separatorIndex).trim(), part.slice(separatorIndex + 1).trim());
|
|
1403
|
+
}
|
|
1404
|
+
for (const setCookieHeader of setCookieHeaders) {
|
|
1405
|
+
const firstPart = setCookieHeader.split(";", 1)[0] ?? "";
|
|
1406
|
+
const separatorIndex = firstPart.indexOf("=");
|
|
1407
|
+
if (separatorIndex <= 0) {
|
|
1408
|
+
continue;
|
|
1409
|
+
}
|
|
1410
|
+
cookies.set(firstPart.slice(0, separatorIndex).trim(), firstPart.slice(separatorIndex + 1).trim());
|
|
1411
|
+
}
|
|
1412
|
+
return [...cookies.entries()]
|
|
1413
|
+
.map(([name, value]) => `${name}=${value}`)
|
|
1414
|
+
.join("; ");
|
|
1415
|
+
}
|
|
1416
|
+
async function bootstrapLinkedInAccountSearchConfig(params) {
|
|
1417
|
+
const cookie = normalizeLinkedInDirectLookupCookieHeader(params.sessionCookie);
|
|
1418
|
+
const existingCsrfToken = params.csrfToken || deriveCsrfTokenFromCookie(cookie);
|
|
1419
|
+
if (existingCsrfToken) {
|
|
1420
|
+
return {
|
|
1421
|
+
csrfToken: existingCsrfToken,
|
|
1422
|
+
identity: params.identity,
|
|
1423
|
+
cookie,
|
|
1424
|
+
userAgent: params.userAgent,
|
|
1425
|
+
};
|
|
1426
|
+
}
|
|
1427
|
+
const response = await fetch(params.queryUrl, {
|
|
1428
|
+
redirect: "manual",
|
|
1429
|
+
headers: {
|
|
1430
|
+
accept: "text/html,application/xhtml+xml",
|
|
1431
|
+
cookie,
|
|
1432
|
+
"user-agent": params.userAgent,
|
|
1433
|
+
},
|
|
1434
|
+
});
|
|
1435
|
+
if (response.status === 429 || response.status === 999) {
|
|
1436
|
+
throw new Error("Sales Navigator is rate-limited while bootstrapping the Account Search API session.");
|
|
1437
|
+
}
|
|
1438
|
+
if (response.status >= 300 && response.status < 400) {
|
|
1439
|
+
await response.arrayBuffer();
|
|
1440
|
+
return null;
|
|
1441
|
+
}
|
|
1442
|
+
const headersWithSetCookie = response.headers;
|
|
1443
|
+
const setCookieHeaders = headersWithSetCookie.getSetCookie?.() ??
|
|
1444
|
+
(response.headers.get("set-cookie")
|
|
1445
|
+
? [response.headers.get("set-cookie")]
|
|
1446
|
+
: []);
|
|
1447
|
+
const refreshedCookie = mergeLinkedInCookieHeader(cookie, setCookieHeaders);
|
|
1448
|
+
await response.arrayBuffer();
|
|
1449
|
+
const csrfToken = deriveCsrfTokenFromCookie(refreshedCookie);
|
|
1450
|
+
if (!csrfToken) {
|
|
1451
|
+
return null;
|
|
1452
|
+
}
|
|
1453
|
+
return {
|
|
1454
|
+
csrfToken,
|
|
1455
|
+
identity: params.identity,
|
|
1456
|
+
cookie: refreshedCookie,
|
|
1457
|
+
userAgent: params.userAgent,
|
|
1458
|
+
};
|
|
1459
|
+
}
|
|
1299
1460
|
function parseLocalLinkedInExtensionTokenLog(content) {
|
|
1300
1461
|
const matches = [
|
|
1301
|
-
...content.matchAll(/\{"csrfToken":"([^"]+)","extractedFrom":"sales-api\/
|
|
1462
|
+
...content.matchAll(/\{"csrfToken":"([^"]+)","extractedFrom":"sales-api\/salesApi(?:Lead|Account)Search"[\s\S]*?"linkedInIdentity":"([^"]+)"[\s\S]*?"sessionCookie":"([\s\S]*?)","syncStatus":"(success|captured)"[\s\S]*?"userAgent":"([^"]+)"\}/g)
|
|
1302
1463
|
];
|
|
1303
1464
|
const last = matches.at(-1);
|
|
1304
1465
|
if (!last) {
|
|
@@ -1327,6 +1488,21 @@ async function readLocalLinkedInExtensionTokenLog(filePath) {
|
|
|
1327
1488
|
return null;
|
|
1328
1489
|
}
|
|
1329
1490
|
}
|
|
1491
|
+
async function sortPathsByMostRecentlyModified(paths) {
|
|
1492
|
+
const candidates = await Promise.all(paths.map(async (filePath) => {
|
|
1493
|
+
try {
|
|
1494
|
+
const fileStat = await stat(filePath);
|
|
1495
|
+
return { filePath, modifiedAtMs: fileStat.mtimeMs };
|
|
1496
|
+
}
|
|
1497
|
+
catch {
|
|
1498
|
+
return { filePath, modifiedAtMs: 0 };
|
|
1499
|
+
}
|
|
1500
|
+
}));
|
|
1501
|
+
return candidates
|
|
1502
|
+
.sort((left, right) => right.modifiedAtMs - left.modifiedAtMs ||
|
|
1503
|
+
right.filePath.localeCompare(left.filePath))
|
|
1504
|
+
.map((candidate) => candidate.filePath);
|
|
1505
|
+
}
|
|
1330
1506
|
async function listChromeExtensionTokenLogCandidates() {
|
|
1331
1507
|
const overrideFile = normalizeLookupWhitespace(process.env.SALESPROMPTER_LINKEDIN_EXTENSION_TOKENS_LOG_PATH);
|
|
1332
1508
|
if (overrideFile) {
|
|
@@ -1336,11 +1512,9 @@ async function listChromeExtensionTokenLogCandidates() {
|
|
|
1336
1512
|
if (overrideDir) {
|
|
1337
1513
|
try {
|
|
1338
1514
|
const files = await readdir(overrideDir);
|
|
1339
|
-
return files
|
|
1515
|
+
return sortPathsByMostRecentlyModified(files
|
|
1340
1516
|
.filter((file) => file.endsWith(".log") || file.endsWith(".ldb"))
|
|
1341
|
-
.map((file) => path.join(overrideDir, file))
|
|
1342
|
-
.sort()
|
|
1343
|
-
.reverse();
|
|
1517
|
+
.map((file) => path.join(overrideDir, file)));
|
|
1344
1518
|
}
|
|
1345
1519
|
catch {
|
|
1346
1520
|
return [];
|
|
@@ -1378,7 +1552,7 @@ async function listChromeExtensionTokenLogCandidates() {
|
|
|
1378
1552
|
continue;
|
|
1379
1553
|
}
|
|
1380
1554
|
for (const file of files) {
|
|
1381
|
-
if (!file.endsWith(".log")) {
|
|
1555
|
+
if (!file.endsWith(".log") && !file.endsWith(".ldb")) {
|
|
1382
1556
|
continue;
|
|
1383
1557
|
}
|
|
1384
1558
|
paths.push(path.join(extensionDir, file));
|
|
@@ -1386,7 +1560,7 @@ async function listChromeExtensionTokenLogCandidates() {
|
|
|
1386
1560
|
}
|
|
1387
1561
|
}
|
|
1388
1562
|
}
|
|
1389
|
-
return paths
|
|
1563
|
+
return sortPathsByMostRecentlyModified(paths);
|
|
1390
1564
|
}
|
|
1391
1565
|
async function readLocalLinkedInExtensionDirectLookupConfig() {
|
|
1392
1566
|
const candidates = await listChromeExtensionTokenLogCandidates();
|
|
@@ -1460,11 +1634,12 @@ async function readStoredLinkedInDirectLookupConfig() {
|
|
|
1460
1634
|
return null;
|
|
1461
1635
|
}
|
|
1462
1636
|
const lookupResult = await supabase
|
|
1463
|
-
.from("
|
|
1637
|
+
.from("salesprompter_linkedin_session_cookies")
|
|
1464
1638
|
.select("metadata")
|
|
1465
1639
|
.eq("session_cookie_sha256", claimed.sessionCookieSha256)
|
|
1466
1640
|
.maybeSingle();
|
|
1467
|
-
if (lookupResult.error &&
|
|
1641
|
+
if (lookupResult.error &&
|
|
1642
|
+
!/salesprompter_linkedin_session_cookies/.test(lookupResult.error.message)) {
|
|
1468
1643
|
throw new Error(`Failed to read stored LinkedIn session metadata: ${lookupResult.error.message}`);
|
|
1469
1644
|
}
|
|
1470
1645
|
const metadata = lookupResult.data?.metadata;
|
|
@@ -1484,7 +1659,195 @@ async function readStoredLinkedInDirectLookupConfig() {
|
|
|
1484
1659
|
userAgent
|
|
1485
1660
|
};
|
|
1486
1661
|
}
|
|
1662
|
+
async function readStoredLinkedInAccountSearchConfig(queryUrl) {
|
|
1663
|
+
const supabase = createLinkedInSessionSupabaseClient();
|
|
1664
|
+
if (!supabase) {
|
|
1665
|
+
return null;
|
|
1666
|
+
}
|
|
1667
|
+
let lastFailure = "No Account Search API-capable LinkedIn session was available.";
|
|
1668
|
+
const configuredAttempts = Number(process.env.SALESPROMPTER_ACCOUNT_SEARCH_SESSION_ATTEMPTS);
|
|
1669
|
+
const maxAttempts = Number.isFinite(configuredAttempts) && configuredAttempts > 0
|
|
1670
|
+
? Math.min(64, Math.trunc(configuredAttempts))
|
|
1671
|
+
: 24;
|
|
1672
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
1673
|
+
const claimed = await claimLinkedInSessionCookieForCli({
|
|
1674
|
+
source: "cli_account_search",
|
|
1675
|
+
supabase,
|
|
1676
|
+
});
|
|
1677
|
+
if (!claimed) {
|
|
1678
|
+
return null;
|
|
1679
|
+
}
|
|
1680
|
+
const lookupResult = await supabase
|
|
1681
|
+
.from("salesprompter_linkedin_session_cookies")
|
|
1682
|
+
.select("metadata")
|
|
1683
|
+
.eq("session_cookie_sha256", claimed.sessionCookieSha256)
|
|
1684
|
+
.maybeSingle();
|
|
1685
|
+
if (lookupResult.error) {
|
|
1686
|
+
throw new Error(`Failed to read stored LinkedIn Account Search metadata: ${lookupResult.error.message}`);
|
|
1687
|
+
}
|
|
1688
|
+
const metadata = lookupResult.data?.metadata;
|
|
1689
|
+
const identity = extractStoredLinkedInLookupValue(metadata, "linkedInIdentity");
|
|
1690
|
+
const csrfToken = extractStoredLinkedInLookupValue(metadata, "csrfToken") ||
|
|
1691
|
+
deriveCsrfTokenFromCookie(claimed.sessionCookie);
|
|
1692
|
+
const userAgent = process.env.SALESPROMPTER_LINKEDIN_USER_AGENT?.trim() ||
|
|
1693
|
+
extractStoredLinkedInLookupValue(metadata, "userAgent") ||
|
|
1694
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36";
|
|
1695
|
+
let config = null;
|
|
1696
|
+
try {
|
|
1697
|
+
config = await bootstrapLinkedInAccountSearchConfig({
|
|
1698
|
+
queryUrl,
|
|
1699
|
+
sessionCookie: claimed.sessionCookie,
|
|
1700
|
+
identity,
|
|
1701
|
+
csrfToken,
|
|
1702
|
+
userAgent,
|
|
1703
|
+
});
|
|
1704
|
+
}
|
|
1705
|
+
catch (error) {
|
|
1706
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1707
|
+
if (/rate-limit/i.test(message)) {
|
|
1708
|
+
throw error;
|
|
1709
|
+
}
|
|
1710
|
+
lastFailure = message;
|
|
1711
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1712
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1713
|
+
source: "cli_account_search_api_preflight",
|
|
1714
|
+
feedStatus: null,
|
|
1715
|
+
salesNavigatorStatus: "error",
|
|
1716
|
+
recruiterStatus: null,
|
|
1717
|
+
productClass: "sales_navigator",
|
|
1718
|
+
isActive: true,
|
|
1719
|
+
inactiveReason: null,
|
|
1720
|
+
validationError: message,
|
|
1721
|
+
details: {
|
|
1722
|
+
attemptNumber: attempt,
|
|
1723
|
+
capability: "account_search_api_bootstrap",
|
|
1724
|
+
},
|
|
1725
|
+
});
|
|
1726
|
+
continue;
|
|
1727
|
+
}
|
|
1728
|
+
if (!config) {
|
|
1729
|
+
lastFailure =
|
|
1730
|
+
"Stored LinkedIn session could not bootstrap Account Search credentials.";
|
|
1731
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1732
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1733
|
+
source: "cli_account_search_api_preflight",
|
|
1734
|
+
feedStatus: null,
|
|
1735
|
+
salesNavigatorStatus: "error",
|
|
1736
|
+
recruiterStatus: null,
|
|
1737
|
+
productClass: "sales_navigator",
|
|
1738
|
+
isActive: false,
|
|
1739
|
+
inactiveReason: "salesnav_account_search_bootstrap_failed",
|
|
1740
|
+
validationError: "salesnav_account_search_bootstrap_failed",
|
|
1741
|
+
details: {
|
|
1742
|
+
attemptNumber: attempt,
|
|
1743
|
+
hasStoredIdentity: Boolean(identity),
|
|
1744
|
+
hasStoredCsrfToken: Boolean(csrfToken),
|
|
1745
|
+
},
|
|
1746
|
+
});
|
|
1747
|
+
continue;
|
|
1748
|
+
}
|
|
1749
|
+
const probe = await probeLinkedInAccountSearchConfig(queryUrl, config);
|
|
1750
|
+
if (probe.status === "ok") {
|
|
1751
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1752
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1753
|
+
source: "cli_account_search_api_preflight",
|
|
1754
|
+
feedStatus: null,
|
|
1755
|
+
salesNavigatorStatus: "ok",
|
|
1756
|
+
recruiterStatus: null,
|
|
1757
|
+
productClass: "sales_navigator",
|
|
1758
|
+
isActive: true,
|
|
1759
|
+
inactiveReason: null,
|
|
1760
|
+
validationError: null,
|
|
1761
|
+
details: {
|
|
1762
|
+
attemptNumber: attempt,
|
|
1763
|
+
statusCode: probe.statusCode,
|
|
1764
|
+
capability: "account_search_api",
|
|
1765
|
+
},
|
|
1766
|
+
});
|
|
1767
|
+
return config;
|
|
1768
|
+
}
|
|
1769
|
+
lastFailure =
|
|
1770
|
+
probe.validationError ??
|
|
1771
|
+
`Sales Navigator Account Search preflight returned ${probe.status}.`;
|
|
1772
|
+
if (probe.status === "rate_limited") {
|
|
1773
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1774
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1775
|
+
source: "cli_account_search_api_preflight",
|
|
1776
|
+
feedStatus: null,
|
|
1777
|
+
salesNavigatorStatus: "rate_limited",
|
|
1778
|
+
recruiterStatus: null,
|
|
1779
|
+
productClass: "sales_navigator",
|
|
1780
|
+
isActive: true,
|
|
1781
|
+
inactiveReason: null,
|
|
1782
|
+
validationError: probe.validationError,
|
|
1783
|
+
details: {
|
|
1784
|
+
attemptNumber: attempt,
|
|
1785
|
+
statusCode: probe.statusCode,
|
|
1786
|
+
capability: "account_search_api",
|
|
1787
|
+
},
|
|
1788
|
+
});
|
|
1789
|
+
throw new Error("Sales Navigator Account Search API is rate-limited. The CLI stopped without retrying this session.");
|
|
1790
|
+
}
|
|
1791
|
+
if (probe.status === "sales_seat_required") {
|
|
1792
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1793
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1794
|
+
source: "cli_account_search_api_preflight",
|
|
1795
|
+
feedStatus: null,
|
|
1796
|
+
salesNavigatorStatus: "upsell",
|
|
1797
|
+
recruiterStatus: null,
|
|
1798
|
+
productClass: null,
|
|
1799
|
+
isActive: false,
|
|
1800
|
+
inactiveReason: "salesnav_account_search_seat_required",
|
|
1801
|
+
validationError: probe.validationError,
|
|
1802
|
+
details: {
|
|
1803
|
+
attemptNumber: attempt,
|
|
1804
|
+
statusCode: probe.statusCode,
|
|
1805
|
+
capability: "account_search_api",
|
|
1806
|
+
},
|
|
1807
|
+
});
|
|
1808
|
+
continue;
|
|
1809
|
+
}
|
|
1810
|
+
if (probe.status === "unauthorized" || probe.status === "forbidden") {
|
|
1811
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1812
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1813
|
+
source: "cli_account_search_api_preflight",
|
|
1814
|
+
feedStatus: null,
|
|
1815
|
+
salesNavigatorStatus: "logged_out",
|
|
1816
|
+
recruiterStatus: null,
|
|
1817
|
+
productClass: "sales_navigator",
|
|
1818
|
+
isActive: false,
|
|
1819
|
+
inactiveReason: "salesnav_account_search_unauthorized",
|
|
1820
|
+
validationError: probe.validationError,
|
|
1821
|
+
details: {
|
|
1822
|
+
attemptNumber: attempt,
|
|
1823
|
+
statusCode: probe.statusCode,
|
|
1824
|
+
capability: "account_search_api",
|
|
1825
|
+
},
|
|
1826
|
+
});
|
|
1827
|
+
continue;
|
|
1828
|
+
}
|
|
1829
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1830
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1831
|
+
source: "cli_account_search_api_preflight",
|
|
1832
|
+
feedStatus: null,
|
|
1833
|
+
salesNavigatorStatus: "error",
|
|
1834
|
+
recruiterStatus: null,
|
|
1835
|
+
productClass: "sales_navigator",
|
|
1836
|
+
isActive: true,
|
|
1837
|
+
inactiveReason: null,
|
|
1838
|
+
validationError: probe.validationError,
|
|
1839
|
+
details: {
|
|
1840
|
+
attemptNumber: attempt,
|
|
1841
|
+
statusCode: probe.statusCode,
|
|
1842
|
+
capability: "account_search_api",
|
|
1843
|
+
probeStatus: probe.status,
|
|
1844
|
+
},
|
|
1845
|
+
});
|
|
1846
|
+
}
|
|
1847
|
+
throw new Error(lastFailure);
|
|
1848
|
+
}
|
|
1487
1849
|
let cachedLinkedInDirectLookupConfig = null;
|
|
1850
|
+
let cachedLinkedInAccountSearchConfig = null;
|
|
1488
1851
|
async function readLinkedInDirectLookupConfig() {
|
|
1489
1852
|
if (cachedLinkedInDirectLookupConfig) {
|
|
1490
1853
|
return cachedLinkedInDirectLookupConfig;
|
|
@@ -1515,6 +1878,46 @@ async function readLinkedInDirectLookupConfig() {
|
|
|
1515
1878
|
}
|
|
1516
1879
|
throw new Error("Missing LinkedIn direct lookup session. Set LINKEDIN_CSRF_TOKEN, LINKEDIN_X_LI_IDENTITY, and LINKEDIN_SALES_NAV_COOKIE, or connect the Salesprompter Chrome extension to sync your LinkedIn session to salesprompter.ai.");
|
|
1517
1880
|
}
|
|
1881
|
+
async function readLinkedInAccountSearchConfig(queryUrl) {
|
|
1882
|
+
if (cachedLinkedInAccountSearchConfig) {
|
|
1883
|
+
return cachedLinkedInAccountSearchConfig;
|
|
1884
|
+
}
|
|
1885
|
+
const envConfig = readLinkedInDirectLookupEnvConfig();
|
|
1886
|
+
if (envConfig) {
|
|
1887
|
+
const probe = await probeLinkedInAccountSearchConfig(queryUrl, envConfig);
|
|
1888
|
+
assertLinkedInAccountSearchConfigProbe(probe);
|
|
1889
|
+
cachedLinkedInAccountSearchConfig = envConfig;
|
|
1890
|
+
return envConfig;
|
|
1891
|
+
}
|
|
1892
|
+
let storedError = null;
|
|
1893
|
+
try {
|
|
1894
|
+
const storedConfig = await readStoredLinkedInAccountSearchConfig(queryUrl);
|
|
1895
|
+
if (storedConfig) {
|
|
1896
|
+
cachedLinkedInAccountSearchConfig = storedConfig;
|
|
1897
|
+
return storedConfig;
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
catch (error) {
|
|
1901
|
+
storedError = error;
|
|
1902
|
+
if (isExplicitLinkedInSessionVaultEnabled() ||
|
|
1903
|
+
/rate-limit/i.test(error instanceof Error ? error.message : String(error))) {
|
|
1904
|
+
throw error;
|
|
1905
|
+
}
|
|
1906
|
+
}
|
|
1907
|
+
if (!shouldDisableLinkedInDirectLookupAutodiscovery()) {
|
|
1908
|
+
const localExtensionConfig = await readLocalLinkedInExtensionDirectLookupConfig();
|
|
1909
|
+
if (localExtensionConfig) {
|
|
1910
|
+
const probe = await probeLinkedInAccountSearchConfig(queryUrl, localExtensionConfig);
|
|
1911
|
+
assertLinkedInAccountSearchConfigProbe(probe);
|
|
1912
|
+
cachedLinkedInAccountSearchConfig = localExtensionConfig;
|
|
1913
|
+
return localExtensionConfig;
|
|
1914
|
+
}
|
|
1915
|
+
}
|
|
1916
|
+
if (storedError) {
|
|
1917
|
+
throw storedError;
|
|
1918
|
+
}
|
|
1919
|
+
throw new Error("Missing LinkedIn Account Search API session. Connect the Salesprompter Chrome extension or configure the Salesprompter session vault.");
|
|
1920
|
+
}
|
|
1518
1921
|
function generateLinkedInSessionId() {
|
|
1519
1922
|
return Array.from({ length: 22 }, () => "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789".charAt(Math.floor(Math.random() * 62))).join("");
|
|
1520
1923
|
}
|
|
@@ -2194,6 +2597,203 @@ function normalizeLocalSalesNavigatorAccounts(responseBody, queryUrl) {
|
|
|
2194
2597
|
}
|
|
2195
2598
|
return accounts;
|
|
2196
2599
|
}
|
|
2600
|
+
async function readLocalSalesNavigatorRawAccountInput(params) {
|
|
2601
|
+
const filePath = path.resolve(params.filePath);
|
|
2602
|
+
const content = await readFile(filePath, "utf8");
|
|
2603
|
+
const accounts = [];
|
|
2604
|
+
const seenAccounts = new Set();
|
|
2605
|
+
const requestUrls = new Set();
|
|
2606
|
+
let parsedLines = 0;
|
|
2607
|
+
let parsedElements = 0;
|
|
2608
|
+
let invalidElements = 0;
|
|
2609
|
+
let duplicateAccounts = 0;
|
|
2610
|
+
let filteredAccounts = 0;
|
|
2611
|
+
const addElement = (element, requestUrl) => {
|
|
2612
|
+
parsedElements += 1;
|
|
2613
|
+
if (typeof element !== "object" || element === null || Array.isArray(element)) {
|
|
2614
|
+
invalidElements += 1;
|
|
2615
|
+
return;
|
|
2616
|
+
}
|
|
2617
|
+
const account = normalizeLocalSalesNavigatorAccount(element, requestUrl);
|
|
2618
|
+
if (!account) {
|
|
2619
|
+
invalidElements += 1;
|
|
2620
|
+
return;
|
|
2621
|
+
}
|
|
2622
|
+
if (seenAccounts.has(account.accountKey)) {
|
|
2623
|
+
duplicateAccounts += 1;
|
|
2624
|
+
return;
|
|
2625
|
+
}
|
|
2626
|
+
seenAccounts.add(account.accountKey);
|
|
2627
|
+
if (!salesNavigatorAccountMatchesLocationFilter(account, params.accountLocationFilter)) {
|
|
2628
|
+
filteredAccounts += 1;
|
|
2629
|
+
return;
|
|
2630
|
+
}
|
|
2631
|
+
accounts.push(account);
|
|
2632
|
+
};
|
|
2633
|
+
const lines = content.split(/\r?\n/);
|
|
2634
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
2635
|
+
const line = lines[index]?.trim() ?? "";
|
|
2636
|
+
if (!line) {
|
|
2637
|
+
continue;
|
|
2638
|
+
}
|
|
2639
|
+
parsedLines += 1;
|
|
2640
|
+
let value;
|
|
2641
|
+
try {
|
|
2642
|
+
value = JSON.parse(line);
|
|
2643
|
+
}
|
|
2644
|
+
catch (error) {
|
|
2645
|
+
throw new Error(`Raw Sales Navigator account input has invalid JSON on line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2646
|
+
}
|
|
2647
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
2648
|
+
invalidElements += 1;
|
|
2649
|
+
continue;
|
|
2650
|
+
}
|
|
2651
|
+
const record = value;
|
|
2652
|
+
const requestUrlCandidate = [
|
|
2653
|
+
record.queryUrl,
|
|
2654
|
+
record.requestUrl,
|
|
2655
|
+
record.slicedQueryUrl,
|
|
2656
|
+
].find((candidate) => typeof candidate === "string" && candidate.trim().length > 0);
|
|
2657
|
+
const requestUrl = requestUrlCandidate?.trim() ?? params.sourceQueryUrl;
|
|
2658
|
+
requestUrls.add(requestUrl);
|
|
2659
|
+
if (Array.isArray(record.elements)) {
|
|
2660
|
+
for (const element of record.elements) {
|
|
2661
|
+
addElement(element, requestUrl);
|
|
2662
|
+
}
|
|
2663
|
+
continue;
|
|
2664
|
+
}
|
|
2665
|
+
if ("element" in record) {
|
|
2666
|
+
addElement(record.element, requestUrl);
|
|
2667
|
+
continue;
|
|
2668
|
+
}
|
|
2669
|
+
if ("account" in record) {
|
|
2670
|
+
addElement(record.account, requestUrl);
|
|
2671
|
+
continue;
|
|
2672
|
+
}
|
|
2673
|
+
if (record.type === "metadata") {
|
|
2674
|
+
continue;
|
|
2675
|
+
}
|
|
2676
|
+
addElement(record, requestUrl);
|
|
2677
|
+
}
|
|
2678
|
+
const totalResults = params.reportedTotal ?? accounts.length;
|
|
2679
|
+
const missingFromReported = Math.max(0, totalResults - accounts.length);
|
|
2680
|
+
const complete = missingFromReported === 0;
|
|
2681
|
+
return {
|
|
2682
|
+
fetchResult: {
|
|
2683
|
+
accounts,
|
|
2684
|
+
filteredAccounts,
|
|
2685
|
+
totalResults,
|
|
2686
|
+
fetchedPages: requestUrls.size,
|
|
2687
|
+
sourceQueryUrl: params.sourceQueryUrl,
|
|
2688
|
+
pacing: {
|
|
2689
|
+
startOffset: 0,
|
|
2690
|
+
delayMinMs: 0,
|
|
2691
|
+
delayMaxMs: 0,
|
|
2692
|
+
totalDelayMs: 0,
|
|
2693
|
+
retryCount: 0,
|
|
2694
|
+
stop: {
|
|
2695
|
+
reason: complete
|
|
2696
|
+
? "reported_total_reached"
|
|
2697
|
+
: "raw_results_incomplete",
|
|
2698
|
+
startOffset: 0,
|
|
2699
|
+
pageSize: accounts.length,
|
|
2700
|
+
lastSuccessfulStart: accounts.length > 0 ? 0 : null,
|
|
2701
|
+
lastSuccessfulCount: accounts.length,
|
|
2702
|
+
lastSuccessfulAdded: accounts.length,
|
|
2703
|
+
stoppedAtStart: accounts.length,
|
|
2704
|
+
stoppedAtCount: 0,
|
|
2705
|
+
nextStart: accounts.length,
|
|
2706
|
+
fetchedAccounts: accounts.length,
|
|
2707
|
+
reportedTotal: totalResults,
|
|
2708
|
+
missingFromReported,
|
|
2709
|
+
},
|
|
2710
|
+
},
|
|
2711
|
+
},
|
|
2712
|
+
stats: {
|
|
2713
|
+
filePath,
|
|
2714
|
+
parsedLines,
|
|
2715
|
+
parsedElements,
|
|
2716
|
+
invalidElements,
|
|
2717
|
+
duplicateAccounts,
|
|
2718
|
+
distinctRequestUrls: requestUrls.size,
|
|
2719
|
+
},
|
|
2720
|
+
};
|
|
2721
|
+
}
|
|
2722
|
+
function decodeSalesNavigatorQueryTextValue(value) {
|
|
2723
|
+
let decoded = value.trim();
|
|
2724
|
+
for (let index = 0; index < 3; index += 1) {
|
|
2725
|
+
try {
|
|
2726
|
+
const next = decodeURIComponent(decoded);
|
|
2727
|
+
if (next === decoded) {
|
|
2728
|
+
break;
|
|
2729
|
+
}
|
|
2730
|
+
decoded = next;
|
|
2731
|
+
}
|
|
2732
|
+
catch {
|
|
2733
|
+
break;
|
|
2734
|
+
}
|
|
2735
|
+
}
|
|
2736
|
+
return decoded.replaceAll("+", " ").replace(/\s+/g, " ").trim();
|
|
2737
|
+
}
|
|
2738
|
+
function extractIncludedSalesNavigatorFilterTexts(searchUrl, filterType) {
|
|
2739
|
+
const decodedQuery = decodeSalesNavigatorQueryParam(searchUrl);
|
|
2740
|
+
if (!decodedQuery) {
|
|
2741
|
+
return [];
|
|
2742
|
+
}
|
|
2743
|
+
const values = [];
|
|
2744
|
+
const seen = new Set();
|
|
2745
|
+
const filterPattern = new RegExp(`type:${filterType},values:List\\((.*?)\\)\\)`, "g");
|
|
2746
|
+
for (const filterMatch of decodedQuery.matchAll(filterPattern)) {
|
|
2747
|
+
const block = filterMatch[1] ?? "";
|
|
2748
|
+
for (const valueMatch of block.matchAll(/text:([^,)]+),selectionType:INCLUDED/g)) {
|
|
2749
|
+
const text = decodeSalesNavigatorQueryTextValue(valueMatch[1] ?? "");
|
|
2750
|
+
const key = normalizeLooseMatchText(text);
|
|
2751
|
+
if (text && !seen.has(key)) {
|
|
2752
|
+
seen.add(key);
|
|
2753
|
+
values.push(text);
|
|
2754
|
+
}
|
|
2755
|
+
}
|
|
2756
|
+
}
|
|
2757
|
+
return values;
|
|
2758
|
+
}
|
|
2759
|
+
function buildSalesNavigatorAccountLocationFilter(searchUrl) {
|
|
2760
|
+
const regionTexts = extractIncludedSalesNavigatorFilterTexts(searchUrl, "REGION");
|
|
2761
|
+
const requiredTerms = regionTexts
|
|
2762
|
+
.flatMap((text) => normalizeLooseMatchText(text).split(" "))
|
|
2763
|
+
.filter((term) => term.length >= 3 &&
|
|
2764
|
+
![
|
|
2765
|
+
"area",
|
|
2766
|
+
"metro",
|
|
2767
|
+
"metropolitan",
|
|
2768
|
+
"region",
|
|
2769
|
+
"greater",
|
|
2770
|
+
"united",
|
|
2771
|
+
"states",
|
|
2772
|
+
"kingdom",
|
|
2773
|
+
"germany",
|
|
2774
|
+
"deutschland",
|
|
2775
|
+
"europe",
|
|
2776
|
+
].includes(term));
|
|
2777
|
+
if (regionTexts.length === 0 || requiredTerms.length === 0) {
|
|
2778
|
+
return null;
|
|
2779
|
+
}
|
|
2780
|
+
return {
|
|
2781
|
+
regionTexts,
|
|
2782
|
+
requiredTerms: [...new Set(requiredTerms)],
|
|
2783
|
+
};
|
|
2784
|
+
}
|
|
2785
|
+
function salesNavigatorAccountMatchesLocationFilter(account, filter) {
|
|
2786
|
+
if (!filter || filter.requiredTerms.length === 0) {
|
|
2787
|
+
return true;
|
|
2788
|
+
}
|
|
2789
|
+
const visibleLocation = typeof account.location === "string"
|
|
2790
|
+
? normalizeLooseMatchText(account.location)
|
|
2791
|
+
: "";
|
|
2792
|
+
if (!visibleLocation) {
|
|
2793
|
+
return true;
|
|
2794
|
+
}
|
|
2795
|
+
return filter.requiredTerms.some((term) => visibleLocation.includes(term));
|
|
2796
|
+
}
|
|
2197
2797
|
function buildLinkedInCompanyLookupVariants(params) {
|
|
2198
2798
|
const variants = [];
|
|
2199
2799
|
const seen = new Set();
|
|
@@ -5296,6 +5896,19 @@ function isMissingSupabaseTableError(code, message) {
|
|
|
5296
5896
|
const normalized = message.toLowerCase();
|
|
5297
5897
|
return code === "42P01" || normalized.includes("does not exist");
|
|
5298
5898
|
}
|
|
5899
|
+
function createSalesNavigatorSupabaseFetch(env = process.env) {
|
|
5900
|
+
const configuredTimeoutMs = Number(env.SALESPROMPTER_SALESNAV_SUPABASE_TIMEOUT_MS);
|
|
5901
|
+
const timeoutMs = Number.isFinite(configuredTimeoutMs) && configuredTimeoutMs >= 500
|
|
5902
|
+
? Math.min(30_000, Math.trunc(configuredTimeoutMs))
|
|
5903
|
+
: 2_500;
|
|
5904
|
+
return (input, init) => {
|
|
5905
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
5906
|
+
const signal = init?.signal
|
|
5907
|
+
? AbortSignal.any([init.signal, timeoutSignal])
|
|
5908
|
+
: timeoutSignal;
|
|
5909
|
+
return fetch(input, { ...init, signal });
|
|
5910
|
+
};
|
|
5911
|
+
}
|
|
5299
5912
|
async function createSalesNavigatorCrawlEventStore(options) {
|
|
5300
5913
|
const config = (() => {
|
|
5301
5914
|
try {
|
|
@@ -5313,7 +5926,10 @@ async function createSalesNavigatorCrawlEventStore(options) {
|
|
|
5313
5926
|
auth: {
|
|
5314
5927
|
persistSession: false,
|
|
5315
5928
|
autoRefreshToken: false
|
|
5316
|
-
}
|
|
5929
|
+
},
|
|
5930
|
+
global: {
|
|
5931
|
+
fetch: createSalesNavigatorSupabaseFetch(process.env),
|
|
5932
|
+
},
|
|
5317
5933
|
});
|
|
5318
5934
|
let writable = true;
|
|
5319
5935
|
return {
|
|
@@ -5337,6 +5953,7 @@ async function createSalesNavigatorCrawlEventStore(options) {
|
|
|
5337
5953
|
writable = false;
|
|
5338
5954
|
return;
|
|
5339
5955
|
}
|
|
5956
|
+
writable = false;
|
|
5340
5957
|
throw new Error(`Failed to write crawl event to Supabase: ${error.message}`);
|
|
5341
5958
|
},
|
|
5342
5959
|
listRecentForJob: async (jobId, limit) => {
|
|
@@ -5909,6 +6526,79 @@ async function fetchCliJson(session, request, schema) {
|
|
|
5909
6526
|
return schema.parse(parsed);
|
|
5910
6527
|
});
|
|
5911
6528
|
}
|
|
6529
|
+
async function launchAffiliateCampaignViaApp(session, payload) {
|
|
6530
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-campaigns`, {
|
|
6531
|
+
method: "POST",
|
|
6532
|
+
headers: {
|
|
6533
|
+
"Content-Type": "application/json",
|
|
6534
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6535
|
+
},
|
|
6536
|
+
body: JSON.stringify(payload)
|
|
6537
|
+
}), AffiliateCampaignLaunchResponseSchema);
|
|
6538
|
+
return value;
|
|
6539
|
+
}
|
|
6540
|
+
async function prepareAffiliateOutreachViaApp(session, payload) {
|
|
6541
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach`, {
|
|
6542
|
+
method: "POST",
|
|
6543
|
+
headers: {
|
|
6544
|
+
"Content-Type": "application/json",
|
|
6545
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6546
|
+
},
|
|
6547
|
+
body: JSON.stringify(payload)
|
|
6548
|
+
}), AffiliateOutreachResponseSchema);
|
|
6549
|
+
return value.outreach;
|
|
6550
|
+
}
|
|
6551
|
+
async function getAffiliateOutreachViaApp(session, audienceRunId) {
|
|
6552
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach/${encodeURIComponent(audienceRunId)}`, {
|
|
6553
|
+
headers: {
|
|
6554
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6555
|
+
}
|
|
6556
|
+
}), AffiliateOutreachResponseSchema);
|
|
6557
|
+
return value.outreach;
|
|
6558
|
+
}
|
|
6559
|
+
async function activateAffiliateOutreachViaApp(session, audienceRunId) {
|
|
6560
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach/${encodeURIComponent(audienceRunId)}/activate`, {
|
|
6561
|
+
method: "POST",
|
|
6562
|
+
headers: {
|
|
6563
|
+
"Content-Type": "application/json",
|
|
6564
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6565
|
+
},
|
|
6566
|
+
body: JSON.stringify({})
|
|
6567
|
+
}), AffiliateOutreachResponseSchema);
|
|
6568
|
+
return value.outreach;
|
|
6569
|
+
}
|
|
6570
|
+
async function enrichAffiliateOutreachViaApp(session, audienceRunId) {
|
|
6571
|
+
const before = await getAffiliateOutreachViaApp(session, audienceRunId);
|
|
6572
|
+
const previousAttemptedAt = typeof before.stats.latestTopUp === "object" && before.stats.latestTopUp !== null
|
|
6573
|
+
? String(before.stats.latestTopUp.attemptedAt ?? "")
|
|
6574
|
+
: "";
|
|
6575
|
+
await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach/${encodeURIComponent(audienceRunId)}/enrich`, {
|
|
6576
|
+
method: "POST",
|
|
6577
|
+
headers: {
|
|
6578
|
+
"Content-Type": "application/json",
|
|
6579
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6580
|
+
},
|
|
6581
|
+
body: JSON.stringify({})
|
|
6582
|
+
}), AffiliateOutreachEnrichStartResponseSchema);
|
|
6583
|
+
const deadline = Date.now() + 10 * 60 * 1000;
|
|
6584
|
+
const configuredPollMs = Number(process.env.SALESPROMPTER_AFFILIATE_ENRICH_POLL_MS);
|
|
6585
|
+
const pollMs = Number.isFinite(configuredPollMs) && configuredPollMs > 0
|
|
6586
|
+
? Math.max(10, Math.trunc(configuredPollMs))
|
|
6587
|
+
: 5_000;
|
|
6588
|
+
while (Date.now() < deadline) {
|
|
6589
|
+
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
|
6590
|
+
const current = await getAffiliateOutreachViaApp(session, audienceRunId);
|
|
6591
|
+
const latestTopUp = typeof current.stats.latestTopUp === "object" &&
|
|
6592
|
+
current.stats.latestTopUp !== null
|
|
6593
|
+
? current.stats.latestTopUp
|
|
6594
|
+
: null;
|
|
6595
|
+
const attemptedAt = String(latestTopUp?.attemptedAt ?? "");
|
|
6596
|
+
if (attemptedAt && attemptedAt !== previousAttemptedAt) {
|
|
6597
|
+
return current;
|
|
6598
|
+
}
|
|
6599
|
+
}
|
|
6600
|
+
throw new Error("Email recovery is still running after 10 minutes. Check affiliate:status and retry later.");
|
|
6601
|
+
}
|
|
5912
6602
|
async function uploadLinkedInProductsCatalog(session, payload, batchSize = 100, traceId) {
|
|
5913
6603
|
let imported = 0;
|
|
5914
6604
|
let upserted = 0;
|
|
@@ -6064,12 +6754,15 @@ function normalizeSalesNavigatorApiQueryValue(rawQuery) {
|
|
|
6064
6754
|
: withRecentSearchId.replace(/^\(/, "(spellCorrectionEnabled:true,");
|
|
6065
6755
|
}
|
|
6066
6756
|
function buildSalesNavigatorApiRequestFromSearchUrl(searchUrl, config, count) {
|
|
6757
|
+
return {
|
|
6758
|
+
url: buildSalesNavigatorLeadApiUrlFromSearchUrl(searchUrl, count),
|
|
6759
|
+
headers: buildLinkedInSalesNavigatorLocalImportHeaders(config),
|
|
6760
|
+
};
|
|
6761
|
+
}
|
|
6762
|
+
function buildSalesNavigatorLeadApiUrlFromSearchUrl(searchUrl, count) {
|
|
6067
6763
|
const source = new URL(searchUrl);
|
|
6068
6764
|
if (source.pathname.includes("/sales-api/salesApiLeadSearch")) {
|
|
6069
|
-
return
|
|
6070
|
-
url: source.toString(),
|
|
6071
|
-
headers: buildLinkedInSalesNavigatorLocalImportHeaders(config),
|
|
6072
|
-
};
|
|
6765
|
+
return source.toString();
|
|
6073
6766
|
}
|
|
6074
6767
|
if (!source.pathname.includes("/sales/search/people")) {
|
|
6075
6768
|
throw new Error("Automatic Sales Navigator import requires a /sales/search/people URL.");
|
|
@@ -6088,10 +6781,7 @@ function buildSalesNavigatorApiRequestFromSearchUrl(searchUrl, config, count) {
|
|
|
6088
6781
|
`?q=searchQuery&query=${apiQuery}&start=0&count=${normalizedCount}` +
|
|
6089
6782
|
`&trackingParam=(sessionId:${rawSessionId})` +
|
|
6090
6783
|
"&decorationId=com.linkedin.sales.deco.desktop.searchv2.LeadSearchResult-14";
|
|
6091
|
-
return
|
|
6092
|
-
url: apiUrl,
|
|
6093
|
-
headers: buildLinkedInSalesNavigatorLocalImportHeaders(config),
|
|
6094
|
-
};
|
|
6784
|
+
return apiUrl;
|
|
6095
6785
|
}
|
|
6096
6786
|
function extractSalesNavigatorDecorationId(requestUrl, fallbackDecorationId) {
|
|
6097
6787
|
try {
|
|
@@ -6118,15 +6808,12 @@ function buildSalesNavigatorAccountApiUrlFromSearchUrl(searchUrl, count, options
|
|
|
6118
6808
|
const baseUrl = compactOptionalText(options.baseUrl) ??
|
|
6119
6809
|
compactOptionalText(process.env.SALESPROMPTER_LINKEDIN_SALES_API_BASE_URL) ??
|
|
6120
6810
|
`${source.protocol}//${source.host}`;
|
|
6121
|
-
const
|
|
6122
|
-
encodeURIComponent(generateLinkedInSessionId());
|
|
6123
|
-
const apiQuery = normalizeSalesNavigatorApiQueryValue(rawQuery);
|
|
6811
|
+
const apiQuery = decodeRawSalesNavigatorQueryParam(rawQuery);
|
|
6124
6812
|
const normalizedCount = Math.max(1, Math.min(100, Math.trunc(count)));
|
|
6125
6813
|
const decorationId = compactOptionalText(options.decorationId) ??
|
|
6126
|
-
"com.linkedin.sales.deco.desktop.searchv2.AccountSearchResult-
|
|
6814
|
+
"com.linkedin.sales.deco.desktop.searchv2.AccountSearchResult-4";
|
|
6127
6815
|
return (`${baseUrl.replace(/\/+$/, "")}/sales-api/salesApiAccountSearch` +
|
|
6128
6816
|
`?q=searchQuery&query=${apiQuery}&start=0&count=${normalizedCount}` +
|
|
6129
|
-
`&trackingParam=(sessionId:${rawSessionId})` +
|
|
6130
6817
|
`&decorationId=${decorationId}`);
|
|
6131
6818
|
}
|
|
6132
6819
|
function buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, count) {
|
|
@@ -6142,26 +6829,136 @@ function buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, co
|
|
|
6142
6829
|
headers: buildLinkedInSalesNavigatorAccountSearchHeaders(config),
|
|
6143
6830
|
};
|
|
6144
6831
|
}
|
|
6145
|
-
function
|
|
6146
|
-
const
|
|
6147
|
-
|
|
6148
|
-
|
|
6149
|
-
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6155
|
-
|
|
6156
|
-
|
|
6157
|
-
|
|
6158
|
-
|
|
6159
|
-
|
|
6160
|
-
|
|
6161
|
-
|
|
6162
|
-
|
|
6163
|
-
|
|
6164
|
-
|
|
6832
|
+
async function probeLinkedInAccountSearchConfig(searchUrl, config) {
|
|
6833
|
+
const request = buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, 1);
|
|
6834
|
+
const controller = new AbortController();
|
|
6835
|
+
const timeout = setTimeout(() => controller.abort(), 8_000);
|
|
6836
|
+
try {
|
|
6837
|
+
const response = await fetch(request.url, {
|
|
6838
|
+
headers: request.headers,
|
|
6839
|
+
signal: controller.signal,
|
|
6840
|
+
});
|
|
6841
|
+
if (response.status === 401) {
|
|
6842
|
+
return {
|
|
6843
|
+
status: "unauthorized",
|
|
6844
|
+
statusCode: response.status,
|
|
6845
|
+
validationError: "salesnav_account_search_http_401",
|
|
6846
|
+
};
|
|
6847
|
+
}
|
|
6848
|
+
if (response.status === 403) {
|
|
6849
|
+
const body = await response.text();
|
|
6850
|
+
let errorCode = "";
|
|
6851
|
+
try {
|
|
6852
|
+
const parsed = JSON.parse(body);
|
|
6853
|
+
errorCode =
|
|
6854
|
+
typeof parsed.code === "string" ? parsed.code.trim() : "";
|
|
6855
|
+
}
|
|
6856
|
+
catch {
|
|
6857
|
+
errorCode = "";
|
|
6858
|
+
}
|
|
6859
|
+
if (errorCode === "SALES_SEAT_REQUIRED") {
|
|
6860
|
+
return {
|
|
6861
|
+
status: "sales_seat_required",
|
|
6862
|
+
statusCode: response.status,
|
|
6863
|
+
validationError: "salesnav_account_search_sales_seat_required",
|
|
6864
|
+
};
|
|
6865
|
+
}
|
|
6866
|
+
return {
|
|
6867
|
+
status: "forbidden",
|
|
6868
|
+
statusCode: response.status,
|
|
6869
|
+
validationError: "salesnav_account_search_http_403",
|
|
6870
|
+
};
|
|
6871
|
+
}
|
|
6872
|
+
if (response.status === 429 || response.status === 999) {
|
|
6873
|
+
return {
|
|
6874
|
+
status: "rate_limited",
|
|
6875
|
+
statusCode: response.status,
|
|
6876
|
+
validationError: "salesnav_account_search_rate_limited",
|
|
6877
|
+
};
|
|
6878
|
+
}
|
|
6879
|
+
if (!response.ok) {
|
|
6880
|
+
return {
|
|
6881
|
+
status: "error",
|
|
6882
|
+
statusCode: response.status,
|
|
6883
|
+
validationError: `salesnav_account_search_http_${response.status}`,
|
|
6884
|
+
};
|
|
6885
|
+
}
|
|
6886
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
6887
|
+
const body = await response.text();
|
|
6888
|
+
if (!/json/i.test(contentType) && /^\s*</.test(body)) {
|
|
6889
|
+
return {
|
|
6890
|
+
status: "invalid_response",
|
|
6891
|
+
statusCode: response.status,
|
|
6892
|
+
validationError: "salesnav_account_search_returned_html",
|
|
6893
|
+
};
|
|
6894
|
+
}
|
|
6895
|
+
try {
|
|
6896
|
+
const parsed = JSON.parse(body);
|
|
6897
|
+
if (typeof parsed !== "object" || parsed === null) {
|
|
6898
|
+
throw new Error("response is not an object");
|
|
6899
|
+
}
|
|
6900
|
+
}
|
|
6901
|
+
catch {
|
|
6902
|
+
return {
|
|
6903
|
+
status: "invalid_response",
|
|
6904
|
+
statusCode: response.status,
|
|
6905
|
+
validationError: "salesnav_account_search_invalid_json",
|
|
6906
|
+
};
|
|
6907
|
+
}
|
|
6908
|
+
return {
|
|
6909
|
+
status: "ok",
|
|
6910
|
+
statusCode: response.status,
|
|
6911
|
+
validationError: null,
|
|
6912
|
+
};
|
|
6913
|
+
}
|
|
6914
|
+
catch (error) {
|
|
6915
|
+
return {
|
|
6916
|
+
status: "error",
|
|
6917
|
+
statusCode: null,
|
|
6918
|
+
validationError: error instanceof Error && error.name === "AbortError"
|
|
6919
|
+
? "salesnav_account_search_timeout"
|
|
6920
|
+
: `salesnav_account_search_request_failed:${error instanceof Error ? error.message : String(error)}`,
|
|
6921
|
+
};
|
|
6922
|
+
}
|
|
6923
|
+
finally {
|
|
6924
|
+
clearTimeout(timeout);
|
|
6925
|
+
}
|
|
6926
|
+
}
|
|
6927
|
+
function assertLinkedInAccountSearchConfigProbe(probe) {
|
|
6928
|
+
if (probe.status === "ok") {
|
|
6929
|
+
return;
|
|
6930
|
+
}
|
|
6931
|
+
if (probe.status === "rate_limited") {
|
|
6932
|
+
throw new Error("Sales Navigator Account Search API is rate-limited. The CLI stopped without retrying the session.");
|
|
6933
|
+
}
|
|
6934
|
+
if (probe.status === "unauthorized" || probe.status === "forbidden") {
|
|
6935
|
+
throw new Error(`The configured LinkedIn session cannot call the Sales Navigator Account Search API (HTTP ${probe.statusCode}). Refresh the Salesprompter LinkedIn session before retrying.`);
|
|
6936
|
+
}
|
|
6937
|
+
if (probe.status === "sales_seat_required") {
|
|
6938
|
+
throw new Error("The configured LinkedIn session is logged in but does not have a Sales Navigator seat.");
|
|
6939
|
+
}
|
|
6940
|
+
throw new Error(`Sales Navigator Account Search API preflight failed: ${probe.validationError ?? probe.status}`);
|
|
6941
|
+
}
|
|
6942
|
+
function buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(searchUrl, templateRequest, count) {
|
|
6943
|
+
const templateUrl = new URL(templateRequest.url);
|
|
6944
|
+
return {
|
|
6945
|
+
url: buildSalesNavigatorAccountApiUrlFromSearchUrl(searchUrl, count, {
|
|
6946
|
+
baseUrl: `${templateUrl.protocol}//${templateUrl.host}`,
|
|
6947
|
+
decorationId: extractSalesNavigatorDecorationId(templateRequest.url, "com.linkedin.sales.deco.desktop.searchv2.AccountSearchResult-4"),
|
|
6948
|
+
}),
|
|
6949
|
+
headers: { ...templateRequest.headers },
|
|
6950
|
+
};
|
|
6951
|
+
}
|
|
6952
|
+
const SALES_NAVIGATOR_PROFILE_LANGUAGE_VALUES = new Map([
|
|
6953
|
+
["de", { id: "de", text: "German", selectionType: "INCLUDED" }],
|
|
6954
|
+
["german", { id: "de", text: "German", selectionType: "INCLUDED" }],
|
|
6955
|
+
["en", { id: "en", text: "English", selectionType: "INCLUDED" }],
|
|
6956
|
+
["english", { id: "en", text: "English", selectionType: "INCLUDED" }],
|
|
6957
|
+
]);
|
|
6958
|
+
const SALES_NAVIGATOR_SENIORITY_VALUES = new Map([
|
|
6959
|
+
[
|
|
6960
|
+
"owner / partner",
|
|
6961
|
+
{ id: "320", text: "Owner / Partner", selectionType: "INCLUDED" },
|
|
6165
6962
|
],
|
|
6166
6963
|
["owner", { id: "320", text: "Owner / Partner", selectionType: "INCLUDED" }],
|
|
6167
6964
|
[
|
|
@@ -6214,12 +7011,17 @@ const SALES_NAVIGATOR_COMPANY_HEADCOUNT_VALUES = new Map([
|
|
|
6214
7011
|
["11-50", { id: "C", text: "11-50", selectionType: "INCLUDED" }],
|
|
6215
7012
|
["51-200", { id: "D", text: "51-200", selectionType: "INCLUDED" }],
|
|
6216
7013
|
["201-500", { id: "E", text: "201-500", selectionType: "INCLUDED" }],
|
|
6217
|
-
["501-1000", { id: "F", text: "501-
|
|
6218
|
-
["
|
|
6219
|
-
["
|
|
6220
|
-
["
|
|
6221
|
-
["10,000
|
|
6222
|
-
["10000
|
|
7014
|
+
["501-1000", { id: "F", text: "501-1,000", selectionType: "INCLUDED" }],
|
|
7015
|
+
["501-1,000", { id: "F", text: "501-1,000", selectionType: "INCLUDED" }],
|
|
7016
|
+
["1001-5000", { id: "G", text: "1,001-5,000", selectionType: "INCLUDED" }],
|
|
7017
|
+
["1,001-5,000", { id: "G", text: "1,001-5,000", selectionType: "INCLUDED" }],
|
|
7018
|
+
["5001-10,000", { id: "H", text: "5,001-10,000", selectionType: "INCLUDED" }],
|
|
7019
|
+
["5001-10000", { id: "H", text: "5,001-10,000", selectionType: "INCLUDED" }],
|
|
7020
|
+
["5,001-10,000", { id: "H", text: "5,001-10,000", selectionType: "INCLUDED" }],
|
|
7021
|
+
["10,000+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
7022
|
+
["10000+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
7023
|
+
["10,001+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
7024
|
+
["10001+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
6223
7025
|
]);
|
|
6224
7026
|
const SALES_NAVIGATOR_TENURE_VALUES = new Map([
|
|
6225
7027
|
[
|
|
@@ -6443,14 +7245,17 @@ function withSalesNavigatorPaging(url, start, count) {
|
|
|
6443
7245
|
return `${base}?${params.join("&")}${hash}`;
|
|
6444
7246
|
}
|
|
6445
7247
|
function buildLinkedInSalesNavigatorLocalImportHeaders(config) {
|
|
6446
|
-
|
|
7248
|
+
const headers = {
|
|
6447
7249
|
accept: "*/*",
|
|
6448
7250
|
"csrf-token": config.csrfToken,
|
|
6449
7251
|
cookie: config.cookie,
|
|
6450
7252
|
"user-agent": config.userAgent,
|
|
6451
|
-
"x-li-identity": config.identity,
|
|
6452
7253
|
"x-restli-protocol-version": "2.0.0",
|
|
6453
7254
|
};
|
|
7255
|
+
if (config.identity) {
|
|
7256
|
+
headers["x-li-identity"] = config.identity;
|
|
7257
|
+
}
|
|
7258
|
+
return headers;
|
|
6454
7259
|
}
|
|
6455
7260
|
function buildLinkedInSalesNavigatorAccountSearchHeaders(config) {
|
|
6456
7261
|
return {
|
|
@@ -6741,10 +7546,18 @@ function extractLocalSalesNavigatorTotalResults(responseBody) {
|
|
|
6741
7546
|
return null;
|
|
6742
7547
|
}
|
|
6743
7548
|
const record = responseBody;
|
|
7549
|
+
const data = typeof record.data === "object" &&
|
|
7550
|
+
record.data !== null &&
|
|
7551
|
+
!Array.isArray(record.data)
|
|
7552
|
+
? record.data
|
|
7553
|
+
: null;
|
|
6744
7554
|
const candidates = [
|
|
6745
7555
|
record.paging,
|
|
7556
|
+
data?.paging,
|
|
6746
7557
|
record.metadata,
|
|
7558
|
+
data?.metadata,
|
|
6747
7559
|
record.searchResultsMetadata,
|
|
7560
|
+
data,
|
|
6748
7561
|
record,
|
|
6749
7562
|
];
|
|
6750
7563
|
for (const candidate of candidates) {
|
|
@@ -6846,6 +7659,781 @@ async function fetchLocalSalesNavigatorRequest(parsedRequest, retryOptions) {
|
|
|
6846
7659
|
}
|
|
6847
7660
|
}
|
|
6848
7661
|
}
|
|
7662
|
+
async function readLocalAccountSearchRelayBody(request) {
|
|
7663
|
+
const chunks = [];
|
|
7664
|
+
let size = 0;
|
|
7665
|
+
for await (const chunk of request) {
|
|
7666
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
7667
|
+
size += buffer.length;
|
|
7668
|
+
if (size > 50 * 1024 * 1024) {
|
|
7669
|
+
throw new Error("Browser relay response exceeded 50 MB.");
|
|
7670
|
+
}
|
|
7671
|
+
chunks.push(buffer);
|
|
7672
|
+
}
|
|
7673
|
+
const text = Buffer.concat(chunks).toString("utf8");
|
|
7674
|
+
return text ? JSON.parse(text) : null;
|
|
7675
|
+
}
|
|
7676
|
+
function writeLocalAccountSearchRelayJson(response, statusCode, payload) {
|
|
7677
|
+
response.writeHead(statusCode, {
|
|
7678
|
+
"content-type": "application/json; charset=utf-8",
|
|
7679
|
+
"cache-control": "no-store",
|
|
7680
|
+
"access-control-allow-origin": "https://www.linkedin.com",
|
|
7681
|
+
"access-control-allow-methods": "GET, POST, OPTIONS",
|
|
7682
|
+
"access-control-allow-headers": "content-type",
|
|
7683
|
+
"access-control-allow-private-network": "true",
|
|
7684
|
+
});
|
|
7685
|
+
response.end(`${JSON.stringify(payload)}\n`);
|
|
7686
|
+
}
|
|
7687
|
+
async function createLocalAccountSearchBrowserRelay(requestedPort) {
|
|
7688
|
+
let pending = null;
|
|
7689
|
+
let requestSequence = 0;
|
|
7690
|
+
const server = createServer(async (request, response) => {
|
|
7691
|
+
try {
|
|
7692
|
+
const requestUrl = new URL(request.url ?? "/", `http://127.0.0.1:${requestedPort}`);
|
|
7693
|
+
if (request.method === "OPTIONS") {
|
|
7694
|
+
writeLocalAccountSearchRelayJson(response, 204, null);
|
|
7695
|
+
return;
|
|
7696
|
+
}
|
|
7697
|
+
if (request.method === "GET" && requestUrl.pathname === "/health") {
|
|
7698
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7699
|
+
status: "ok",
|
|
7700
|
+
pending: Boolean(pending),
|
|
7701
|
+
});
|
|
7702
|
+
return;
|
|
7703
|
+
}
|
|
7704
|
+
if (request.method === "GET" && requestUrl.pathname === "/next") {
|
|
7705
|
+
writeLocalAccountSearchRelayJson(response, 200, pending
|
|
7706
|
+
? {
|
|
7707
|
+
status: "task",
|
|
7708
|
+
requestId: pending.requestId,
|
|
7709
|
+
url: pending.url,
|
|
7710
|
+
}
|
|
7711
|
+
: { status: "idle" });
|
|
7712
|
+
return;
|
|
7713
|
+
}
|
|
7714
|
+
if (request.method === "POST" && requestUrl.pathname === "/result") {
|
|
7715
|
+
if (!pending) {
|
|
7716
|
+
writeLocalAccountSearchRelayJson(response, 409, {
|
|
7717
|
+
status: "error",
|
|
7718
|
+
error: "No Account Search relay task is pending.",
|
|
7719
|
+
});
|
|
7720
|
+
return;
|
|
7721
|
+
}
|
|
7722
|
+
const value = await readLocalAccountSearchRelayBody(request);
|
|
7723
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
7724
|
+
throw new Error("Browser relay result must be a JSON object.");
|
|
7725
|
+
}
|
|
7726
|
+
const record = value;
|
|
7727
|
+
if (record.requestId !== pending.requestId) {
|
|
7728
|
+
writeLocalAccountSearchRelayJson(response, 409, {
|
|
7729
|
+
status: "error",
|
|
7730
|
+
error: "Browser relay request id does not match the pending task.",
|
|
7731
|
+
});
|
|
7732
|
+
return;
|
|
7733
|
+
}
|
|
7734
|
+
const status = Number(record.status);
|
|
7735
|
+
const retryAfter = typeof record.retryAfter === "string" ? record.retryAfter : null;
|
|
7736
|
+
const body = record.body;
|
|
7737
|
+
const bodyPreview = JSON.stringify(body ?? null).slice(0, 300);
|
|
7738
|
+
const current = pending;
|
|
7739
|
+
pending = null;
|
|
7740
|
+
if (isSalesNavigatorRateLimitStatus(status, bodyPreview)) {
|
|
7741
|
+
current.reject(new LocalSalesNavigatorRateLimitError(status, bodyPreview, parseRetryAfterHeader(retryAfter)));
|
|
7742
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7743
|
+
status: "rate_limited",
|
|
7744
|
+
});
|
|
7745
|
+
return;
|
|
7746
|
+
}
|
|
7747
|
+
if (!Number.isFinite(status) || status < 200 || status >= 300) {
|
|
7748
|
+
current.reject(new Error(`Sales Navigator browser relay failed with HTTP ${status}: ${bodyPreview}`));
|
|
7749
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7750
|
+
status: "failed",
|
|
7751
|
+
});
|
|
7752
|
+
return;
|
|
7753
|
+
}
|
|
7754
|
+
current.resolve({ body, retryCount: 0, retryDelayMs: 0 });
|
|
7755
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7756
|
+
status: "accepted",
|
|
7757
|
+
});
|
|
7758
|
+
return;
|
|
7759
|
+
}
|
|
7760
|
+
writeLocalAccountSearchRelayJson(response, 404, {
|
|
7761
|
+
status: "error",
|
|
7762
|
+
error: "Not found",
|
|
7763
|
+
});
|
|
7764
|
+
}
|
|
7765
|
+
catch (error) {
|
|
7766
|
+
writeLocalAccountSearchRelayJson(response, 500, {
|
|
7767
|
+
status: "error",
|
|
7768
|
+
error: error instanceof Error ? error.message : String(error),
|
|
7769
|
+
});
|
|
7770
|
+
}
|
|
7771
|
+
});
|
|
7772
|
+
await new Promise((resolve, reject) => {
|
|
7773
|
+
server.once("error", reject);
|
|
7774
|
+
server.listen(requestedPort, "127.0.0.1", () => resolve());
|
|
7775
|
+
});
|
|
7776
|
+
const address = server.address();
|
|
7777
|
+
if (!address || typeof address === "string") {
|
|
7778
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
7779
|
+
throw new Error("Account Search browser relay failed to bind.");
|
|
7780
|
+
}
|
|
7781
|
+
return {
|
|
7782
|
+
port: address.port,
|
|
7783
|
+
request(url) {
|
|
7784
|
+
if (pending) {
|
|
7785
|
+
throw new Error("Account Search browser relay already has a pending task.");
|
|
7786
|
+
}
|
|
7787
|
+
const parsedUrl = new URL(url);
|
|
7788
|
+
if (!/(^|\.)linkedin\.com$/i.test(parsedUrl.hostname) ||
|
|
7789
|
+
!(parsedUrl.pathname.includes("/sales-api/salesApiAccountSearch") ||
|
|
7790
|
+
parsedUrl.pathname.includes("/sales-api/salesApiLeadSearch"))) {
|
|
7791
|
+
throw new Error("Browser relay only accepts LinkedIn Account or Lead Search API URLs.");
|
|
7792
|
+
}
|
|
7793
|
+
requestSequence += 1;
|
|
7794
|
+
return new Promise((resolve, reject) => {
|
|
7795
|
+
pending = {
|
|
7796
|
+
requestId: `sales-nav-search-${requestSequence}`,
|
|
7797
|
+
url,
|
|
7798
|
+
resolve,
|
|
7799
|
+
reject,
|
|
7800
|
+
};
|
|
7801
|
+
});
|
|
7802
|
+
},
|
|
7803
|
+
async close() {
|
|
7804
|
+
if (pending) {
|
|
7805
|
+
pending.reject(new Error("Account Search browser relay closed."));
|
|
7806
|
+
pending = null;
|
|
7807
|
+
}
|
|
7808
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
7809
|
+
},
|
|
7810
|
+
};
|
|
7811
|
+
}
|
|
7812
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_RESULT_WINDOW = 1600;
|
|
7813
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION = 3;
|
|
7814
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS = "coverage-complete";
|
|
7815
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_PASSES = [
|
|
7816
|
+
{
|
|
7817
|
+
name: LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS,
|
|
7818
|
+
filterTypes: [
|
|
7819
|
+
"COMPANY_HEADCOUNT",
|
|
7820
|
+
"NUM_OF_FOLLOWERS",
|
|
7821
|
+
],
|
|
7822
|
+
},
|
|
7823
|
+
{
|
|
7824
|
+
name: "headcount-first",
|
|
7825
|
+
filterTypes: [
|
|
7826
|
+
"COMPANY_HEADCOUNT",
|
|
7827
|
+
"NUM_OF_FOLLOWERS",
|
|
7828
|
+
"ANNUAL_REVENUE",
|
|
7829
|
+
"COMPANY_HEADCOUNT_GROWTH",
|
|
7830
|
+
],
|
|
7831
|
+
},
|
|
7832
|
+
{
|
|
7833
|
+
name: "followers-first",
|
|
7834
|
+
filterTypes: [
|
|
7835
|
+
"NUM_OF_FOLLOWERS",
|
|
7836
|
+
"COMPANY_HEADCOUNT",
|
|
7837
|
+
"ANNUAL_REVENUE",
|
|
7838
|
+
"COMPANY_HEADCOUNT_GROWTH",
|
|
7839
|
+
],
|
|
7840
|
+
},
|
|
7841
|
+
{
|
|
7842
|
+
name: "revenue-first",
|
|
7843
|
+
filterTypes: [
|
|
7844
|
+
"ANNUAL_REVENUE",
|
|
7845
|
+
"COMPANY_HEADCOUNT",
|
|
7846
|
+
"NUM_OF_FOLLOWERS",
|
|
7847
|
+
"COMPANY_HEADCOUNT_GROWTH",
|
|
7848
|
+
],
|
|
7849
|
+
},
|
|
7850
|
+
];
|
|
7851
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_KEYWORD_TOKENS = [
|
|
7852
|
+
"software",
|
|
7853
|
+
"ai",
|
|
7854
|
+
"technology",
|
|
7855
|
+
"data",
|
|
7856
|
+
"platform",
|
|
7857
|
+
"services",
|
|
7858
|
+
"solutions",
|
|
7859
|
+
"digital",
|
|
7860
|
+
"cloud",
|
|
7861
|
+
"business",
|
|
7862
|
+
"internet",
|
|
7863
|
+
"online",
|
|
7864
|
+
"app",
|
|
7865
|
+
"systems",
|
|
7866
|
+
"company",
|
|
7867
|
+
"management",
|
|
7868
|
+
"development",
|
|
7869
|
+
"network",
|
|
7870
|
+
"security",
|
|
7871
|
+
"automation",
|
|
7872
|
+
"marketing",
|
|
7873
|
+
"financial",
|
|
7874
|
+
"media",
|
|
7875
|
+
"consulting",
|
|
7876
|
+
"health",
|
|
7877
|
+
"mobile",
|
|
7878
|
+
"enterprise",
|
|
7879
|
+
"analytics",
|
|
7880
|
+
"engineering",
|
|
7881
|
+
"global",
|
|
7882
|
+
"design",
|
|
7883
|
+
"web",
|
|
7884
|
+
];
|
|
7885
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_GROWTH_DIMENSION = {
|
|
7886
|
+
key: "company-headcount-growth",
|
|
7887
|
+
filterType: "COMPANY_HEADCOUNT_GROWTH",
|
|
7888
|
+
combinableValues: false,
|
|
7889
|
+
values: [
|
|
7890
|
+
{ text: "-1,000,000% to -1%", rangeValue: { min: -1_000_000, max: -1 } },
|
|
7891
|
+
{ text: "0%", rangeValue: { min: 0, max: 0 } },
|
|
7892
|
+
{ text: "1% to 10%", rangeValue: { min: 1, max: 10 } },
|
|
7893
|
+
{ text: "11% to 25%", rangeValue: { min: 11, max: 25 } },
|
|
7894
|
+
{ text: "26% to 50%", rangeValue: { min: 26, max: 50 } },
|
|
7895
|
+
{ text: "51% to 100%", rangeValue: { min: 51, max: 100 } },
|
|
7896
|
+
{ text: "101%+", rangeValue: { min: 101, max: 1_000_000 } },
|
|
7897
|
+
],
|
|
7898
|
+
};
|
|
7899
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_PARENT_REGION = {
|
|
7900
|
+
id: "90000084",
|
|
7901
|
+
text: "San Francisco Bay Area",
|
|
7902
|
+
selectionType: "INCLUDED",
|
|
7903
|
+
};
|
|
7904
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES = [
|
|
7905
|
+
{
|
|
7906
|
+
id: "102839053",
|
|
7907
|
+
text: "Alameda County, California, United States",
|
|
7908
|
+
selectionType: "INCLUDED",
|
|
7909
|
+
},
|
|
7910
|
+
{
|
|
7911
|
+
id: "102476922",
|
|
7912
|
+
text: "Contra Costa County, California, United States",
|
|
7913
|
+
selectionType: "INCLUDED",
|
|
7914
|
+
},
|
|
7915
|
+
{
|
|
7916
|
+
id: "104929029",
|
|
7917
|
+
text: "Marin County, California, United States",
|
|
7918
|
+
selectionType: "INCLUDED",
|
|
7919
|
+
},
|
|
7920
|
+
{
|
|
7921
|
+
id: "105546787",
|
|
7922
|
+
text: "Napa County, California, United States",
|
|
7923
|
+
selectionType: "INCLUDED",
|
|
7924
|
+
},
|
|
7925
|
+
{
|
|
7926
|
+
id: "100901743",
|
|
7927
|
+
text: "San Francisco County, California, United States",
|
|
7928
|
+
selectionType: "INCLUDED",
|
|
7929
|
+
},
|
|
7930
|
+
{
|
|
7931
|
+
id: "104739033",
|
|
7932
|
+
text: "San Mateo County, California, United States",
|
|
7933
|
+
selectionType: "INCLUDED",
|
|
7934
|
+
},
|
|
7935
|
+
{
|
|
7936
|
+
id: "105416315",
|
|
7937
|
+
text: "Santa Clara County, California, United States",
|
|
7938
|
+
selectionType: "INCLUDED",
|
|
7939
|
+
},
|
|
7940
|
+
{
|
|
7941
|
+
id: "103079873",
|
|
7942
|
+
text: "Solano County, California, United States",
|
|
7943
|
+
selectionType: "INCLUDED",
|
|
7944
|
+
},
|
|
7945
|
+
{
|
|
7946
|
+
id: "101079764",
|
|
7947
|
+
text: "Sonoma County, California, United States",
|
|
7948
|
+
selectionType: "INCLUDED",
|
|
7949
|
+
},
|
|
7950
|
+
];
|
|
7951
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_SAN_FRANCISCO_CITY = {
|
|
7952
|
+
id: "102277331",
|
|
7953
|
+
text: "San Francisco, California, United States",
|
|
7954
|
+
selectionType: "INCLUDED",
|
|
7955
|
+
};
|
|
7956
|
+
function isLocalAccountSearchCollectorNodeTask(value) {
|
|
7957
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
7958
|
+
return false;
|
|
7959
|
+
}
|
|
7960
|
+
const task = value;
|
|
7961
|
+
return (task.kind === "node" &&
|
|
7962
|
+
typeof task.id === "string" &&
|
|
7963
|
+
typeof task.pass === "string" &&
|
|
7964
|
+
Array.isArray(task.filters) &&
|
|
7965
|
+
Array.isArray(task.remainingDimensions));
|
|
7966
|
+
}
|
|
7967
|
+
function localAccountSearchCollectorHasFilterType(filters, filterType) {
|
|
7968
|
+
return filters.some((filter) => filter.type === filterType);
|
|
7969
|
+
}
|
|
7970
|
+
function buildLocalAccountSearchCollectorDimensionPartitions(dimension) {
|
|
7971
|
+
const partitions = dimension.values.map((value, index) => ({
|
|
7972
|
+
idSuffix: String(index),
|
|
7973
|
+
filter: {
|
|
7974
|
+
type: dimension.filterType,
|
|
7975
|
+
values: [{ ...value }],
|
|
7976
|
+
},
|
|
7977
|
+
}));
|
|
7978
|
+
const residualValues = dimension.values
|
|
7979
|
+
.filter((value) => typeof value.id === "string" && value.id.length > 0)
|
|
7980
|
+
.map((value) => ({
|
|
7981
|
+
...value,
|
|
7982
|
+
selectionType: "EXCLUDED",
|
|
7983
|
+
}));
|
|
7984
|
+
if (dimension.supportsResidualExclusion !== false &&
|
|
7985
|
+
residualValues.length === dimension.values.length &&
|
|
7986
|
+
residualValues.length > 0) {
|
|
7987
|
+
partitions.push({
|
|
7988
|
+
idSuffix: "residual",
|
|
7989
|
+
filter: {
|
|
7990
|
+
type: dimension.filterType,
|
|
7991
|
+
values: residualValues,
|
|
7992
|
+
},
|
|
7993
|
+
});
|
|
7994
|
+
}
|
|
7995
|
+
return partitions;
|
|
7996
|
+
}
|
|
7997
|
+
function withLocalAccountSearchCollectorKeywords(searchUrl, keywordExpression) {
|
|
7998
|
+
const url = new URL(searchUrl);
|
|
7999
|
+
const query = url.searchParams.get("query");
|
|
8000
|
+
if (!query) {
|
|
8001
|
+
return null;
|
|
8002
|
+
}
|
|
8003
|
+
let nextQuery;
|
|
8004
|
+
if (/^\(keywords:[\s\S]*?,filters:List\(/.test(query)) {
|
|
8005
|
+
nextQuery = query.replace(/^\(keywords:[\s\S]*?,filters:List\(/, `(keywords:${keywordExpression},filters:List(`);
|
|
8006
|
+
}
|
|
8007
|
+
else if (query.startsWith("(filters:List(")) {
|
|
8008
|
+
nextQuery = `(keywords:${keywordExpression},${query.slice(1)}`;
|
|
8009
|
+
}
|
|
8010
|
+
else {
|
|
8011
|
+
return null;
|
|
8012
|
+
}
|
|
8013
|
+
url.searchParams.set("query", nextQuery);
|
|
8014
|
+
return url.toString();
|
|
8015
|
+
}
|
|
8016
|
+
function buildLocalAccountSearchCollectorKeywordPartitions(searchUrl, task) {
|
|
8017
|
+
const keywordDepth = Math.max(0, Math.trunc(task.keywordDepth ?? 0));
|
|
8018
|
+
const token = LOCAL_ACCOUNT_SEARCH_COLLECTOR_KEYWORD_TOKENS[keywordDepth];
|
|
8019
|
+
if (!token) {
|
|
8020
|
+
return null;
|
|
8021
|
+
}
|
|
8022
|
+
const keywordClauses = Array.isArray(task.keywordClauses)
|
|
8023
|
+
? task.keywordClauses.filter((clause) => clause.trim().length > 0)
|
|
8024
|
+
: [];
|
|
8025
|
+
const branches = [token, `NOT ${token}`].map((clause) => {
|
|
8026
|
+
const nextClauses = [...keywordClauses, clause];
|
|
8027
|
+
return {
|
|
8028
|
+
clause,
|
|
8029
|
+
nextClauses,
|
|
8030
|
+
searchUrl: withLocalAccountSearchCollectorKeywords(searchUrl, nextClauses.join(" AND ")),
|
|
8031
|
+
};
|
|
8032
|
+
});
|
|
8033
|
+
if (branches.some((branch) => !branch.searchUrl)) {
|
|
8034
|
+
return null;
|
|
8035
|
+
}
|
|
8036
|
+
return {
|
|
8037
|
+
splitOn: `KEYWORDS_${token}`,
|
|
8038
|
+
partitions: branches.map((branch, index) => ({
|
|
8039
|
+
idSuffix: index === 0 ? token : `not-${token}`,
|
|
8040
|
+
searchUrl: branch.searchUrl,
|
|
8041
|
+
keywordClauses: branch.nextClauses,
|
|
8042
|
+
keywordDepth: keywordDepth + 1,
|
|
8043
|
+
})),
|
|
8044
|
+
};
|
|
8045
|
+
}
|
|
8046
|
+
function buildLocalAccountSearchCollectorIndustryPartitions(searchUrl) {
|
|
8047
|
+
const url = new URL(searchUrl);
|
|
8048
|
+
const query = url.searchParams.get("query");
|
|
8049
|
+
if (!query) {
|
|
8050
|
+
return null;
|
|
8051
|
+
}
|
|
8052
|
+
const parentIndustryPattern = /\(type:INDUSTRY,values:List\(\(id:6,text:Technology%2C%20Information%20and%20Internet,selectionType:INCLUDED\)\)\)/;
|
|
8053
|
+
if (!parentIndustryPattern.test(query)) {
|
|
8054
|
+
return null;
|
|
8055
|
+
}
|
|
8056
|
+
const industryBlocks = [
|
|
8057
|
+
{
|
|
8058
|
+
idSuffix: "software-development",
|
|
8059
|
+
block: "(type:INDUSTRY,values:List((id:4,text:Software%20Development,selectionType:INCLUDED)))",
|
|
8060
|
+
},
|
|
8061
|
+
{
|
|
8062
|
+
idSuffix: "technology-internet-residual",
|
|
8063
|
+
block: "(type:INDUSTRY,values:List((id:6,text:Technology%2C%20Information%20and%20Internet,selectionType:INCLUDED),(id:4,text:Software%20Development,selectionType:EXCLUDED)))",
|
|
8064
|
+
},
|
|
8065
|
+
];
|
|
8066
|
+
return {
|
|
8067
|
+
splitOn: "INDUSTRY_SOFTWARE_DEVELOPMENT",
|
|
8068
|
+
partitions: industryBlocks.map((partition) => {
|
|
8069
|
+
const childUrl = new URL(url);
|
|
8070
|
+
childUrl.searchParams.set("query", query.replace(parentIndustryPattern, partition.block));
|
|
8071
|
+
return {
|
|
8072
|
+
idSuffix: partition.idSuffix,
|
|
8073
|
+
searchUrl: childUrl.toString(),
|
|
8074
|
+
};
|
|
8075
|
+
}),
|
|
8076
|
+
};
|
|
8077
|
+
}
|
|
8078
|
+
function isSanFranciscoBayAreaAccountSearch(sourceQueryUrl) {
|
|
8079
|
+
const query = new URL(sourceQueryUrl).searchParams.get("query") ?? "";
|
|
8080
|
+
return (query.includes("type:REGION") &&
|
|
8081
|
+
query.includes(`id:${LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_PARENT_REGION.id}`));
|
|
8082
|
+
}
|
|
8083
|
+
function buildLocalAccountSearchCollectorBayAreaPartitions(sourceQueryUrl, filters) {
|
|
8084
|
+
if (!isSanFranciscoBayAreaAccountSearch(sourceQueryUrl)) {
|
|
8085
|
+
return null;
|
|
8086
|
+
}
|
|
8087
|
+
const regionFilter = filters.find((filter) => filter.type === "REGION");
|
|
8088
|
+
if (!regionFilter) {
|
|
8089
|
+
const countyPartitions = LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES.map((county) => [
|
|
8090
|
+
...filters,
|
|
8091
|
+
{ type: "REGION", values: [{ ...county }] },
|
|
8092
|
+
]);
|
|
8093
|
+
const residualPartition = [
|
|
8094
|
+
...filters,
|
|
8095
|
+
{
|
|
8096
|
+
type: "REGION",
|
|
8097
|
+
values: [
|
|
8098
|
+
{ ...LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_PARENT_REGION },
|
|
8099
|
+
...LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES.map((county) => ({
|
|
8100
|
+
...county,
|
|
8101
|
+
selectionType: "EXCLUDED",
|
|
8102
|
+
})),
|
|
8103
|
+
],
|
|
8104
|
+
},
|
|
8105
|
+
];
|
|
8106
|
+
return {
|
|
8107
|
+
splitOn: "REGION_BAY_AREA_COUNTY",
|
|
8108
|
+
partitions: [...countyPartitions, residualPartition],
|
|
8109
|
+
};
|
|
8110
|
+
}
|
|
8111
|
+
const includedRegions = regionFilter.values.filter((value) => (value.selectionType ?? "INCLUDED") === "INCLUDED");
|
|
8112
|
+
const excludedRegions = regionFilter.values.filter((value) => value.selectionType === "EXCLUDED");
|
|
8113
|
+
const sanFranciscoCounty = LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES.find((county) => county.id === "100901743");
|
|
8114
|
+
if (sanFranciscoCounty &&
|
|
8115
|
+
includedRegions.length === 1 &&
|
|
8116
|
+
includedRegions[0]?.id === sanFranciscoCounty.id &&
|
|
8117
|
+
excludedRegions.length === 0) {
|
|
8118
|
+
return {
|
|
8119
|
+
splitOn: "REGION_SAN_FRANCISCO_CITY",
|
|
8120
|
+
partitions: [
|
|
8121
|
+
[
|
|
8122
|
+
...filters.filter((filter) => filter.type !== "REGION"),
|
|
8123
|
+
{
|
|
8124
|
+
type: "REGION",
|
|
8125
|
+
values: [{ ...LOCAL_ACCOUNT_SEARCH_COLLECTOR_SAN_FRANCISCO_CITY }],
|
|
8126
|
+
},
|
|
8127
|
+
],
|
|
8128
|
+
[
|
|
8129
|
+
...filters.filter((filter) => filter.type !== "REGION"),
|
|
8130
|
+
{
|
|
8131
|
+
type: "REGION",
|
|
8132
|
+
values: [
|
|
8133
|
+
{ ...sanFranciscoCounty },
|
|
8134
|
+
{
|
|
8135
|
+
...LOCAL_ACCOUNT_SEARCH_COLLECTOR_SAN_FRANCISCO_CITY,
|
|
8136
|
+
selectionType: "EXCLUDED",
|
|
8137
|
+
},
|
|
8138
|
+
],
|
|
8139
|
+
},
|
|
8140
|
+
],
|
|
8141
|
+
],
|
|
8142
|
+
};
|
|
8143
|
+
}
|
|
8144
|
+
return null;
|
|
8145
|
+
}
|
|
8146
|
+
function migrateLocalAccountSearchCollectorCheckpoint(checkpoint) {
|
|
8147
|
+
let changed = false;
|
|
8148
|
+
const growthFilterType = LOCAL_ACCOUNT_SEARCH_COLLECTOR_GROWTH_DIMENSION.filterType;
|
|
8149
|
+
const needsCoverageMigration = checkpoint.coveragePassVersion !==
|
|
8150
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION;
|
|
8151
|
+
if (needsCoverageMigration) {
|
|
8152
|
+
checkpoint.taskQueue = checkpoint.taskQueue.filter((task) => !task.pass.startsWith("coverage-"));
|
|
8153
|
+
checkpoint.seenQueries = checkpoint.seenQueries.filter((queryKey) => !queryKey.startsWith("coverage-"));
|
|
8154
|
+
checkpoint.coveragePassSeeded = false;
|
|
8155
|
+
checkpoint.coveragePassVersion =
|
|
8156
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION;
|
|
8157
|
+
changed = true;
|
|
8158
|
+
}
|
|
8159
|
+
for (const task of checkpoint.taskQueue) {
|
|
8160
|
+
if (task.kind === "node" && task.pass.startsWith("coverage-")) {
|
|
8161
|
+
const remainingDimensions = task.remainingDimensions.filter((filterType) => filterType !== growthFilterType);
|
|
8162
|
+
if (remainingDimensions.length !== task.remainingDimensions.length) {
|
|
8163
|
+
task.remainingDimensions = remainingDimensions;
|
|
8164
|
+
changed = true;
|
|
8165
|
+
}
|
|
8166
|
+
continue;
|
|
8167
|
+
}
|
|
8168
|
+
if (task.kind === "node" &&
|
|
8169
|
+
!localAccountSearchCollectorHasFilterType(task.filters, growthFilterType) &&
|
|
8170
|
+
!task.remainingDimensions.includes(growthFilterType)) {
|
|
8171
|
+
task.remainingDimensions.push(growthFilterType);
|
|
8172
|
+
changed = true;
|
|
8173
|
+
}
|
|
8174
|
+
}
|
|
8175
|
+
const queuedNodeQueries = new Set(checkpoint.taskQueue
|
|
8176
|
+
.filter(isLocalAccountSearchCollectorNodeTask)
|
|
8177
|
+
.map((task) => localAccountSearchCollectorQueryKey(buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, task.filters).slicedQueryUrl)));
|
|
8178
|
+
const remainingFailures = [];
|
|
8179
|
+
for (const failure of checkpoint.failures) {
|
|
8180
|
+
const error = typeof failure.error === "string" ? failure.error : "";
|
|
8181
|
+
const task = failure.task;
|
|
8182
|
+
const isTerminalSlice = error.includes("with no split dimension left");
|
|
8183
|
+
if (!isTerminalSlice || !isLocalAccountSearchCollectorNodeTask(task)) {
|
|
8184
|
+
remainingFailures.push(failure);
|
|
8185
|
+
continue;
|
|
8186
|
+
}
|
|
8187
|
+
const hasGrowthFilter = localAccountSearchCollectorHasFilterType(task.filters, growthFilterType);
|
|
8188
|
+
const canUseBayAreaFallback = Boolean(buildLocalAccountSearchCollectorBayAreaPartitions(checkpoint.sourceQueryUrl, task.filters));
|
|
8189
|
+
if (hasGrowthFilter && !canUseBayAreaFallback) {
|
|
8190
|
+
remainingFailures.push(failure);
|
|
8191
|
+
continue;
|
|
8192
|
+
}
|
|
8193
|
+
const remainingDimensions = [...task.remainingDimensions];
|
|
8194
|
+
if (!hasGrowthFilter && !remainingDimensions.includes(growthFilterType)) {
|
|
8195
|
+
remainingDimensions.push(growthFilterType);
|
|
8196
|
+
}
|
|
8197
|
+
const queryKey = localAccountSearchCollectorQueryKey(buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, task.filters).slicedQueryUrl);
|
|
8198
|
+
if (!queuedNodeQueries.has(queryKey)) {
|
|
8199
|
+
checkpoint.taskQueue.push({
|
|
8200
|
+
...task,
|
|
8201
|
+
filters: task.filters.map((filter) => ({
|
|
8202
|
+
...filter,
|
|
8203
|
+
values: filter.values.map((value) => ({ ...value })),
|
|
8204
|
+
})),
|
|
8205
|
+
remainingDimensions,
|
|
8206
|
+
});
|
|
8207
|
+
queuedNodeQueries.add(queryKey);
|
|
8208
|
+
}
|
|
8209
|
+
changed = true;
|
|
8210
|
+
}
|
|
8211
|
+
if (remainingFailures.length !== checkpoint.failures.length) {
|
|
8212
|
+
checkpoint.failures = remainingFailures;
|
|
8213
|
+
}
|
|
8214
|
+
const isFreshCheckpoint = checkpoint.taskQueue.length === 0 && checkpoint.seenQueries.length === 0;
|
|
8215
|
+
if (!isFreshCheckpoint &&
|
|
8216
|
+
needsCoverageMigration &&
|
|
8217
|
+
(checkpoint.rootTotal ?? 0) > checkpoint.cap) {
|
|
8218
|
+
seedLocalAccountSearchCollectorPassTasks(checkpoint, LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS, true);
|
|
8219
|
+
checkpoint.coveragePassSeeded = true;
|
|
8220
|
+
changed = true;
|
|
8221
|
+
}
|
|
8222
|
+
const coverageTasks = checkpoint.taskQueue.filter((task) => task.pass.startsWith("coverage-"));
|
|
8223
|
+
if (coverageTasks.length > 0 &&
|
|
8224
|
+
checkpoint.taskQueue.some((task, index) => task.pass.startsWith("coverage-") && index >= coverageTasks.length)) {
|
|
8225
|
+
checkpoint.taskQueue = [
|
|
8226
|
+
...coverageTasks,
|
|
8227
|
+
...checkpoint.taskQueue.filter((task) => !task.pass.startsWith("coverage-")),
|
|
8228
|
+
];
|
|
8229
|
+
changed = true;
|
|
8230
|
+
}
|
|
8231
|
+
return changed;
|
|
8232
|
+
}
|
|
8233
|
+
function parseLocalAccountSearchCollectorCheckpoint(value) {
|
|
8234
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
8235
|
+
throw new Error("Account Search checkpoint must be a JSON object.");
|
|
8236
|
+
}
|
|
8237
|
+
const checkpoint = value;
|
|
8238
|
+
if (checkpoint.version !== 1 ||
|
|
8239
|
+
typeof checkpoint.sourceQueryUrl !== "string" ||
|
|
8240
|
+
!Array.isArray(checkpoint.taskQueue) ||
|
|
8241
|
+
!Array.isArray(checkpoint.leaves) ||
|
|
8242
|
+
!Array.isArray(checkpoint.windows) ||
|
|
8243
|
+
!Array.isArray(checkpoint.failures)) {
|
|
8244
|
+
throw new Error("Unsupported or invalid Account Search checkpoint.");
|
|
8245
|
+
}
|
|
8246
|
+
const seenQueryValues = Array.isArray(checkpoint.seenQueries)
|
|
8247
|
+
? checkpoint.seenQueries
|
|
8248
|
+
: checkpoint.seenQueries && typeof checkpoint.seenQueries === "object"
|
|
8249
|
+
? Object.keys(checkpoint.seenQueries)
|
|
8250
|
+
: [];
|
|
8251
|
+
return {
|
|
8252
|
+
...checkpoint,
|
|
8253
|
+
seenQueries: seenQueryValues.map(localAccountSearchCollectorQueryKey),
|
|
8254
|
+
};
|
|
8255
|
+
}
|
|
8256
|
+
function localAccountSearchCollectorQueryKey(searchUrlOrQuery) {
|
|
8257
|
+
try {
|
|
8258
|
+
const query = new URL(searchUrlOrQuery).searchParams.get("query");
|
|
8259
|
+
return query ?? searchUrlOrQuery;
|
|
8260
|
+
}
|
|
8261
|
+
catch {
|
|
8262
|
+
return searchUrlOrQuery;
|
|
8263
|
+
}
|
|
8264
|
+
}
|
|
8265
|
+
function localAccountSearchCollectorSeenKey(pass, searchUrlOrQuery) {
|
|
8266
|
+
const queryKey = localAccountSearchCollectorQueryKey(searchUrlOrQuery);
|
|
8267
|
+
return pass.startsWith("coverage-") ? `${pass}:${queryKey}` : queryKey;
|
|
8268
|
+
}
|
|
8269
|
+
function createLocalAccountSearchCollectorCheckpoint(params) {
|
|
8270
|
+
const now = new Date().toISOString();
|
|
8271
|
+
return {
|
|
8272
|
+
version: 1,
|
|
8273
|
+
sourceQueryUrl: params.sourceQueryUrl,
|
|
8274
|
+
rootTotal: params.rootTotal,
|
|
8275
|
+
cap: params.cap,
|
|
8276
|
+
pageSize: params.pageSize,
|
|
8277
|
+
delayMinMs: params.delayMinMs,
|
|
8278
|
+
delayMaxMs: params.delayMaxMs,
|
|
8279
|
+
requests: 0,
|
|
8280
|
+
pagesWritten: 0,
|
|
8281
|
+
rowsWritten: 0,
|
|
8282
|
+
startedAt: now,
|
|
8283
|
+
updatedAt: now,
|
|
8284
|
+
taskQueue: [],
|
|
8285
|
+
seenQueries: [],
|
|
8286
|
+
duplicateQueries: 0,
|
|
8287
|
+
leaves: [],
|
|
8288
|
+
windows: [],
|
|
8289
|
+
failures: [],
|
|
8290
|
+
rateLimited: null,
|
|
8291
|
+
coveragePassSeeded: false,
|
|
8292
|
+
coveragePassVersion: LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION,
|
|
8293
|
+
distinctAccounts: 0,
|
|
8294
|
+
completedByDistinctCount: false,
|
|
8295
|
+
};
|
|
8296
|
+
}
|
|
8297
|
+
function seedLocalAccountSearchCollectorTasks(checkpoint) {
|
|
8298
|
+
for (const pass of LOCAL_ACCOUNT_SEARCH_COLLECTOR_PASSES) {
|
|
8299
|
+
seedLocalAccountSearchCollectorPassTasks(checkpoint, pass.name, false);
|
|
8300
|
+
}
|
|
8301
|
+
checkpoint.coveragePassSeeded = true;
|
|
8302
|
+
checkpoint.coveragePassVersion =
|
|
8303
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION;
|
|
8304
|
+
}
|
|
8305
|
+
function seedLocalAccountSearchCollectorPassTasks(checkpoint, passName, prepend) {
|
|
8306
|
+
const dimensions = new Map(DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS.map((dimension) => [
|
|
8307
|
+
dimension.filterType,
|
|
8308
|
+
dimension,
|
|
8309
|
+
]));
|
|
8310
|
+
const seenQueries = new Set(checkpoint.seenQueries);
|
|
8311
|
+
const pass = LOCAL_ACCOUNT_SEARCH_COLLECTOR_PASSES.find((candidate) => candidate.name === passName);
|
|
8312
|
+
if (!pass) {
|
|
8313
|
+
return false;
|
|
8314
|
+
}
|
|
8315
|
+
const [firstFilterType, ...remainingDimensions] = pass.filterTypes;
|
|
8316
|
+
const dimension = dimensions.get(firstFilterType);
|
|
8317
|
+
if (!dimension) {
|
|
8318
|
+
throw new Error(`Missing Account Search split dimension ${firstFilterType}.`);
|
|
8319
|
+
}
|
|
8320
|
+
const tasks = [];
|
|
8321
|
+
for (const partition of buildLocalAccountSearchCollectorDimensionPartitions(dimension)) {
|
|
8322
|
+
const filters = [partition.filter];
|
|
8323
|
+
const searchUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, filters).slicedQueryUrl;
|
|
8324
|
+
const queryKey = localAccountSearchCollectorSeenKey(pass.name, searchUrl);
|
|
8325
|
+
if (seenQueries.has(queryKey)) {
|
|
8326
|
+
continue;
|
|
8327
|
+
}
|
|
8328
|
+
seenQueries.add(queryKey);
|
|
8329
|
+
tasks.push({
|
|
8330
|
+
kind: "node",
|
|
8331
|
+
id: `${pass.name}-${partition.idSuffix}`,
|
|
8332
|
+
pass: pass.name,
|
|
8333
|
+
filters,
|
|
8334
|
+
remainingDimensions: [...remainingDimensions],
|
|
8335
|
+
});
|
|
8336
|
+
}
|
|
8337
|
+
if (prepend) {
|
|
8338
|
+
checkpoint.taskQueue.unshift(...tasks);
|
|
8339
|
+
}
|
|
8340
|
+
else {
|
|
8341
|
+
checkpoint.taskQueue.push(...tasks);
|
|
8342
|
+
}
|
|
8343
|
+
checkpoint.seenQueries = [...seenQueries];
|
|
8344
|
+
return tasks.length > 0;
|
|
8345
|
+
}
|
|
8346
|
+
function enqueueLocalAccountSearchCollectorTasks(checkpoint, parentTask, tasks) {
|
|
8347
|
+
if (tasks.length === 0) {
|
|
8348
|
+
return;
|
|
8349
|
+
}
|
|
8350
|
+
if (parentTask.pass.startsWith("coverage-")) {
|
|
8351
|
+
checkpoint.taskQueue.splice(1, 0, ...tasks);
|
|
8352
|
+
return;
|
|
8353
|
+
}
|
|
8354
|
+
checkpoint.taskQueue.push(...tasks);
|
|
8355
|
+
}
|
|
8356
|
+
function extractLocalSalesNavigatorElements(body) {
|
|
8357
|
+
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
8358
|
+
return [];
|
|
8359
|
+
}
|
|
8360
|
+
const record = body;
|
|
8361
|
+
const elements = record.elements;
|
|
8362
|
+
if (Array.isArray(elements)) {
|
|
8363
|
+
return elements;
|
|
8364
|
+
}
|
|
8365
|
+
const data = typeof record.data === "object" &&
|
|
8366
|
+
record.data !== null &&
|
|
8367
|
+
!Array.isArray(record.data)
|
|
8368
|
+
? record.data
|
|
8369
|
+
: null;
|
|
8370
|
+
const nestedElements = data?.elements;
|
|
8371
|
+
if (Array.isArray(nestedElements)) {
|
|
8372
|
+
return nestedElements;
|
|
8373
|
+
}
|
|
8374
|
+
const references = data?.["*elements"];
|
|
8375
|
+
const included = record.included;
|
|
8376
|
+
if (!Array.isArray(references) || !Array.isArray(included)) {
|
|
8377
|
+
return [];
|
|
8378
|
+
}
|
|
8379
|
+
const referencedUrns = new Set(references.filter((reference) => typeof reference === "string" && reference.length > 0));
|
|
8380
|
+
return included.filter((value) => {
|
|
8381
|
+
if (typeof value !== "object" ||
|
|
8382
|
+
value === null ||
|
|
8383
|
+
Array.isArray(value)) {
|
|
8384
|
+
return false;
|
|
8385
|
+
}
|
|
8386
|
+
const entityUrn = value.entityUrn;
|
|
8387
|
+
return typeof entityUrn === "string" && referencedUrns.has(entityUrn);
|
|
8388
|
+
});
|
|
8389
|
+
}
|
|
8390
|
+
async function appendLocalAccountSearchRawElements(params) {
|
|
8391
|
+
if (params.elements.length === 0) {
|
|
8392
|
+
return;
|
|
8393
|
+
}
|
|
8394
|
+
await mkdir(path.dirname(params.rawOutputPath), { recursive: true });
|
|
8395
|
+
await appendFile(params.rawOutputPath, `${params.elements
|
|
8396
|
+
.map((element) => JSON.stringify({ queryUrl: params.searchUrl, element }))
|
|
8397
|
+
.join("\n")}\n`, "utf8");
|
|
8398
|
+
}
|
|
8399
|
+
function addLocalAccountSearchElementKey(accountKeys, element, queryUrl) {
|
|
8400
|
+
if (typeof element !== "object" || element === null || Array.isArray(element)) {
|
|
8401
|
+
return;
|
|
8402
|
+
}
|
|
8403
|
+
const companyId = extractLinkedInSalesCompanyIdFromUrn(element.entityUrn);
|
|
8404
|
+
if (companyId) {
|
|
8405
|
+
accountKeys.add(`company-id:${companyId}`);
|
|
8406
|
+
return;
|
|
8407
|
+
}
|
|
8408
|
+
const account = normalizeLocalSalesNavigatorAccount(element, queryUrl);
|
|
8409
|
+
if (account) {
|
|
8410
|
+
accountKeys.add(account.accountKey);
|
|
8411
|
+
}
|
|
8412
|
+
}
|
|
8413
|
+
async function loadLocalAccountSearchRawAccountKeys(rawOutputPath) {
|
|
8414
|
+
const accountKeys = new Set();
|
|
8415
|
+
if (!(await fileExists(rawOutputPath))) {
|
|
8416
|
+
return accountKeys;
|
|
8417
|
+
}
|
|
8418
|
+
const raw = await readFile(rawOutputPath, "utf8");
|
|
8419
|
+
for (const line of raw.split(/\r?\n/)) {
|
|
8420
|
+
if (!line.trim()) {
|
|
8421
|
+
continue;
|
|
8422
|
+
}
|
|
8423
|
+
try {
|
|
8424
|
+
const row = JSON.parse(line);
|
|
8425
|
+
addLocalAccountSearchElementKey(accountKeys, row.element, typeof row.queryUrl === "string" ? row.queryUrl : "");
|
|
8426
|
+
}
|
|
8427
|
+
catch {
|
|
8428
|
+
// The import command reports malformed raw rows with line-level context.
|
|
8429
|
+
}
|
|
8430
|
+
}
|
|
8431
|
+
return accountKeys;
|
|
8432
|
+
}
|
|
8433
|
+
async function saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint) {
|
|
8434
|
+
checkpoint.updatedAt = new Date().toISOString();
|
|
8435
|
+
await writeJsonFile(checkpointPath, checkpoint);
|
|
8436
|
+
}
|
|
6849
8437
|
async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
|
|
6850
8438
|
const people = [];
|
|
6851
8439
|
const seen = new Set();
|
|
@@ -6869,7 +8457,9 @@ async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
|
|
|
6869
8457
|
...parsedRequest,
|
|
6870
8458
|
url: withSalesNavigatorPaging(parsedRequest.url, start, currentCount),
|
|
6871
8459
|
};
|
|
6872
|
-
const response =
|
|
8460
|
+
const response = options.executeRequest
|
|
8461
|
+
? await options.executeRequest(pageRequest)
|
|
8462
|
+
: await fetchLocalSalesNavigatorRequest(pageRequest, options.retry);
|
|
6873
8463
|
retryCount += response.retryCount;
|
|
6874
8464
|
totalDelayMs += response.retryDelayMs;
|
|
6875
8465
|
const responseBody = response.body;
|
|
@@ -6915,6 +8505,7 @@ async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
|
|
|
6915
8505
|
async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
6916
8506
|
const accounts = [];
|
|
6917
8507
|
const seen = new Set();
|
|
8508
|
+
let filteredAccounts = 0;
|
|
6918
8509
|
let totalResults = null;
|
|
6919
8510
|
let fetchedPages = 0;
|
|
6920
8511
|
let totalDelayMs = 0;
|
|
@@ -6989,13 +8580,20 @@ async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
|
6989
8580
|
}
|
|
6990
8581
|
const pageAccounts = normalizeLocalSalesNavigatorAccounts(responseBody, pageRequest.url);
|
|
6991
8582
|
let added = 0;
|
|
8583
|
+
let addedOrFiltered = 0;
|
|
6992
8584
|
for (const account of pageAccounts) {
|
|
6993
8585
|
if (seen.has(account.accountKey)) {
|
|
6994
8586
|
continue;
|
|
6995
8587
|
}
|
|
6996
8588
|
seen.add(account.accountKey);
|
|
8589
|
+
if (!salesNavigatorAccountMatchesLocationFilter(account, options.accountLocationFilter)) {
|
|
8590
|
+
filteredAccounts += 1;
|
|
8591
|
+
addedOrFiltered += 1;
|
|
8592
|
+
continue;
|
|
8593
|
+
}
|
|
6997
8594
|
accounts.push(account);
|
|
6998
8595
|
added += 1;
|
|
8596
|
+
addedOrFiltered += 1;
|
|
6999
8597
|
if (requestedAccounts != null && accounts.length >= requestedAccounts) {
|
|
7000
8598
|
break;
|
|
7001
8599
|
}
|
|
@@ -7020,7 +8618,7 @@ async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
|
7020
8618
|
nextStart = start;
|
|
7021
8619
|
break;
|
|
7022
8620
|
}
|
|
7023
|
-
if (
|
|
8621
|
+
if (addedOrFiltered === 0) {
|
|
7024
8622
|
stopReason = "duplicate_page";
|
|
7025
8623
|
stoppedAtStart = start;
|
|
7026
8624
|
stoppedAtCount = currentCount;
|
|
@@ -7044,6 +8642,7 @@ async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
|
7044
8642
|
const missingFromReported = totalResults == null ? null : Math.max(0, totalResults - fetchedThrough);
|
|
7045
8643
|
return {
|
|
7046
8644
|
accounts,
|
|
8645
|
+
filteredAccounts,
|
|
7047
8646
|
totalResults,
|
|
7048
8647
|
fetchedPages,
|
|
7049
8648
|
sourceQueryUrl: parsedRequest.url,
|
|
@@ -7125,6 +8724,13 @@ async function importLocalSalesNavigatorPeopleViaApp(session, payload) {
|
|
|
7125
8724
|
}, SalesNavigatorLocalImportResponseSchema);
|
|
7126
8725
|
return value;
|
|
7127
8726
|
}
|
|
8727
|
+
const SALESPROMPTER_SUPABASE_TABLES = {
|
|
8728
|
+
accountSearchRuns: "salesprompter_linkedin_sales_nav_account_search_runs",
|
|
8729
|
+
accountSearchSlices: "salesprompter_linkedin_sales_nav_account_search_slices",
|
|
8730
|
+
accountSearchAccounts: "salesprompter_linkedin_sales_nav_accounts",
|
|
8731
|
+
accountRunMemberships: "salesprompter_linkedin_sales_nav_account_run_memberships",
|
|
8732
|
+
linkedinCompaniesProcessed: "salesprompter_linkedin_companies_processed",
|
|
8733
|
+
};
|
|
7128
8734
|
async function createSalesNavigatorAccountSearchStore() {
|
|
7129
8735
|
const databaseUrl = process.env.SALESPROMPTER_DATABASE_URL?.trim() ||
|
|
7130
8736
|
process.env.DATABASE_URL?.trim() ||
|
|
@@ -7155,7 +8761,7 @@ async function closeSalesNavigatorAccountSearchStore(store) {
|
|
|
7155
8761
|
}
|
|
7156
8762
|
async function createSalesNavigatorAccountSearchRun(params) {
|
|
7157
8763
|
if (params.store.kind === "postgres") {
|
|
7158
|
-
const result = await params.store.client.query(`INSERT INTO public.
|
|
8764
|
+
const result = await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_account_search_runs (
|
|
7159
8765
|
org_id, source_query_url, slice_preset, max_results_per_search, page_size, status, raw_payload
|
|
7160
8766
|
)
|
|
7161
8767
|
VALUES ($1, $2, $3, $4, $5, 'running', $6::jsonb)
|
|
@@ -7169,12 +8775,12 @@ async function createSalesNavigatorAccountSearchRun(params) {
|
|
|
7169
8775
|
]);
|
|
7170
8776
|
const id = result.rows[0]?.id ?? "";
|
|
7171
8777
|
if (!id) {
|
|
7172
|
-
throw new Error("Failed to create account search run in
|
|
8778
|
+
throw new Error("Failed to create account search run in Supabase: missing run id.");
|
|
7173
8779
|
}
|
|
7174
8780
|
return id;
|
|
7175
8781
|
}
|
|
7176
8782
|
const { data, error } = await params.store.client
|
|
7177
|
-
.from(
|
|
8783
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchRuns)
|
|
7178
8784
|
.insert({
|
|
7179
8785
|
org_id: params.orgId,
|
|
7180
8786
|
source_query_url: params.sourceQueryUrl,
|
|
@@ -7187,17 +8793,17 @@ async function createSalesNavigatorAccountSearchRun(params) {
|
|
|
7187
8793
|
.select("id")
|
|
7188
8794
|
.single();
|
|
7189
8795
|
if (error) {
|
|
7190
|
-
throw new Error(`Failed to create account search run in
|
|
8796
|
+
throw new Error(`Failed to create account search run in Supabase: ${error.message}`);
|
|
7191
8797
|
}
|
|
7192
8798
|
const id = typeof data?.id === "string" ? data.id : "";
|
|
7193
8799
|
if (!id) {
|
|
7194
|
-
throw new Error("Failed to create account search run in
|
|
8800
|
+
throw new Error("Failed to create account search run in Supabase: missing run id.");
|
|
7195
8801
|
}
|
|
7196
8802
|
return id;
|
|
7197
8803
|
}
|
|
7198
8804
|
async function updateSalesNavigatorAccountSearchRun(params) {
|
|
7199
8805
|
if (params.store.kind === "postgres") {
|
|
7200
|
-
await params.store.client.query(`UPDATE public.
|
|
8806
|
+
await params.store.client.query(`UPDATE public.salesprompter_linkedin_sales_nav_account_search_runs
|
|
7201
8807
|
SET status = $2,
|
|
7202
8808
|
total_slices = $3,
|
|
7203
8809
|
stored_accounts = $4,
|
|
@@ -7230,12 +8836,12 @@ async function updateSalesNavigatorAccountSearchRun(params) {
|
|
|
7230
8836
|
let mergedRawPayload = null;
|
|
7231
8837
|
if (params.rawPayload) {
|
|
7232
8838
|
const { data: existing, error: selectError } = await params.store.client
|
|
7233
|
-
.from(
|
|
8839
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchRuns)
|
|
7234
8840
|
.select("raw_payload")
|
|
7235
8841
|
.eq("id", params.runId)
|
|
7236
8842
|
.maybeSingle();
|
|
7237
8843
|
if (selectError) {
|
|
7238
|
-
throw new Error(`Failed to read account search run payload in
|
|
8844
|
+
throw new Error(`Failed to read account search run payload in Supabase: ${selectError.message}`);
|
|
7239
8845
|
}
|
|
7240
8846
|
const existingPayload = typeof existing?.raw_payload === "object" &&
|
|
7241
8847
|
existing.raw_payload !== null &&
|
|
@@ -7248,7 +8854,7 @@ async function updateSalesNavigatorAccountSearchRun(params) {
|
|
|
7248
8854
|
};
|
|
7249
8855
|
}
|
|
7250
8856
|
const { error } = await params.store.client
|
|
7251
|
-
.from(
|
|
8857
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchRuns)
|
|
7252
8858
|
.update({
|
|
7253
8859
|
status: params.status,
|
|
7254
8860
|
total_slices: params.totalSlices,
|
|
@@ -7268,12 +8874,12 @@ async function updateSalesNavigatorAccountSearchRun(params) {
|
|
|
7268
8874
|
})
|
|
7269
8875
|
.eq("id", params.runId);
|
|
7270
8876
|
if (error) {
|
|
7271
|
-
throw new Error(`Failed to update account search run in
|
|
8877
|
+
throw new Error(`Failed to update account search run in Supabase: ${error.message}`);
|
|
7272
8878
|
}
|
|
7273
8879
|
}
|
|
7274
8880
|
async function upsertSalesNavigatorAccountSearchSlice(params) {
|
|
7275
8881
|
if (params.store.kind === "postgres") {
|
|
7276
|
-
const result = await params.store.client.query(`INSERT INTO public.
|
|
8882
|
+
const result = await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_account_search_slices (
|
|
7277
8883
|
run_id, org_id, sliced_query_url, applied_filters, split_trail, depth, status,
|
|
7278
8884
|
total_results, fetched_pages, stored_accounts, retry_count, retry_delay_ms, last_error, raw_payload
|
|
7279
8885
|
)
|
|
@@ -7309,12 +8915,12 @@ async function upsertSalesNavigatorAccountSearchSlice(params) {
|
|
|
7309
8915
|
]);
|
|
7310
8916
|
const id = result.rows[0]?.id ?? "";
|
|
7311
8917
|
if (!id) {
|
|
7312
|
-
throw new Error("Failed to upsert account search slice in
|
|
8918
|
+
throw new Error("Failed to upsert account search slice in Supabase: missing slice id.");
|
|
7313
8919
|
}
|
|
7314
8920
|
return id;
|
|
7315
8921
|
}
|
|
7316
8922
|
const { data, error } = await params.store.client
|
|
7317
|
-
.from(
|
|
8923
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchSlices)
|
|
7318
8924
|
.upsert({
|
|
7319
8925
|
run_id: params.runId,
|
|
7320
8926
|
org_id: params.orgId,
|
|
@@ -7334,11 +8940,11 @@ async function upsertSalesNavigatorAccountSearchSlice(params) {
|
|
|
7334
8940
|
.select("id")
|
|
7335
8941
|
.single();
|
|
7336
8942
|
if (error) {
|
|
7337
|
-
throw new Error(`Failed to upsert account search slice in
|
|
8943
|
+
throw new Error(`Failed to upsert account search slice in Supabase: ${error.message}`);
|
|
7338
8944
|
}
|
|
7339
8945
|
const id = typeof data?.id === "string" ? data.id : "";
|
|
7340
8946
|
if (!id) {
|
|
7341
|
-
throw new Error("Failed to upsert account search slice in
|
|
8947
|
+
throw new Error("Failed to upsert account search slice in Supabase: missing slice id.");
|
|
7342
8948
|
}
|
|
7343
8949
|
return id;
|
|
7344
8950
|
}
|
|
@@ -7365,6 +8971,632 @@ function sanitizeForPostgresJson(value) {
|
|
|
7365
8971
|
function sanitizeOptionalPostgresText(value) {
|
|
7366
8972
|
return typeof value === "string" ? stripPostgresUnsafeStringBytes(value) : null;
|
|
7367
8973
|
}
|
|
8974
|
+
function normalizeLinkedInCompanyScrapeUrl(rawUrl) {
|
|
8975
|
+
let parsed;
|
|
8976
|
+
try {
|
|
8977
|
+
parsed = new URL(rawUrl.startsWith("http") ? rawUrl : `https://${rawUrl}`);
|
|
8978
|
+
}
|
|
8979
|
+
catch {
|
|
8980
|
+
throw new Error(`Invalid LinkedIn company URL: ${rawUrl}`);
|
|
8981
|
+
}
|
|
8982
|
+
const idMatch = parsed.pathname.match(/\/(?:sales\/)?company\/([0-9]+)/i);
|
|
8983
|
+
if (!idMatch) {
|
|
8984
|
+
throw new Error(`LinkedIn company URL must include a numeric company id: ${rawUrl}`);
|
|
8985
|
+
}
|
|
8986
|
+
const id = Number(idMatch[1]);
|
|
8987
|
+
if (!Number.isFinite(id)) {
|
|
8988
|
+
throw new Error(`LinkedIn company URL has an invalid company id: ${rawUrl}`);
|
|
8989
|
+
}
|
|
8990
|
+
return {
|
|
8991
|
+
id,
|
|
8992
|
+
sourceUrl: parsed.toString(),
|
|
8993
|
+
companyUrl: `https://www.linkedin.com/company/${id}`,
|
|
8994
|
+
salesNavigatorLink: `https://www.linkedin.com/sales/company/${id}/`,
|
|
8995
|
+
};
|
|
8996
|
+
}
|
|
8997
|
+
function normalizeDealroomCompanyScrapeUrl(rawUrl) {
|
|
8998
|
+
let parsed;
|
|
8999
|
+
try {
|
|
9000
|
+
parsed = new URL(rawUrl.startsWith("http") ? rawUrl : `https://${rawUrl}`);
|
|
9001
|
+
}
|
|
9002
|
+
catch {
|
|
9003
|
+
throw new Error(`Invalid Dealroom company URL: ${rawUrl}`);
|
|
9004
|
+
}
|
|
9005
|
+
const slugMatch = parsed.pathname.match(/\/companies\/([^/?#]+)/i);
|
|
9006
|
+
if (!slugMatch) {
|
|
9007
|
+
throw new Error(`Dealroom company URL must include /companies/<slug>: ${rawUrl}`);
|
|
9008
|
+
}
|
|
9009
|
+
const slug = decodeURIComponent(slugMatch[1] || "").trim();
|
|
9010
|
+
if (!slug) {
|
|
9011
|
+
throw new Error(`Dealroom company URL has an invalid company slug: ${rawUrl}`);
|
|
9012
|
+
}
|
|
9013
|
+
return {
|
|
9014
|
+
slug,
|
|
9015
|
+
sourceUrl: parsed.toString(),
|
|
9016
|
+
companyUrl: `https://app.dealroom.co/companies/${encodeURIComponent(slug)}`,
|
|
9017
|
+
};
|
|
9018
|
+
}
|
|
9019
|
+
function normalizeLocalCompanyDomain(website) {
|
|
9020
|
+
if (!website) {
|
|
9021
|
+
return null;
|
|
9022
|
+
}
|
|
9023
|
+
try {
|
|
9024
|
+
const url = new URL(website.startsWith("http") ? website : `https://${website}`);
|
|
9025
|
+
return url.hostname.replace(/^www\./i, "").toLowerCase();
|
|
9026
|
+
}
|
|
9027
|
+
catch {
|
|
9028
|
+
return null;
|
|
9029
|
+
}
|
|
9030
|
+
}
|
|
9031
|
+
function normalizeDealroomWebsite(value) {
|
|
9032
|
+
if (!value) {
|
|
9033
|
+
return null;
|
|
9034
|
+
}
|
|
9035
|
+
const trimmed = value.trim();
|
|
9036
|
+
if (!trimmed || /^view job openings$/i.test(trimmed)) {
|
|
9037
|
+
return null;
|
|
9038
|
+
}
|
|
9039
|
+
try {
|
|
9040
|
+
return new URL(trimmed.startsWith("http") ? trimmed : `https://${trimmed}`).toString();
|
|
9041
|
+
}
|
|
9042
|
+
catch {
|
|
9043
|
+
return trimmed;
|
|
9044
|
+
}
|
|
9045
|
+
}
|
|
9046
|
+
function parseLinkedInAboutTextValue(lines, label) {
|
|
9047
|
+
const labelIndex = lines.findIndex((line) => line.toLowerCase() === label.toLowerCase());
|
|
9048
|
+
if (labelIndex === -1) {
|
|
9049
|
+
return null;
|
|
9050
|
+
}
|
|
9051
|
+
return lines[labelIndex + 1]?.trim() || null;
|
|
9052
|
+
}
|
|
9053
|
+
function parseLabeledTextValue(lines, label) {
|
|
9054
|
+
const expected = label.toLowerCase();
|
|
9055
|
+
const labelIndex = lines.findIndex((line) => line.toLowerCase() === expected);
|
|
9056
|
+
if (labelIndex !== -1) {
|
|
9057
|
+
return lines[labelIndex + 1]?.trim() || null;
|
|
9058
|
+
}
|
|
9059
|
+
const inline = lines.find((line) => line.toLowerCase().startsWith(`${expected} `));
|
|
9060
|
+
if (!inline) {
|
|
9061
|
+
return null;
|
|
9062
|
+
}
|
|
9063
|
+
return inline.slice(label.length).trim() || null;
|
|
9064
|
+
}
|
|
9065
|
+
function parseLinkedInCompactInteger(value) {
|
|
9066
|
+
if (!value) {
|
|
9067
|
+
return null;
|
|
9068
|
+
}
|
|
9069
|
+
const match = value.replace(/,/g, "").match(/([0-9]+(?:\.[0-9]+)?)\s*([kmb])?/i);
|
|
9070
|
+
if (!match) {
|
|
9071
|
+
return null;
|
|
9072
|
+
}
|
|
9073
|
+
const base = Number(match[1]);
|
|
9074
|
+
if (!Number.isFinite(base)) {
|
|
9075
|
+
return null;
|
|
9076
|
+
}
|
|
9077
|
+
const suffix = match[2]?.toLowerCase();
|
|
9078
|
+
const multiplier = suffix === "b" ? 1_000_000_000 : suffix === "m" ? 1_000_000 : suffix === "k" ? 1_000 : 1;
|
|
9079
|
+
return Math.round(base * multiplier);
|
|
9080
|
+
}
|
|
9081
|
+
function parseDealroomEnterpriseValueUsd(value) {
|
|
9082
|
+
if (!value) {
|
|
9083
|
+
return null;
|
|
9084
|
+
}
|
|
9085
|
+
const match = value.replace(/,/g, "").match(/([$€£])?\s*([0-9]+(?:\.[0-9]+)?)\s*([kmb])?/i);
|
|
9086
|
+
if (!match) {
|
|
9087
|
+
return null;
|
|
9088
|
+
}
|
|
9089
|
+
const base = Number(match[2]);
|
|
9090
|
+
if (!Number.isFinite(base)) {
|
|
9091
|
+
return null;
|
|
9092
|
+
}
|
|
9093
|
+
const suffix = match[3]?.toLowerCase();
|
|
9094
|
+
const multiplier = suffix === "b" ? 1_000_000_000 : suffix === "m" ? 1_000_000 : suffix === "k" ? 1_000 : 1;
|
|
9095
|
+
return Math.round(base * multiplier);
|
|
9096
|
+
}
|
|
9097
|
+
function parseLinkedInNumber(value) {
|
|
9098
|
+
if (!value) {
|
|
9099
|
+
return null;
|
|
9100
|
+
}
|
|
9101
|
+
const normalized = value.replace(/,/g, "").match(/[0-9]+(?:\.[0-9]+)?/);
|
|
9102
|
+
if (!normalized) {
|
|
9103
|
+
return null;
|
|
9104
|
+
}
|
|
9105
|
+
const parsed = Number(normalized[0]);
|
|
9106
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
9107
|
+
}
|
|
9108
|
+
function parseLinkedInGrowth(value) {
|
|
9109
|
+
if (!value) {
|
|
9110
|
+
return { value: null, label: null };
|
|
9111
|
+
}
|
|
9112
|
+
const label = value.replace(/^▲\s*/i, "").trim();
|
|
9113
|
+
if (!label || /^n\/a$/i.test(label)) {
|
|
9114
|
+
return { value: null, label: label || null };
|
|
9115
|
+
}
|
|
9116
|
+
const parsed = parseLinkedInNumber(label);
|
|
9117
|
+
return { value: parsed, label };
|
|
9118
|
+
}
|
|
9119
|
+
function parseLinkedInTenureYears(lines) {
|
|
9120
|
+
const tenureLine = lines.find((line) => /median employee tenure/i.test(line));
|
|
9121
|
+
return parseLinkedInNumber(tenureLine);
|
|
9122
|
+
}
|
|
9123
|
+
function isLikelyLinkedInGrowthToken(value) {
|
|
9124
|
+
if (!value) {
|
|
9125
|
+
return false;
|
|
9126
|
+
}
|
|
9127
|
+
return /^(?:▲\s*)?(?:[0-9][0-9,]*(?:\.[0-9]+)?%?\+?|N\/A)$/i.test(value.trim());
|
|
9128
|
+
}
|
|
9129
|
+
function parseLinkedInInsightRows(lines) {
|
|
9130
|
+
const insights = [];
|
|
9131
|
+
const jobOpeningsIndex = lines.findIndex((line) => /^Total job openings$/i.test(line));
|
|
9132
|
+
if (jobOpeningsIndex >= 0) {
|
|
9133
|
+
const totalValue = parseLinkedInNumber(lines[jobOpeningsIndex + 1]);
|
|
9134
|
+
const period = lines[jobOpeningsIndex + 2] || null;
|
|
9135
|
+
insights.push({
|
|
9136
|
+
insightType: "job_openings_total",
|
|
9137
|
+
label: "Total",
|
|
9138
|
+
period,
|
|
9139
|
+
metricValue: totalValue,
|
|
9140
|
+
metricUnit: "jobs",
|
|
9141
|
+
growth3mPercent: null,
|
|
9142
|
+
growth6mPercent: null,
|
|
9143
|
+
growth1yPercent: null,
|
|
9144
|
+
growth2yPercent: null,
|
|
9145
|
+
growth3mLabel: null,
|
|
9146
|
+
growth6mLabel: null,
|
|
9147
|
+
growth1yLabel: null,
|
|
9148
|
+
growth2yLabel: null,
|
|
9149
|
+
rawPayload: { source: "total_job_openings", lines: lines.slice(jobOpeningsIndex, jobOpeningsIndex + 12) },
|
|
9150
|
+
});
|
|
9151
|
+
const jobEndIndex = lines.findIndex((line, index) => index > jobOpeningsIndex && /^Total employee count$/i.test(line));
|
|
9152
|
+
const jobRows = lines.slice(jobOpeningsIndex + 3, jobEndIndex === -1 ? lines.length : jobEndIndex);
|
|
9153
|
+
for (let index = 0; index < jobRows.length - 2; index += 1) {
|
|
9154
|
+
const label = jobRows[index];
|
|
9155
|
+
const growth3m = jobRows[index + 1];
|
|
9156
|
+
const growth6m = jobRows[index + 2];
|
|
9157
|
+
if (!label || !isLikelyLinkedInGrowthToken(growth3m) || !isLikelyLinkedInGrowthToken(growth6m)) {
|
|
9158
|
+
continue;
|
|
9159
|
+
}
|
|
9160
|
+
const parsed3m = parseLinkedInGrowth(growth3m);
|
|
9161
|
+
const parsed6m = parseLinkedInGrowth(growth6m);
|
|
9162
|
+
insights.push({
|
|
9163
|
+
insightType: "job_openings_function",
|
|
9164
|
+
label,
|
|
9165
|
+
period,
|
|
9166
|
+
metricValue: null,
|
|
9167
|
+
metricUnit: "jobs",
|
|
9168
|
+
growth3mPercent: parsed3m.value,
|
|
9169
|
+
growth6mPercent: parsed6m.value,
|
|
9170
|
+
growth1yPercent: null,
|
|
9171
|
+
growth2yPercent: null,
|
|
9172
|
+
growth3mLabel: parsed3m.label,
|
|
9173
|
+
growth6mLabel: parsed6m.label,
|
|
9174
|
+
growth1yLabel: null,
|
|
9175
|
+
growth2yLabel: null,
|
|
9176
|
+
rawPayload: { source: "job_openings_function", label, growth3m, growth6m },
|
|
9177
|
+
});
|
|
9178
|
+
index += 2;
|
|
9179
|
+
}
|
|
9180
|
+
}
|
|
9181
|
+
const employeeCountIndex = lines.findIndex((line) => /^Total employee count$/i.test(line));
|
|
9182
|
+
if (employeeCountIndex >= 0) {
|
|
9183
|
+
const growth6m = parseLinkedInGrowth(lines[employeeCountIndex + 3]);
|
|
9184
|
+
const growth1y = parseLinkedInGrowth(lines[employeeCountIndex + 4]);
|
|
9185
|
+
const growth2y = parseLinkedInGrowth(lines[employeeCountIndex + 5]);
|
|
9186
|
+
insights.push({
|
|
9187
|
+
insightType: "employee_count_total",
|
|
9188
|
+
label: "Total",
|
|
9189
|
+
period: "current",
|
|
9190
|
+
metricValue: parseLinkedInNumber(lines[employeeCountIndex + 1]),
|
|
9191
|
+
metricUnit: "employees",
|
|
9192
|
+
growth3mPercent: null,
|
|
9193
|
+
growth6mPercent: growth6m.value,
|
|
9194
|
+
growth1yPercent: growth1y.value,
|
|
9195
|
+
growth2yPercent: growth2y.value,
|
|
9196
|
+
growth3mLabel: null,
|
|
9197
|
+
growth6mLabel: growth6m.label,
|
|
9198
|
+
growth1yLabel: growth1y.label,
|
|
9199
|
+
growth2yLabel: growth2y.label,
|
|
9200
|
+
rawPayload: { source: "total_employee_count", lines: lines.slice(employeeCountIndex, employeeCountIndex + 8) },
|
|
9201
|
+
});
|
|
9202
|
+
}
|
|
9203
|
+
const functionIndex = lines.findIndex((line) => /^Employee distribution and headcount growth by function$/i.test(line));
|
|
9204
|
+
if (functionIndex >= 0) {
|
|
9205
|
+
const period = lines[functionIndex + 2] || null;
|
|
9206
|
+
const functionEndIndex = lines.findIndex((line, index) => index > functionIndex && /^Select insights not available$/i.test(line));
|
|
9207
|
+
const functionRows = lines.slice(functionIndex + 3, functionEndIndex === -1 ? lines.length : functionEndIndex);
|
|
9208
|
+
for (let index = 0; index < functionRows.length - 2; index += 1) {
|
|
9209
|
+
const label = functionRows[index];
|
|
9210
|
+
const growth6m = functionRows[index + 1];
|
|
9211
|
+
const growth1y = functionRows[index + 2];
|
|
9212
|
+
if (!label || !isLikelyLinkedInGrowthToken(growth6m) || !isLikelyLinkedInGrowthToken(growth1y)) {
|
|
9213
|
+
continue;
|
|
9214
|
+
}
|
|
9215
|
+
const parsed6m = parseLinkedInGrowth(growth6m);
|
|
9216
|
+
const parsed1y = parseLinkedInGrowth(growth1y);
|
|
9217
|
+
insights.push({
|
|
9218
|
+
insightType: "employee_function_distribution",
|
|
9219
|
+
label,
|
|
9220
|
+
period,
|
|
9221
|
+
metricValue: null,
|
|
9222
|
+
metricUnit: "employees",
|
|
9223
|
+
growth3mPercent: null,
|
|
9224
|
+
growth6mPercent: parsed6m.value,
|
|
9225
|
+
growth1yPercent: parsed1y.value,
|
|
9226
|
+
growth2yPercent: null,
|
|
9227
|
+
growth3mLabel: null,
|
|
9228
|
+
growth6mLabel: parsed6m.label,
|
|
9229
|
+
growth1yLabel: parsed1y.label,
|
|
9230
|
+
growth2yLabel: null,
|
|
9231
|
+
rawPayload: { source: "employee_function_distribution", label, growth6m, growth1y },
|
|
9232
|
+
});
|
|
9233
|
+
index += 2;
|
|
9234
|
+
}
|
|
9235
|
+
}
|
|
9236
|
+
const unavailableIndex = lines.findIndex((line) => /^Select insights not available$/i.test(line));
|
|
9237
|
+
if (unavailableIndex >= 0) {
|
|
9238
|
+
const detail = lines[unavailableIndex + 1] || null;
|
|
9239
|
+
insights.push({
|
|
9240
|
+
insightType: "unavailable_insight",
|
|
9241
|
+
label: detail,
|
|
9242
|
+
period: null,
|
|
9243
|
+
metricValue: null,
|
|
9244
|
+
metricUnit: null,
|
|
9245
|
+
growth3mPercent: null,
|
|
9246
|
+
growth6mPercent: null,
|
|
9247
|
+
growth1yPercent: null,
|
|
9248
|
+
growth2yPercent: null,
|
|
9249
|
+
growth3mLabel: null,
|
|
9250
|
+
growth6mLabel: null,
|
|
9251
|
+
growth1yLabel: null,
|
|
9252
|
+
growth2yLabel: null,
|
|
9253
|
+
rawPayload: { source: "select_insights_not_available", detail },
|
|
9254
|
+
});
|
|
9255
|
+
}
|
|
9256
|
+
return insights;
|
|
9257
|
+
}
|
|
9258
|
+
function parseLocalLinkedInCompanyText(text, rawUrl, insightsText = "") {
|
|
9259
|
+
const urls = normalizeLinkedInCompanyScrapeUrl(rawUrl);
|
|
9260
|
+
const lines = text
|
|
9261
|
+
.split(/\r?\n/)
|
|
9262
|
+
.map((line) => line.replace(/\s+/g, " ").trim())
|
|
9263
|
+
.filter(Boolean);
|
|
9264
|
+
const insightLines = insightsText
|
|
9265
|
+
.split(/\r?\n/)
|
|
9266
|
+
.map((line) => line.replace(/\s+/g, " ").trim())
|
|
9267
|
+
.filter(Boolean);
|
|
9268
|
+
const website = parseLinkedInAboutTextValue(lines, "Website");
|
|
9269
|
+
const industry = parseLinkedInAboutTextValue(lines, "Industry");
|
|
9270
|
+
const companySize = parseLinkedInAboutTextValue(lines, "Company size");
|
|
9271
|
+
const associatedMembersText = lines.find((line) => /associated members\b/i.test(line)) ?? null;
|
|
9272
|
+
const founded = parseLinkedInCompactInteger(parseLinkedInAboutTextValue(lines, "Founded"));
|
|
9273
|
+
const headquarters = parseLinkedInAboutTextValue(lines, "Headquarters");
|
|
9274
|
+
const overviewIndex = lines.findIndex((line) => line.toLowerCase() === "overview");
|
|
9275
|
+
const description = overviewIndex >= 0
|
|
9276
|
+
? lines
|
|
9277
|
+
.slice(overviewIndex + 1)
|
|
9278
|
+
.find((line) => !["Website", "Industry", "Company size", "Founded", "Locations (1)", "Headquarters"].includes(line)) ?? null
|
|
9279
|
+
: null;
|
|
9280
|
+
const name = lines.find((line, index) => {
|
|
9281
|
+
if (index > 12) {
|
|
9282
|
+
return false;
|
|
9283
|
+
}
|
|
9284
|
+
return (!["Home", "About", "Posts", "Jobs", "People", "Insights", "Message", "Following", "Follow"].includes(line) &&
|
|
9285
|
+
!line.includes("followers") &&
|
|
9286
|
+
!line.includes("employees") &&
|
|
9287
|
+
!line.includes("connections"));
|
|
9288
|
+
}) ?? null;
|
|
9289
|
+
const insights = parseLinkedInInsightRows(insightLines);
|
|
9290
|
+
return {
|
|
9291
|
+
...urls,
|
|
9292
|
+
name,
|
|
9293
|
+
website,
|
|
9294
|
+
domain: normalizeLocalCompanyDomain(website),
|
|
9295
|
+
industry,
|
|
9296
|
+
companySize,
|
|
9297
|
+
employeesOnLinkedIn: parseLinkedInCompactInteger(associatedMembersText),
|
|
9298
|
+
totalEmployeeCount: parseLinkedInCompactInteger(associatedMembersText),
|
|
9299
|
+
followerCount: parseLinkedInCompactInteger(lines.find((line) => /followers\b/i.test(line))),
|
|
9300
|
+
headquarters,
|
|
9301
|
+
founded,
|
|
9302
|
+
description,
|
|
9303
|
+
medianEmployeeTenureYears: parseLinkedInTenureYears(insightLines),
|
|
9304
|
+
insights,
|
|
9305
|
+
rawPayload: {
|
|
9306
|
+
sourceTextLines: lines,
|
|
9307
|
+
insightsTextLines: insightLines,
|
|
9308
|
+
},
|
|
9309
|
+
timestamp: new Date().toISOString(),
|
|
9310
|
+
};
|
|
9311
|
+
}
|
|
9312
|
+
function parseLocalDealroomCompanyText(text, rawUrl) {
|
|
9313
|
+
const urls = normalizeDealroomCompanyScrapeUrl(rawUrl);
|
|
9314
|
+
const lines = text
|
|
9315
|
+
.split(/\r?\n/)
|
|
9316
|
+
.map((line) => line.replace(/\s+/g, " ").trim())
|
|
9317
|
+
.filter(Boolean);
|
|
9318
|
+
const reservedTopLines = new Set([
|
|
9319
|
+
"dealroom.co",
|
|
9320
|
+
"Dashboard",
|
|
9321
|
+
"News",
|
|
9322
|
+
"Companies",
|
|
9323
|
+
"Stats & Insights",
|
|
9324
|
+
"Sectors",
|
|
9325
|
+
"Locations",
|
|
9326
|
+
"Investors",
|
|
9327
|
+
"Transactions",
|
|
9328
|
+
"Public Multiples",
|
|
9329
|
+
"People",
|
|
9330
|
+
"More organizations",
|
|
9331
|
+
"Deep Dives",
|
|
9332
|
+
"Knowledge base",
|
|
9333
|
+
"Login",
|
|
9334
|
+
"Book a Demo",
|
|
9335
|
+
"Save",
|
|
9336
|
+
"Overview",
|
|
9337
|
+
"Similar companies",
|
|
9338
|
+
"Investments (1)",
|
|
9339
|
+
"Job openings",
|
|
9340
|
+
]);
|
|
9341
|
+
const name = lines.find((line) => {
|
|
9342
|
+
if (reservedTopLines.has(line)) {
|
|
9343
|
+
return false;
|
|
9344
|
+
}
|
|
9345
|
+
if (/search for companies/i.test(line)) {
|
|
9346
|
+
return false;
|
|
9347
|
+
}
|
|
9348
|
+
return true;
|
|
9349
|
+
}) ?? null;
|
|
9350
|
+
const tagline = name ? lines[lines.indexOf(name) + 1] ?? null : null;
|
|
9351
|
+
const website = normalizeDealroomWebsite(parseLabeledTextValue(lines, "Website"));
|
|
9352
|
+
const recentDealText = parseLabeledTextValue(lines, "Recent deals");
|
|
9353
|
+
const signalLine = lines.find((line) => /^\d{1,3}$/.test(line) && Number(line) <= 100 && lines[lines.indexOf(line) + 1] === "Dealroom.co signal");
|
|
9354
|
+
const sector = lines.find((line) => /#\d+\)?$/i.test(line) && !/^\d+$/.test(line)) ?? null;
|
|
9355
|
+
const knownLabels = new Set([
|
|
9356
|
+
"HQ location",
|
|
9357
|
+
"Website",
|
|
9358
|
+
"Launch date",
|
|
9359
|
+
"Employees",
|
|
9360
|
+
"Enterprise value",
|
|
9361
|
+
"Company register number",
|
|
9362
|
+
"Jobs",
|
|
9363
|
+
"Recent deals",
|
|
9364
|
+
]);
|
|
9365
|
+
const nonTagValues = new Set();
|
|
9366
|
+
for (const label of knownLabels) {
|
|
9367
|
+
const value = parseLabeledTextValue(lines, label);
|
|
9368
|
+
if (value) {
|
|
9369
|
+
nonTagValues.add(value);
|
|
9370
|
+
}
|
|
9371
|
+
}
|
|
9372
|
+
const tagStartIndex = lines.findIndex((line) => /^B2B$/i.test(line) || /^\$?\s*saas$/i.test(line));
|
|
9373
|
+
const tagEndIndex = sector ? lines.indexOf(sector) : -1;
|
|
9374
|
+
const tags = tagStartIndex >= 0
|
|
9375
|
+
? lines
|
|
9376
|
+
.slice(tagStartIndex, tagEndIndex > tagStartIndex ? tagEndIndex : undefined)
|
|
9377
|
+
.filter((line) => !knownLabels.has(line) && !nonTagValues.has(line))
|
|
9378
|
+
: [];
|
|
9379
|
+
const socialLinks = {};
|
|
9380
|
+
for (const line of lines) {
|
|
9381
|
+
if (/^https?:\/\/(www\.)?linkedin\.com\//i.test(line)) {
|
|
9382
|
+
socialLinks.linkedin = line;
|
|
9383
|
+
}
|
|
9384
|
+
else if (/^https?:\/\/(www\.)?(x|twitter)\.com\//i.test(line)) {
|
|
9385
|
+
socialLinks.x = line;
|
|
9386
|
+
}
|
|
9387
|
+
}
|
|
9388
|
+
const enterpriseValueText = parseLabeledTextValue(lines, "Enterprise value");
|
|
9389
|
+
return {
|
|
9390
|
+
...urls,
|
|
9391
|
+
name,
|
|
9392
|
+
tagline,
|
|
9393
|
+
hqLocation: parseLabeledTextValue(lines, "HQ location"),
|
|
9394
|
+
website,
|
|
9395
|
+
domain: normalizeLocalCompanyDomain(website),
|
|
9396
|
+
launchDateText: parseLabeledTextValue(lines, "Launch date"),
|
|
9397
|
+
employeeRange: parseLabeledTextValue(lines, "Employees"),
|
|
9398
|
+
enterpriseValueText,
|
|
9399
|
+
enterpriseValueUsd: parseDealroomEnterpriseValueUsd(enterpriseValueText),
|
|
9400
|
+
companyRegisterNumber: parseLabeledTextValue(lines, "Company register number"),
|
|
9401
|
+
jobsText: parseLabeledTextValue(lines, "Jobs"),
|
|
9402
|
+
recentDeals: recentDealText ? [recentDealText] : [],
|
|
9403
|
+
tags,
|
|
9404
|
+
sector,
|
|
9405
|
+
signalScore: signalLine ? Number(signalLine) : null,
|
|
9406
|
+
socialLinks,
|
|
9407
|
+
rawPayload: { sourceTextLines: lines },
|
|
9408
|
+
timestamp: new Date().toISOString(),
|
|
9409
|
+
};
|
|
9410
|
+
}
|
|
9411
|
+
async function upsertLocalLinkedInCompanyScrape(row) {
|
|
9412
|
+
const databaseUrl = process.env.SALESPROMPTER_DATABASE_URL?.trim() ||
|
|
9413
|
+
process.env.DATABASE_URL?.trim() ||
|
|
9414
|
+
"";
|
|
9415
|
+
if (!databaseUrl) {
|
|
9416
|
+
throw new Error("Missing DATABASE_URL or SALESPROMPTER_DATABASE_URL for LinkedIn company scraper storage.");
|
|
9417
|
+
}
|
|
9418
|
+
const client = new pg.Client({
|
|
9419
|
+
connectionString: databaseUrl,
|
|
9420
|
+
ssl: { rejectUnauthorized: false },
|
|
9421
|
+
});
|
|
9422
|
+
await client.connect();
|
|
9423
|
+
let committed = false;
|
|
9424
|
+
try {
|
|
9425
|
+
await client.query("BEGIN");
|
|
9426
|
+
await client.query(`INSERT INTO public.salesprompter_linkedin_companies_processed (
|
|
9427
|
+
error, query, timestamp, name, description, follower_count, website, domain, industry,
|
|
9428
|
+
company_size, founded, company_address, main_company_id, linkedin_id, sales_navigator_link,
|
|
9429
|
+
employees_on_linked_in, company_name, total_employee_count, company_url, headquarters
|
|
9430
|
+
) VALUES (
|
|
9431
|
+
NULL, $1, $2, $3, $4, $5, $6, $7, $8,
|
|
9432
|
+
$9, $10, $11, $12, $13, $14,
|
|
9433
|
+
$15, $16, $17, $18, $19
|
|
9434
|
+
)`, [
|
|
9435
|
+
`linkedin.com/company/${row.id}`,
|
|
9436
|
+
row.timestamp,
|
|
9437
|
+
row.name,
|
|
9438
|
+
row.description,
|
|
9439
|
+
row.followerCount,
|
|
9440
|
+
row.website,
|
|
9441
|
+
row.domain,
|
|
9442
|
+
row.industry,
|
|
9443
|
+
row.companySize,
|
|
9444
|
+
row.founded,
|
|
9445
|
+
row.headquarters,
|
|
9446
|
+
row.id,
|
|
9447
|
+
String(row.id),
|
|
9448
|
+
row.salesNavigatorLink,
|
|
9449
|
+
row.employeesOnLinkedIn,
|
|
9450
|
+
row.name,
|
|
9451
|
+
row.totalEmployeeCount,
|
|
9452
|
+
row.companyUrl,
|
|
9453
|
+
row.headquarters,
|
|
9454
|
+
]);
|
|
9455
|
+
const snapshotResult = await client.query(`INSERT INTO public.linkedin_company_local_scrape_snapshots (
|
|
9456
|
+
linkedin_company_id, linkedin_company_url, sales_navigator_link, company_name,
|
|
9457
|
+
scrape_source, scraped_at, observed_at, overview_text, website, domain, industry,
|
|
9458
|
+
company_size, employees_on_linkedin, total_employee_count, follower_count,
|
|
9459
|
+
headquarters, founded, median_employee_tenure_years, raw_payload
|
|
9460
|
+
) VALUES (
|
|
9461
|
+
$1, $2, $3, $4,
|
|
9462
|
+
'linkedin_company_scraper_local', $5, $5, $6, $7, $8, $9,
|
|
9463
|
+
$10, $11, $12, $13,
|
|
9464
|
+
$14, $15, $16, $17::jsonb
|
|
9465
|
+
)
|
|
9466
|
+
RETURNING id`, [
|
|
9467
|
+
row.id,
|
|
9468
|
+
row.companyUrl,
|
|
9469
|
+
row.salesNavigatorLink,
|
|
9470
|
+
row.name,
|
|
9471
|
+
row.timestamp,
|
|
9472
|
+
row.description,
|
|
9473
|
+
row.website,
|
|
9474
|
+
row.domain,
|
|
9475
|
+
row.industry,
|
|
9476
|
+
row.companySize,
|
|
9477
|
+
row.employeesOnLinkedIn,
|
|
9478
|
+
row.totalEmployeeCount,
|
|
9479
|
+
row.followerCount,
|
|
9480
|
+
row.headquarters,
|
|
9481
|
+
row.founded,
|
|
9482
|
+
row.medianEmployeeTenureYears,
|
|
9483
|
+
JSON.stringify(sanitizeForPostgresJson(row.rawPayload)),
|
|
9484
|
+
]);
|
|
9485
|
+
const snapshotId = snapshotResult.rows[0]?.id;
|
|
9486
|
+
if (!snapshotId) {
|
|
9487
|
+
throw new Error("Failed to create LinkedIn company local scrape snapshot.");
|
|
9488
|
+
}
|
|
9489
|
+
for (const insight of row.insights) {
|
|
9490
|
+
await client.query(`INSERT INTO public.linkedin_company_local_scrape_insights (
|
|
9491
|
+
snapshot_id, linkedin_company_id, insight_type, label, period, metric_value,
|
|
9492
|
+
metric_unit, growth_3m_percent, growth_6m_percent, growth_1y_percent,
|
|
9493
|
+
growth_2y_percent, growth_3m_label, growth_6m_label, growth_1y_label,
|
|
9494
|
+
growth_2y_label, raw_payload, scraped_at
|
|
9495
|
+
) VALUES (
|
|
9496
|
+
$1, $2, $3, $4, $5, $6,
|
|
9497
|
+
$7, $8, $9, $10,
|
|
9498
|
+
$11, $12, $13, $14,
|
|
9499
|
+
$15, $16::jsonb, $17
|
|
9500
|
+
)`, [
|
|
9501
|
+
snapshotId,
|
|
9502
|
+
row.id,
|
|
9503
|
+
insight.insightType,
|
|
9504
|
+
insight.label,
|
|
9505
|
+
insight.period,
|
|
9506
|
+
insight.metricValue,
|
|
9507
|
+
insight.metricUnit,
|
|
9508
|
+
insight.growth3mPercent,
|
|
9509
|
+
insight.growth6mPercent,
|
|
9510
|
+
insight.growth1yPercent,
|
|
9511
|
+
insight.growth2yPercent,
|
|
9512
|
+
insight.growth3mLabel,
|
|
9513
|
+
insight.growth6mLabel,
|
|
9514
|
+
insight.growth1yLabel,
|
|
9515
|
+
insight.growth2yLabel,
|
|
9516
|
+
JSON.stringify(sanitizeForPostgresJson(insight.rawPayload)),
|
|
9517
|
+
row.timestamp,
|
|
9518
|
+
]);
|
|
9519
|
+
}
|
|
9520
|
+
await client.query("COMMIT");
|
|
9521
|
+
committed = true;
|
|
9522
|
+
}
|
|
9523
|
+
finally {
|
|
9524
|
+
if (!committed) {
|
|
9525
|
+
try {
|
|
9526
|
+
await client.query("ROLLBACK");
|
|
9527
|
+
}
|
|
9528
|
+
catch {
|
|
9529
|
+
// The transaction may already be closed by the database.
|
|
9530
|
+
}
|
|
9531
|
+
}
|
|
9532
|
+
await client.end();
|
|
9533
|
+
}
|
|
9534
|
+
}
|
|
9535
|
+
async function insertLocalDealroomCompanyScrape(row) {
|
|
9536
|
+
const databaseUrl = process.env.SALESPROMPTER_DATABASE_URL?.trim() ||
|
|
9537
|
+
process.env.DATABASE_URL?.trim() ||
|
|
9538
|
+
"";
|
|
9539
|
+
if (!databaseUrl) {
|
|
9540
|
+
throw new Error("Missing DATABASE_URL or SALESPROMPTER_DATABASE_URL for Dealroom company scraper storage.");
|
|
9541
|
+
}
|
|
9542
|
+
const client = new pg.Client({
|
|
9543
|
+
connectionString: databaseUrl,
|
|
9544
|
+
ssl: { rejectUnauthorized: false },
|
|
9545
|
+
});
|
|
9546
|
+
await client.connect();
|
|
9547
|
+
let committed = false;
|
|
9548
|
+
try {
|
|
9549
|
+
await client.query("BEGIN");
|
|
9550
|
+
await client.query(`INSERT INTO public.dealroom_company_local_scrape_snapshots (
|
|
9551
|
+
dealroom_company_slug, dealroom_company_url, source_url, scrape_source,
|
|
9552
|
+
scraped_at, observed_at, company_name, tagline, hq_location, website,
|
|
9553
|
+
domain, launch_date_text, employee_range, enterprise_value_text,
|
|
9554
|
+
enterprise_value_usd, company_register_number, jobs_text, recent_deals,
|
|
9555
|
+
tags, sector, signal_score, social_links, raw_payload
|
|
9556
|
+
) VALUES (
|
|
9557
|
+
$1, $2, $3, 'dealroom_company_scraper_local',
|
|
9558
|
+
$4, $4, $5, $6, $7, $8,
|
|
9559
|
+
$9, $10, $11, $12,
|
|
9560
|
+
$13, $14, $15, $16,
|
|
9561
|
+
$17, $18, $19, $20::jsonb, $21::jsonb
|
|
9562
|
+
)`, [
|
|
9563
|
+
row.slug,
|
|
9564
|
+
row.companyUrl,
|
|
9565
|
+
row.sourceUrl,
|
|
9566
|
+
row.timestamp,
|
|
9567
|
+
row.name,
|
|
9568
|
+
row.tagline,
|
|
9569
|
+
row.hqLocation,
|
|
9570
|
+
row.website,
|
|
9571
|
+
row.domain,
|
|
9572
|
+
row.launchDateText,
|
|
9573
|
+
row.employeeRange,
|
|
9574
|
+
row.enterpriseValueText,
|
|
9575
|
+
row.enterpriseValueUsd,
|
|
9576
|
+
row.companyRegisterNumber,
|
|
9577
|
+
row.jobsText,
|
|
9578
|
+
row.recentDeals,
|
|
9579
|
+
row.tags,
|
|
9580
|
+
row.sector,
|
|
9581
|
+
row.signalScore,
|
|
9582
|
+
JSON.stringify(sanitizeForPostgresJson(row.socialLinks)),
|
|
9583
|
+
JSON.stringify(sanitizeForPostgresJson(row.rawPayload)),
|
|
9584
|
+
]);
|
|
9585
|
+
await client.query("COMMIT");
|
|
9586
|
+
committed = true;
|
|
9587
|
+
}
|
|
9588
|
+
finally {
|
|
9589
|
+
if (!committed) {
|
|
9590
|
+
try {
|
|
9591
|
+
await client.query("ROLLBACK");
|
|
9592
|
+
}
|
|
9593
|
+
catch {
|
|
9594
|
+
// The transaction may already be closed by the database.
|
|
9595
|
+
}
|
|
9596
|
+
}
|
|
9597
|
+
await client.end();
|
|
9598
|
+
}
|
|
9599
|
+
}
|
|
7368
9600
|
async function upsertSalesNavigatorAccounts(params) {
|
|
7369
9601
|
if (params.accounts.length === 0) {
|
|
7370
9602
|
return 0;
|
|
@@ -7397,7 +9629,7 @@ async function upsertSalesNavigatorAccounts(params) {
|
|
|
7397
9629
|
}));
|
|
7398
9630
|
if (params.store.kind === "postgres") {
|
|
7399
9631
|
for (const row of rows) {
|
|
7400
|
-
await params.store.client.query(`INSERT INTO public.
|
|
9632
|
+
await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_accounts (
|
|
7401
9633
|
org_id, last_run_id, last_slice_id, account_key, sales_nav_company_url,
|
|
7402
9634
|
linkedin_company_url, company_id, company_name, industry, location, headquarters,
|
|
7403
9635
|
employee_count, follower_count, company_type, description, website, search_query_url,
|
|
@@ -7446,16 +9678,61 @@ async function upsertSalesNavigatorAccounts(params) {
|
|
|
7446
9678
|
row.scraped_at,
|
|
7447
9679
|
JSON.stringify(row.raw_payload),
|
|
7448
9680
|
]);
|
|
9681
|
+
await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_account_run_memberships (
|
|
9682
|
+
run_id, org_id, account_key, last_slice_id
|
|
9683
|
+
)
|
|
9684
|
+
VALUES ($1, $2, $3, $4)
|
|
9685
|
+
ON CONFLICT (run_id, account_key)
|
|
9686
|
+
DO UPDATE SET
|
|
9687
|
+
last_slice_id = excluded.last_slice_id,
|
|
9688
|
+
updated_at = now()`, [
|
|
9689
|
+
row.last_run_id,
|
|
9690
|
+
row.org_id,
|
|
9691
|
+
row.account_key,
|
|
9692
|
+
row.last_slice_id,
|
|
9693
|
+
]);
|
|
7449
9694
|
}
|
|
7450
9695
|
return rows.length;
|
|
7451
9696
|
}
|
|
7452
|
-
const
|
|
7453
|
-
|
|
7454
|
-
.
|
|
9697
|
+
const supabaseBatchSize = 250;
|
|
9698
|
+
for (let index = 0; index < rows.length; index += supabaseBatchSize) {
|
|
9699
|
+
const batch = rows.slice(index, index + supabaseBatchSize);
|
|
9700
|
+
const { error } = await params.store.client
|
|
9701
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchAccounts)
|
|
9702
|
+
.upsert(batch, { onConflict: "org_id,account_key" });
|
|
9703
|
+
if (error) {
|
|
9704
|
+
throw new Error(`Failed to upsert Sales Navigator accounts in Supabase: ${error.message}`);
|
|
9705
|
+
}
|
|
9706
|
+
const { error: membershipError } = await params.store.client
|
|
9707
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountRunMemberships)
|
|
9708
|
+
.upsert(batch.map((row) => ({
|
|
9709
|
+
run_id: row.last_run_id,
|
|
9710
|
+
org_id: row.org_id,
|
|
9711
|
+
account_key: row.account_key,
|
|
9712
|
+
last_slice_id: row.last_slice_id,
|
|
9713
|
+
updated_at: new Date().toISOString(),
|
|
9714
|
+
})), { onConflict: "run_id,account_key" });
|
|
9715
|
+
if (membershipError) {
|
|
9716
|
+
throw new Error(`Failed to record Sales Navigator account run membership in Supabase: ${membershipError.message}`);
|
|
9717
|
+
}
|
|
9718
|
+
}
|
|
9719
|
+
return rows.length;
|
|
9720
|
+
}
|
|
9721
|
+
async function countSalesNavigatorAccountsForRun(params) {
|
|
9722
|
+
if (params.store.kind === "postgres") {
|
|
9723
|
+
const result = await params.store.client.query(`SELECT count(*)::text AS count
|
|
9724
|
+
FROM public.salesprompter_linkedin_sales_nav_account_run_memberships
|
|
9725
|
+
WHERE run_id = $1`, [params.runId]);
|
|
9726
|
+
return Number(result.rows[0]?.count ?? 0);
|
|
9727
|
+
}
|
|
9728
|
+
const { count, error } = await params.store.client
|
|
9729
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountRunMemberships)
|
|
9730
|
+
.select("account_key", { count: "exact", head: true })
|
|
9731
|
+
.eq("run_id", params.runId);
|
|
7455
9732
|
if (error) {
|
|
7456
|
-
throw new Error(`Failed to
|
|
9733
|
+
throw new Error(`Failed to count Sales Navigator account run membership in Supabase: ${error.message}`);
|
|
7457
9734
|
}
|
|
7458
|
-
return
|
|
9735
|
+
return count ?? 0;
|
|
7459
9736
|
}
|
|
7460
9737
|
function asJsonObject(value) {
|
|
7461
9738
|
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
@@ -7515,8 +9792,8 @@ async function listSalesNavigatorAccountSearchRepairSlices(params) {
|
|
|
7515
9792
|
runs.slice_preset,
|
|
7516
9793
|
runs.max_results_per_search,
|
|
7517
9794
|
runs.page_size
|
|
7518
|
-
FROM public.
|
|
7519
|
-
INNER JOIN public.
|
|
9795
|
+
FROM public.salesprompter_linkedin_sales_nav_account_search_slices AS slices
|
|
9796
|
+
INNER JOIN public.salesprompter_linkedin_sales_nav_account_search_runs AS runs
|
|
7520
9797
|
ON runs.id = slices.run_id
|
|
7521
9798
|
WHERE slices.run_id = $1
|
|
7522
9799
|
AND slices.status IN ('failed', 'unresolved')
|
|
@@ -7548,32 +9825,52 @@ async function listSalesNavigatorAccountSearchRepairSlices(params) {
|
|
|
7548
9825
|
}));
|
|
7549
9826
|
}
|
|
7550
9827
|
async function summarizeSalesNavigatorAccountSearchRunFromSlices(params) {
|
|
7551
|
-
if (params.store.kind
|
|
7552
|
-
|
|
9828
|
+
if (params.store.kind === "supabase") {
|
|
9829
|
+
const { data, error } = await params.store.client
|
|
9830
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchSlices)
|
|
9831
|
+
.select("status,last_error,updated_at")
|
|
9832
|
+
.eq("run_id", params.runId);
|
|
9833
|
+
if (error) {
|
|
9834
|
+
throw new Error(`Failed to summarize Sales Navigator account slices in Supabase: ${error.message}`);
|
|
9835
|
+
}
|
|
9836
|
+
const rows = (data ?? []);
|
|
9837
|
+
const finalRows = rows.filter((row) => row.status !== "split");
|
|
9838
|
+
const failedRows = rows.filter((row) => row.status === "failed");
|
|
9839
|
+
const unresolvedRows = rows.filter((row) => row.status === "unresolved");
|
|
9840
|
+
const latestProblem = [...failedRows, ...unresolvedRows].sort((a, b) => Date.parse(b.updated_at) - Date.parse(a.updated_at))[0];
|
|
9841
|
+
return {
|
|
9842
|
+
totalSlices: finalRows.length,
|
|
9843
|
+
storedAccounts: await countSalesNavigatorAccountsForRun(params),
|
|
9844
|
+
attemptedSlices: finalRows.length,
|
|
9845
|
+
splitEvents: rows.filter((row) => row.status === "split").length,
|
|
9846
|
+
failedSlices: failedRows.length,
|
|
9847
|
+
unresolvedSlices: unresolvedRows.length,
|
|
9848
|
+
truncated: failedRows.length > 0 || unresolvedRows.length > 0,
|
|
9849
|
+
lastError: latestProblem?.last_error ?? null,
|
|
9850
|
+
};
|
|
7553
9851
|
}
|
|
7554
9852
|
const result = await params.store.client.query(`SELECT
|
|
7555
9853
|
count(*) FILTER (WHERE status <> 'split')::int AS total_slices,
|
|
7556
|
-
COALESCE(sum(stored_accounts) FILTER (WHERE status <> 'split'), 0)::int AS stored_accounts,
|
|
7557
9854
|
count(*) FILTER (WHERE status <> 'split')::int AS attempted_slices,
|
|
7558
9855
|
count(*) FILTER (WHERE status = 'split')::int AS split_events,
|
|
7559
9856
|
count(*) FILTER (WHERE status = 'failed')::int AS failed_slices,
|
|
7560
9857
|
count(*) FILTER (WHERE status = 'unresolved')::int AS unresolved_slices,
|
|
7561
9858
|
(
|
|
7562
9859
|
SELECT last_error
|
|
7563
|
-
FROM public.
|
|
9860
|
+
FROM public.salesprompter_linkedin_sales_nav_account_search_slices
|
|
7564
9861
|
WHERE run_id = $1
|
|
7565
9862
|
AND status IN ('failed', 'unresolved')
|
|
7566
9863
|
ORDER BY updated_at DESC
|
|
7567
9864
|
LIMIT 1
|
|
7568
9865
|
) AS last_error
|
|
7569
|
-
FROM public.
|
|
9866
|
+
FROM public.salesprompter_linkedin_sales_nav_account_search_slices
|
|
7570
9867
|
WHERE run_id = $1`, [params.runId]);
|
|
7571
9868
|
const row = result.rows[0] ?? {};
|
|
7572
9869
|
const failedSlices = Number(row.failed_slices ?? 0);
|
|
7573
9870
|
const unresolvedSlices = Number(row.unresolved_slices ?? 0);
|
|
7574
9871
|
return {
|
|
7575
9872
|
totalSlices: Number(row.total_slices ?? 0),
|
|
7576
|
-
storedAccounts:
|
|
9873
|
+
storedAccounts: await countSalesNavigatorAccountsForRun(params),
|
|
7577
9874
|
attemptedSlices: Number(row.attempted_slices ?? 0),
|
|
7578
9875
|
splitEvents: Number(row.split_events ?? 0),
|
|
7579
9876
|
failedSlices,
|
|
@@ -10599,6 +12896,10 @@ program.hook("preAction", async (_thisCommand, actionCommand) => {
|
|
|
10599
12896
|
commandName === "accounts:local-import" ||
|
|
10600
12897
|
commandName === "salesnav:accounts:repair" ||
|
|
10601
12898
|
commandName === "accounts:repair" ||
|
|
12899
|
+
commandName === "linkedin-companies:scrape-local" ||
|
|
12900
|
+
commandName === "companies:scrape-linkedin" ||
|
|
12901
|
+
commandName === "dealroom-companies:scrape-local" ||
|
|
12902
|
+
commandName === "companies:scrape-dealroom" ||
|
|
10602
12903
|
commandName.startsWith("packs:") ||
|
|
10603
12904
|
((commandName === "list" || commandName === "add") && parentCommandName === "packs")) {
|
|
10604
12905
|
return;
|
|
@@ -10865,6 +13166,187 @@ program
|
|
|
10865
13166
|
}
|
|
10866
13167
|
});
|
|
10867
13168
|
});
|
|
13169
|
+
async function runAffiliateLaunchCommand(options) {
|
|
13170
|
+
const affiliateLink = z.string().url().parse(options.affiliateLink);
|
|
13171
|
+
const linkedInUrl = z.string().url().parse(options.linkedinUrl);
|
|
13172
|
+
const maxResults = z.coerce.number().int().min(1).max(1000).parse(options.maxResults);
|
|
13173
|
+
const session = await requireAuthSession();
|
|
13174
|
+
const isSalesNavigatorPeopleSearch = new URL(linkedInUrl).pathname.includes("/sales/search/people");
|
|
13175
|
+
let localResults;
|
|
13176
|
+
let localCollection;
|
|
13177
|
+
if (!options.dryRun && isSalesNavigatorPeopleSearch) {
|
|
13178
|
+
const pageSize = z.coerce.number().int().min(1).max(100).parse(options.pageSize);
|
|
13179
|
+
const pageDelayMinMs = z.coerce.number().int().min(0).parse(options.pageDelayMinMs);
|
|
13180
|
+
const pageDelayMaxMs = z.coerce.number().int().min(pageDelayMinMs).parse(options.pageDelayMaxMs);
|
|
13181
|
+
const browserRelayPort = options.browserRelayPort == null
|
|
13182
|
+
? null
|
|
13183
|
+
: z.coerce.number().int().min(0).max(65535).parse(options.browserRelayPort);
|
|
13184
|
+
if (options.curlFile && browserRelayPort != null) {
|
|
13185
|
+
throw new Error("Use either --curl-file or --browser-relay-port, not both.");
|
|
13186
|
+
}
|
|
13187
|
+
const parsedRequest = options.curlFile
|
|
13188
|
+
? parseSalesNavigatorCurlRequest(await readFile(path.resolve(String(options.curlFile)), "utf8"))
|
|
13189
|
+
: browserRelayPort != null
|
|
13190
|
+
? {
|
|
13191
|
+
url: buildSalesNavigatorLeadApiUrlFromSearchUrl(linkedInUrl, pageSize),
|
|
13192
|
+
headers: {},
|
|
13193
|
+
}
|
|
13194
|
+
: buildSalesNavigatorApiRequestFromSearchUrl(linkedInUrl, await readLinkedInDirectLookupConfig(), pageSize);
|
|
13195
|
+
if (!new URL(parsedRequest.url).pathname.includes("/sales-api/salesApiLeadSearch")) {
|
|
13196
|
+
throw new Error("Affiliate --curl-file must contain a /sales-api/salesApiLeadSearch request.");
|
|
13197
|
+
}
|
|
13198
|
+
const browserRelay = browserRelayPort == null
|
|
13199
|
+
? null
|
|
13200
|
+
: await createLocalAccountSearchBrowserRelay(browserRelayPort);
|
|
13201
|
+
let collected;
|
|
13202
|
+
try {
|
|
13203
|
+
collected = await fetchAllLocalSalesNavigatorPeople(parsedRequest, {
|
|
13204
|
+
requestedProfiles: maxResults,
|
|
13205
|
+
pageSize,
|
|
13206
|
+
pageDelayMinMs,
|
|
13207
|
+
pageDelayMaxMs,
|
|
13208
|
+
retry: {
|
|
13209
|
+
maxRetries: 2,
|
|
13210
|
+
retryBaseDelayMs: 2_000,
|
|
13211
|
+
retryMaxDelayMs: 10_000,
|
|
13212
|
+
},
|
|
13213
|
+
executeRequest: browserRelay
|
|
13214
|
+
? (request) => browserRelay.request(request.url)
|
|
13215
|
+
: undefined,
|
|
13216
|
+
});
|
|
13217
|
+
}
|
|
13218
|
+
finally {
|
|
13219
|
+
await browserRelay?.close();
|
|
13220
|
+
}
|
|
13221
|
+
localResults = collected.people;
|
|
13222
|
+
localCollection = {
|
|
13223
|
+
totalResults: collected.totalResults,
|
|
13224
|
+
fetchedPages: collected.fetchedPages,
|
|
13225
|
+
pacing: collected.pacing,
|
|
13226
|
+
};
|
|
13227
|
+
}
|
|
13228
|
+
const payload = await launchAffiliateCampaignViaApp(session, {
|
|
13229
|
+
affiliateLink,
|
|
13230
|
+
linkedInUrl,
|
|
13231
|
+
maxResults,
|
|
13232
|
+
dryRun: Boolean(options.dryRun),
|
|
13233
|
+
localResults,
|
|
13234
|
+
localCollection,
|
|
13235
|
+
});
|
|
13236
|
+
if (options.out) {
|
|
13237
|
+
await writeJsonFile(options.out, payload);
|
|
13238
|
+
}
|
|
13239
|
+
return { session, payload };
|
|
13240
|
+
}
|
|
13241
|
+
function addAffiliateAudienceOptions(command) {
|
|
13242
|
+
return command
|
|
13243
|
+
.requiredOption("--affiliate-link <url>", "Affiliate link to promote")
|
|
13244
|
+
.requiredOption("--linkedin-url <url>", "Sales Navigator people or LinkedIn content search URL")
|
|
13245
|
+
.option("--max-results <number>", "Maximum audience results", "1000")
|
|
13246
|
+
.option("--curl-file <path>", "Optional copied Sales Navigator Lead Search curl request")
|
|
13247
|
+
.option("--browser-relay-port <number>", "Use the signed-in browser through a loopback relay")
|
|
13248
|
+
.option("--page-size <number>", "Direct Sales Navigator page size", "100")
|
|
13249
|
+
.option("--page-delay-min-ms <number>", "Minimum delay between direct pages", "5000")
|
|
13250
|
+
.option("--page-delay-max-ms <number>", "Maximum delay between direct pages", "8000")
|
|
13251
|
+
.option("--dry-run", "Validate and preview without creating remote resources", false)
|
|
13252
|
+
.option("--out <path>", "Optional local JSON output path");
|
|
13253
|
+
}
|
|
13254
|
+
addAffiliateAudienceOptions(program
|
|
13255
|
+
.command("affiliate:launch")
|
|
13256
|
+
.description("Build an affiliate audience from an affiliate link and LinkedIn search URL."))
|
|
13257
|
+
.action(async (options) => {
|
|
13258
|
+
const { payload } = await runAffiliateLaunchCommand(options);
|
|
13259
|
+
printOutput(payload);
|
|
13260
|
+
});
|
|
13261
|
+
addAffiliateAudienceOptions(program
|
|
13262
|
+
.command("affiliate:run")
|
|
13263
|
+
.description("Build an affiliate audience and prepare a reviewed Instantly campaign end to end."))
|
|
13264
|
+
.option("--campaign-name <name>", "Optional Instantly campaign name")
|
|
13265
|
+
.option("--language <language>", "Sequence language", "English")
|
|
13266
|
+
.option("--steps <number>", "Number of sequence emails", "3")
|
|
13267
|
+
.option("--min-email-score <number>", "Minimum Hunter confidence score", "80")
|
|
13268
|
+
.option("--allow-accept-all", "Include catch-all domains that meet the score threshold", false)
|
|
13269
|
+
.option("--daily-limit <number>", "Maximum new leads contacted per day", "25")
|
|
13270
|
+
.option("--timezone <timezone>", "Instantly sending timezone", "America/Detroit")
|
|
13271
|
+
.option("--send-from <time>", "Weekday sending window start (HH:MM)", "09:00")
|
|
13272
|
+
.option("--send-to <time>", "Weekday sending window end (HH:MM)", "16:00")
|
|
13273
|
+
.option("--activate", "Activate the prepared Instantly campaign after review data is returned", false)
|
|
13274
|
+
.action(async (options) => {
|
|
13275
|
+
const launched = await runAffiliateLaunchCommand(options);
|
|
13276
|
+
if (options.dryRun) {
|
|
13277
|
+
printOutput({
|
|
13278
|
+
...launched.payload,
|
|
13279
|
+
outreach: {
|
|
13280
|
+
status: "dry-run",
|
|
13281
|
+
provider: "instantly",
|
|
13282
|
+
activation: "not_requested"
|
|
13283
|
+
}
|
|
13284
|
+
});
|
|
13285
|
+
return;
|
|
13286
|
+
}
|
|
13287
|
+
const runId = launched.payload.extraction &&
|
|
13288
|
+
typeof launched.payload.extraction.runId === "string"
|
|
13289
|
+
? launched.payload.extraction.runId
|
|
13290
|
+
: null;
|
|
13291
|
+
if (!runId) {
|
|
13292
|
+
throw new Error("The audience provider did not return a durable run id. End-to-end outreach requires a Sales Navigator people search.");
|
|
13293
|
+
}
|
|
13294
|
+
const outreach = await prepareAffiliateOutreachViaApp(launched.session, {
|
|
13295
|
+
audienceRunId: z.string().uuid().parse(runId),
|
|
13296
|
+
campaignName: options.campaignName?.trim() || undefined,
|
|
13297
|
+
language: z.string().trim().min(1).max(40).parse(options.language),
|
|
13298
|
+
numberOfSteps: z.coerce.number().int().min(1).max(6).parse(options.steps),
|
|
13299
|
+
minEmailScore: z.coerce.number().int().min(0).max(100).parse(options.minEmailScore),
|
|
13300
|
+
allowAcceptAll: Boolean(options.allowAcceptAll),
|
|
13301
|
+
dailyLimit: z.coerce.number().int().min(1).max(1000).parse(options.dailyLimit),
|
|
13302
|
+
timezone: z.string().trim().min(1).max(100).parse(options.timezone),
|
|
13303
|
+
sendFrom: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/).parse(options.sendFrom),
|
|
13304
|
+
sendTo: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/).parse(options.sendTo)
|
|
13305
|
+
});
|
|
13306
|
+
const finalOutreach = options.activate
|
|
13307
|
+
? await activateAffiliateOutreachViaApp(launched.session, runId)
|
|
13308
|
+
: outreach;
|
|
13309
|
+
printOutput({
|
|
13310
|
+
status: "ok",
|
|
13311
|
+
audience: launched.payload,
|
|
13312
|
+
outreach: finalOutreach,
|
|
13313
|
+
next: finalOutreach.status === "ready"
|
|
13314
|
+
? `Review the sequence, then run: salesprompter affiliate:activate ${runId}`
|
|
13315
|
+
: "The Instantly campaign is active."
|
|
13316
|
+
});
|
|
13317
|
+
});
|
|
13318
|
+
program
|
|
13319
|
+
.command("affiliate:status <run-id>")
|
|
13320
|
+
.description("Show enrichment, sequence, Instantly, and activation status for an affiliate run.")
|
|
13321
|
+
.action(async (runId) => {
|
|
13322
|
+
const session = await requireAuthSession();
|
|
13323
|
+
const outreach = await getAffiliateOutreachViaApp(session, z.string().uuid().parse(runId));
|
|
13324
|
+
printOutput({ status: "ok", outreach });
|
|
13325
|
+
});
|
|
13326
|
+
program
|
|
13327
|
+
.command("affiliate:enrich <run-id>")
|
|
13328
|
+
.description("Find additional verified emails and add only new leads to the campaign.")
|
|
13329
|
+
.action(async (runId) => {
|
|
13330
|
+
const session = await requireAuthSession();
|
|
13331
|
+
const outreach = await enrichAffiliateOutreachViaApp(session, z.string().uuid().parse(runId));
|
|
13332
|
+
printOutput({
|
|
13333
|
+
status: "ok",
|
|
13334
|
+
outreach,
|
|
13335
|
+
message: "Email recovery finished and new verified leads were added."
|
|
13336
|
+
});
|
|
13337
|
+
});
|
|
13338
|
+
program
|
|
13339
|
+
.command("affiliate:activate <run-id>")
|
|
13340
|
+
.description("Activate a reviewed Instantly affiliate campaign.")
|
|
13341
|
+
.action(async (runId) => {
|
|
13342
|
+
const session = await requireAuthSession();
|
|
13343
|
+
const outreach = await activateAffiliateOutreachViaApp(session, z.string().uuid().parse(runId));
|
|
13344
|
+
printOutput({
|
|
13345
|
+
status: "ok",
|
|
13346
|
+
outreach,
|
|
13347
|
+
message: "The Instantly campaign is active."
|
|
13348
|
+
});
|
|
13349
|
+
});
|
|
10868
13350
|
program
|
|
10869
13351
|
.command("sync:crm")
|
|
10870
13352
|
.description("Dry-run sync scored leads into a CRM target.")
|
|
@@ -10945,6 +13427,76 @@ program
|
|
|
10945
13427
|
mergeSql: result.mergeSql
|
|
10946
13428
|
});
|
|
10947
13429
|
});
|
|
13430
|
+
program
|
|
13431
|
+
.command("linkedin-companies:scrape-local")
|
|
13432
|
+
.alias("companies:scrape-linkedin")
|
|
13433
|
+
.description("LinkedIn Company Scraper: parse a locally opened LinkedIn company page and store it in Salesprompter.")
|
|
13434
|
+
.requiredOption("--url <url>", "LinkedIn company URL with a numeric company id")
|
|
13435
|
+
.option("--text-file <path>", "Path to text copied from the LinkedIn company About page")
|
|
13436
|
+
.option("--insights-text-file <path>", "Path to text copied from the LinkedIn company Insights page")
|
|
13437
|
+
.option("--apply", "Store the scraped company profile in the Salesprompter database", false)
|
|
13438
|
+
.option("--no-open", "Do not open the LinkedIn company URL in the local browser")
|
|
13439
|
+
.action(async (options) => {
|
|
13440
|
+
const target = normalizeLinkedInCompanyScrapeUrl(options.url);
|
|
13441
|
+
if (options.open !== false) {
|
|
13442
|
+
await new Promise((resolve) => {
|
|
13443
|
+
const child = spawn("open", [target.companyUrl], { stdio: "ignore" });
|
|
13444
|
+
child.on("close", () => resolve());
|
|
13445
|
+
child.on("error", () => resolve());
|
|
13446
|
+
});
|
|
13447
|
+
}
|
|
13448
|
+
const inputText = options.textFile
|
|
13449
|
+
? await readFile(path.resolve(String(options.textFile)), "utf8")
|
|
13450
|
+
: await readAllStdin();
|
|
13451
|
+
if (!inputText.trim()) {
|
|
13452
|
+
throw new Error("No LinkedIn company page text provided. Open the company page, copy the About page text, then pass --text-file or pipe it on stdin.");
|
|
13453
|
+
}
|
|
13454
|
+
const insightsText = options.insightsTextFile
|
|
13455
|
+
? await readFile(path.resolve(String(options.insightsTextFile)), "utf8")
|
|
13456
|
+
: "";
|
|
13457
|
+
const profile = parseLocalLinkedInCompanyText(inputText, target.sourceUrl, insightsText);
|
|
13458
|
+
if (options.apply) {
|
|
13459
|
+
await upsertLocalLinkedInCompanyScrape(profile);
|
|
13460
|
+
}
|
|
13461
|
+
printOutput({
|
|
13462
|
+
status: "ok",
|
|
13463
|
+
stored: Boolean(options.apply),
|
|
13464
|
+
profile,
|
|
13465
|
+
});
|
|
13466
|
+
});
|
|
13467
|
+
program
|
|
13468
|
+
.command("dealroom-companies:scrape-local")
|
|
13469
|
+
.alias("companies:scrape-dealroom")
|
|
13470
|
+
.description("Dealroom Company Scraper: parse a locally opened Dealroom company page and store it in Salesprompter.")
|
|
13471
|
+
.requiredOption("--url <url>", "Dealroom company URL with /companies/<slug>")
|
|
13472
|
+
.option("--text-file <path>", "Path to text copied from the Dealroom company page")
|
|
13473
|
+
.option("--apply", "Store the scraped Dealroom company profile in the Salesprompter database", false)
|
|
13474
|
+
.option("--no-open", "Do not open the Dealroom company URL in the local browser")
|
|
13475
|
+
.action(async (options) => {
|
|
13476
|
+
const target = normalizeDealroomCompanyScrapeUrl(options.url);
|
|
13477
|
+
if (options.open !== false) {
|
|
13478
|
+
await new Promise((resolve) => {
|
|
13479
|
+
const child = spawn("open", [target.companyUrl], { stdio: "ignore" });
|
|
13480
|
+
child.on("close", () => resolve());
|
|
13481
|
+
child.on("error", () => resolve());
|
|
13482
|
+
});
|
|
13483
|
+
}
|
|
13484
|
+
const inputText = options.textFile
|
|
13485
|
+
? await readFile(path.resolve(String(options.textFile)), "utf8")
|
|
13486
|
+
: await readAllStdin();
|
|
13487
|
+
if (!inputText.trim()) {
|
|
13488
|
+
throw new Error("No Dealroom company page text provided. Open the company page, copy the visible profile text, then pass --text-file or pipe it on stdin.");
|
|
13489
|
+
}
|
|
13490
|
+
const profile = parseLocalDealroomCompanyText(inputText, target.sourceUrl);
|
|
13491
|
+
if (options.apply) {
|
|
13492
|
+
await insertLocalDealroomCompanyScrape(profile);
|
|
13493
|
+
}
|
|
13494
|
+
printOutput({
|
|
13495
|
+
status: "ok",
|
|
13496
|
+
stored: Boolean(options.apply),
|
|
13497
|
+
profile,
|
|
13498
|
+
});
|
|
13499
|
+
});
|
|
10948
13500
|
program
|
|
10949
13501
|
.command("linkedin-products:scrape")
|
|
10950
13502
|
.alias("market:scrape")
|
|
@@ -11570,16 +14122,533 @@ program
|
|
|
11570
14122
|
await writeJsonFile(samplesPath, samples);
|
|
11571
14123
|
printOutput(payload);
|
|
11572
14124
|
});
|
|
14125
|
+
program
|
|
14126
|
+
.command("salesnav:accounts:collect")
|
|
14127
|
+
.alias("accounts:collect")
|
|
14128
|
+
.description("Collect an exhaustive Sales Navigator account search locally with a durable checkpoint.")
|
|
14129
|
+
.requiredOption("--query-url <url>", "LinkedIn Sales Navigator account search URL")
|
|
14130
|
+
.requiredOption("--checkpoint <path>", "Checkpoint file used to resume after rate limits or interruptions")
|
|
14131
|
+
.requiredOption("--raw-out <path>", "Line-delimited raw account results used by accounts:local-import")
|
|
14132
|
+
.option("--curl-file <path>", "Path to a saved account-search curl request used as the local request template")
|
|
14133
|
+
.option("--browser-relay-port <number>", "Use the signed-in local browser as Account Search transport through this loopback port")
|
|
14134
|
+
.option("--reported-total <number>", "Known root result total. Skips the initial one-request root count probe.")
|
|
14135
|
+
.option("--max-results-per-search <number>", "Maximum pageable results per final filter slice", String(LOCAL_ACCOUNT_SEARCH_COLLECTOR_RESULT_WINDOW))
|
|
14136
|
+
.option("--page-size <number>", "Accounts requested per Account Search API request", "100")
|
|
14137
|
+
.option("--delay-min-ms <number>", "Minimum randomized delay between Account Search API requests", "5000")
|
|
14138
|
+
.option("--delay-max-ms <number>", "Maximum randomized delay between Account Search API requests", "8000")
|
|
14139
|
+
.option("--max-requests <number>", "Maximum API requests in this invocation, or 'all'", "all")
|
|
14140
|
+
.option("--max-retries <number>", "Retries for transient network/5xx failures. Rate limits stop immediately.", "1")
|
|
14141
|
+
.option("--retry-base-delay-ms <number>", "Base delay for full-jitter retry backoff", "2000")
|
|
14142
|
+
.option("--retry-max-delay-ms <number>", "Maximum delay for full-jitter retry backoff", "30000")
|
|
14143
|
+
.option("--out <path>", "Optional JSON summary output path")
|
|
14144
|
+
.action(async (options) => {
|
|
14145
|
+
const queryUrl = z.string().url().parse(options.queryUrl);
|
|
14146
|
+
const checkpointPath = path.resolve(z.string().min(1).parse(options.checkpoint));
|
|
14147
|
+
const rawOutputPath = path.resolve(z.string().min(1).parse(options.rawOut));
|
|
14148
|
+
const cap = z.coerce
|
|
14149
|
+
.number()
|
|
14150
|
+
.int()
|
|
14151
|
+
.min(1)
|
|
14152
|
+
.max(LOCAL_ACCOUNT_SEARCH_COLLECTOR_RESULT_WINDOW)
|
|
14153
|
+
.parse(options.maxResultsPerSearch);
|
|
14154
|
+
const pageSize = z.coerce
|
|
14155
|
+
.number()
|
|
14156
|
+
.int()
|
|
14157
|
+
.min(1)
|
|
14158
|
+
.max(100)
|
|
14159
|
+
.parse(options.pageSize);
|
|
14160
|
+
const delayMinMs = z.coerce
|
|
14161
|
+
.number()
|
|
14162
|
+
.int()
|
|
14163
|
+
.min(0)
|
|
14164
|
+
.max(300000)
|
|
14165
|
+
.parse(options.delayMinMs);
|
|
14166
|
+
const delayMaxMs = z.coerce
|
|
14167
|
+
.number()
|
|
14168
|
+
.int()
|
|
14169
|
+
.min(0)
|
|
14170
|
+
.max(300000)
|
|
14171
|
+
.parse(options.delayMaxMs);
|
|
14172
|
+
if (delayMaxMs < delayMinMs) {
|
|
14173
|
+
throw new Error("--delay-max-ms must be greater than or equal to --delay-min-ms.");
|
|
14174
|
+
}
|
|
14175
|
+
const maxRequestsOption = String(options.maxRequests ?? "all")
|
|
14176
|
+
.trim()
|
|
14177
|
+
.toLowerCase();
|
|
14178
|
+
const maxRequests = maxRequestsOption === "all"
|
|
14179
|
+
? null
|
|
14180
|
+
: z.coerce.number().int().min(1).max(100000).parse(options.maxRequests);
|
|
14181
|
+
const reportedTotal = options.reportedTotal == null
|
|
14182
|
+
? null
|
|
14183
|
+
: z.coerce
|
|
14184
|
+
.number()
|
|
14185
|
+
.int()
|
|
14186
|
+
.min(1)
|
|
14187
|
+
.max(1_000_000)
|
|
14188
|
+
.parse(options.reportedTotal);
|
|
14189
|
+
const retry = {
|
|
14190
|
+
maxRetries: z.coerce
|
|
14191
|
+
.number()
|
|
14192
|
+
.int()
|
|
14193
|
+
.min(0)
|
|
14194
|
+
.max(5)
|
|
14195
|
+
.parse(options.maxRetries),
|
|
14196
|
+
retryBaseDelayMs: z.coerce
|
|
14197
|
+
.number()
|
|
14198
|
+
.int()
|
|
14199
|
+
.min(0)
|
|
14200
|
+
.max(300000)
|
|
14201
|
+
.parse(options.retryBaseDelayMs),
|
|
14202
|
+
retryMaxDelayMs: z.coerce
|
|
14203
|
+
.number()
|
|
14204
|
+
.int()
|
|
14205
|
+
.min(0)
|
|
14206
|
+
.max(300000)
|
|
14207
|
+
.parse(options.retryMaxDelayMs),
|
|
14208
|
+
};
|
|
14209
|
+
const normalizedSourceQueryUrl = buildSalesNavigatorAccountQuery(queryUrl, []).sourceQueryUrl;
|
|
14210
|
+
let checkpoint;
|
|
14211
|
+
if (await fileExists(checkpointPath)) {
|
|
14212
|
+
checkpoint = parseLocalAccountSearchCollectorCheckpoint(JSON.parse(await readFile(checkpointPath, "utf8")));
|
|
14213
|
+
const normalizedCheckpointUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, []).sourceQueryUrl;
|
|
14214
|
+
if (normalizedCheckpointUrl !== normalizedSourceQueryUrl) {
|
|
14215
|
+
throw new Error("The checkpoint belongs to a different Sales Navigator account search.");
|
|
14216
|
+
}
|
|
14217
|
+
if (checkpoint.pageSize !== pageSize || checkpoint.cap > cap) {
|
|
14218
|
+
throw new Error(`Checkpoint paging does not match this invocation. Use --max-results-per-search ${checkpoint.cap} --page-size ${checkpoint.pageSize}.`);
|
|
14219
|
+
}
|
|
14220
|
+
if (checkpoint.cap < cap) {
|
|
14221
|
+
checkpoint.cap = cap;
|
|
14222
|
+
}
|
|
14223
|
+
checkpoint.delayMinMs = delayMinMs;
|
|
14224
|
+
checkpoint.delayMaxMs = delayMaxMs;
|
|
14225
|
+
if (reportedTotal != null && checkpoint.rootTotal == null) {
|
|
14226
|
+
checkpoint.rootTotal = reportedTotal;
|
|
14227
|
+
}
|
|
14228
|
+
}
|
|
14229
|
+
else {
|
|
14230
|
+
checkpoint = createLocalAccountSearchCollectorCheckpoint({
|
|
14231
|
+
sourceQueryUrl: queryUrl,
|
|
14232
|
+
rootTotal: reportedTotal,
|
|
14233
|
+
cap,
|
|
14234
|
+
pageSize,
|
|
14235
|
+
delayMinMs,
|
|
14236
|
+
delayMaxMs,
|
|
14237
|
+
});
|
|
14238
|
+
}
|
|
14239
|
+
if (migrateLocalAccountSearchCollectorCheckpoint(checkpoint)) {
|
|
14240
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14241
|
+
}
|
|
14242
|
+
const curlFile = typeof options.curlFile === "string" && options.curlFile.trim().length > 0
|
|
14243
|
+
? options.curlFile.trim()
|
|
14244
|
+
: null;
|
|
14245
|
+
const browserRelayPort = options.browserRelayPort == null
|
|
14246
|
+
? null
|
|
14247
|
+
: z.coerce
|
|
14248
|
+
.number()
|
|
14249
|
+
.int()
|
|
14250
|
+
.min(1)
|
|
14251
|
+
.max(65535)
|
|
14252
|
+
.parse(options.browserRelayPort);
|
|
14253
|
+
if (curlFile && browserRelayPort != null) {
|
|
14254
|
+
throw new Error("Use either --curl-file or --browser-relay-port, not both.");
|
|
14255
|
+
}
|
|
14256
|
+
const templateRequest = curlFile
|
|
14257
|
+
? parseSalesNavigatorCurlRequest(await readFile(curlFile, "utf8"))
|
|
14258
|
+
: null;
|
|
14259
|
+
if (templateRequest &&
|
|
14260
|
+
!new URL(templateRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
14261
|
+
throw new Error("Account collection --curl-file must contain a /sales-api/salesApiAccountSearch request.");
|
|
14262
|
+
}
|
|
14263
|
+
const config = templateRequest || browserRelayPort != null
|
|
14264
|
+
? null
|
|
14265
|
+
: await readLinkedInAccountSearchConfig(queryUrl);
|
|
14266
|
+
const browserRelay = browserRelayPort == null
|
|
14267
|
+
? null
|
|
14268
|
+
: await createLocalAccountSearchBrowserRelay(browserRelayPort);
|
|
14269
|
+
const buildRequest = (searchUrl, start, count) => {
|
|
14270
|
+
const request = browserRelay
|
|
14271
|
+
? {
|
|
14272
|
+
url: buildSalesNavigatorAccountApiUrlFromSearchUrl(searchUrl, pageSize),
|
|
14273
|
+
headers: {},
|
|
14274
|
+
}
|
|
14275
|
+
: templateRequest
|
|
14276
|
+
? buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(searchUrl, templateRequest, pageSize)
|
|
14277
|
+
: buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, pageSize);
|
|
14278
|
+
return {
|
|
14279
|
+
...request,
|
|
14280
|
+
url: withSalesNavigatorPaging(request.url, start, count),
|
|
14281
|
+
};
|
|
14282
|
+
};
|
|
14283
|
+
const executeRequest = (request) => browserRelay
|
|
14284
|
+
? browserRelay.request(request.url)
|
|
14285
|
+
: fetchLocalSalesNavigatorRequest(request, retry);
|
|
14286
|
+
try {
|
|
14287
|
+
let requestsThisRun = 0;
|
|
14288
|
+
let delayThisRunMs = 0;
|
|
14289
|
+
const distinctAccountKeys = await loadLocalAccountSearchRawAccountKeys(rawOutputPath);
|
|
14290
|
+
checkpoint.distinctAccounts = distinctAccountKeys.size;
|
|
14291
|
+
checkpoint.completedByDistinctCount =
|
|
14292
|
+
checkpoint.rootTotal != null &&
|
|
14293
|
+
distinctAccountKeys.size >= checkpoint.rootTotal;
|
|
14294
|
+
if (checkpoint.completedByDistinctCount) {
|
|
14295
|
+
checkpoint.taskQueue = [];
|
|
14296
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14297
|
+
}
|
|
14298
|
+
const requestBudgetAvailable = () => maxRequests == null || requestsThisRun < maxRequests;
|
|
14299
|
+
const stopForRateLimit = async (error, task) => {
|
|
14300
|
+
checkpoint.rateLimited = {
|
|
14301
|
+
at: new Date().toISOString(),
|
|
14302
|
+
retryAfter: error.retryAfterMs,
|
|
14303
|
+
status: error.status,
|
|
14304
|
+
task,
|
|
14305
|
+
};
|
|
14306
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14307
|
+
};
|
|
14308
|
+
if (checkpoint.rootTotal == null && requestBudgetAvailable()) {
|
|
14309
|
+
try {
|
|
14310
|
+
const rootResponse = await executeRequest(buildRequest(queryUrl, 0, 1));
|
|
14311
|
+
requestsThisRun += 1;
|
|
14312
|
+
checkpoint.requests += 1;
|
|
14313
|
+
checkpoint.rootTotal = extractLocalSalesNavigatorTotalResults(rootResponse.body);
|
|
14314
|
+
checkpoint.rateLimited = null;
|
|
14315
|
+
if (checkpoint.rootTotal == null) {
|
|
14316
|
+
throw new Error("Sales Navigator Account Search root response did not include a result total.");
|
|
14317
|
+
}
|
|
14318
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14319
|
+
}
|
|
14320
|
+
catch (error) {
|
|
14321
|
+
if (error instanceof LocalSalesNavigatorRateLimitError) {
|
|
14322
|
+
await stopForRateLimit(error, { kind: "root" });
|
|
14323
|
+
const payload = {
|
|
14324
|
+
status: "rate_limited",
|
|
14325
|
+
mode: "account-local-collector",
|
|
14326
|
+
checkpointPath,
|
|
14327
|
+
rawOutputPath,
|
|
14328
|
+
requestsThisRun,
|
|
14329
|
+
requests: checkpoint.requests,
|
|
14330
|
+
remainingTasks: checkpoint.taskQueue.length,
|
|
14331
|
+
rateLimited: checkpoint.rateLimited,
|
|
14332
|
+
};
|
|
14333
|
+
if (options.out) {
|
|
14334
|
+
await writeJsonFile(options.out, payload);
|
|
14335
|
+
}
|
|
14336
|
+
printOutput(payload);
|
|
14337
|
+
return;
|
|
14338
|
+
}
|
|
14339
|
+
throw error;
|
|
14340
|
+
}
|
|
14341
|
+
}
|
|
14342
|
+
if (checkpoint.rootTotal != null &&
|
|
14343
|
+
checkpoint.taskQueue.length === 0 &&
|
|
14344
|
+
checkpoint.seenQueries.length === 0) {
|
|
14345
|
+
seedLocalAccountSearchCollectorTasks(checkpoint);
|
|
14346
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14347
|
+
}
|
|
14348
|
+
const dimensions = new Map([
|
|
14349
|
+
...DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS,
|
|
14350
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_GROWTH_DIMENSION,
|
|
14351
|
+
].map((dimension) => [dimension.filterType, dimension]));
|
|
14352
|
+
const seenQueries = new Set(checkpoint.seenQueries);
|
|
14353
|
+
let stoppedByRateLimit = false;
|
|
14354
|
+
while (checkpoint.taskQueue.length > 0 && requestBudgetAvailable()) {
|
|
14355
|
+
const task = checkpoint.taskQueue[0];
|
|
14356
|
+
const searchUrl = task.kind === "node"
|
|
14357
|
+
? task.searchUrlOverride ??
|
|
14358
|
+
buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, task.filters).slicedQueryUrl
|
|
14359
|
+
: task.searchUrl;
|
|
14360
|
+
const start = task.kind === "page" ? task.start : 0;
|
|
14361
|
+
let response;
|
|
14362
|
+
try {
|
|
14363
|
+
response = await executeRequest(buildRequest(searchUrl, start, pageSize));
|
|
14364
|
+
}
|
|
14365
|
+
catch (error) {
|
|
14366
|
+
if (error instanceof LocalSalesNavigatorRateLimitError) {
|
|
14367
|
+
stoppedByRateLimit = true;
|
|
14368
|
+
await stopForRateLimit(error, task);
|
|
14369
|
+
break;
|
|
14370
|
+
}
|
|
14371
|
+
checkpoint.failures.push({
|
|
14372
|
+
at: new Date().toISOString(),
|
|
14373
|
+
task,
|
|
14374
|
+
error: error instanceof Error ? error.message : String(error),
|
|
14375
|
+
});
|
|
14376
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14377
|
+
throw error;
|
|
14378
|
+
}
|
|
14379
|
+
requestsThisRun += 1;
|
|
14380
|
+
checkpoint.requests += 1;
|
|
14381
|
+
checkpoint.rateLimited = null;
|
|
14382
|
+
const elements = extractLocalSalesNavigatorElements(response.body);
|
|
14383
|
+
await appendLocalAccountSearchRawElements({
|
|
14384
|
+
rawOutputPath,
|
|
14385
|
+
searchUrl,
|
|
14386
|
+
elements,
|
|
14387
|
+
});
|
|
14388
|
+
for (const element of elements) {
|
|
14389
|
+
addLocalAccountSearchElementKey(distinctAccountKeys, element, searchUrl);
|
|
14390
|
+
}
|
|
14391
|
+
checkpoint.distinctAccounts = distinctAccountKeys.size;
|
|
14392
|
+
checkpoint.pagesWritten += 1;
|
|
14393
|
+
checkpoint.rowsWritten += elements.length;
|
|
14394
|
+
if (checkpoint.rootTotal != null &&
|
|
14395
|
+
distinctAccountKeys.size >= checkpoint.rootTotal) {
|
|
14396
|
+
checkpoint.taskQueue = [];
|
|
14397
|
+
checkpoint.completedByDistinctCount = true;
|
|
14398
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14399
|
+
break;
|
|
14400
|
+
}
|
|
14401
|
+
if (task.kind === "node") {
|
|
14402
|
+
const total = extractLocalSalesNavigatorTotalResults(response.body);
|
|
14403
|
+
if (total == null) {
|
|
14404
|
+
checkpoint.failures.push({
|
|
14405
|
+
at: new Date().toISOString(),
|
|
14406
|
+
task,
|
|
14407
|
+
error: "Account Search response did not include a result total.",
|
|
14408
|
+
});
|
|
14409
|
+
}
|
|
14410
|
+
else if (total <= cap) {
|
|
14411
|
+
checkpoint.leaves.push({
|
|
14412
|
+
id: task.id,
|
|
14413
|
+
pass: task.pass,
|
|
14414
|
+
total,
|
|
14415
|
+
searchUrl,
|
|
14416
|
+
filters: task.filters,
|
|
14417
|
+
});
|
|
14418
|
+
const pageTasks = [];
|
|
14419
|
+
for (let pageStart = pageSize; pageStart < total; pageStart += pageSize) {
|
|
14420
|
+
pageTasks.push({
|
|
14421
|
+
kind: "page",
|
|
14422
|
+
id: task.id,
|
|
14423
|
+
pass: task.pass,
|
|
14424
|
+
searchUrl,
|
|
14425
|
+
start: pageStart,
|
|
14426
|
+
total,
|
|
14427
|
+
});
|
|
14428
|
+
}
|
|
14429
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, pageTasks);
|
|
14430
|
+
}
|
|
14431
|
+
else {
|
|
14432
|
+
const [nextFilterType, ...remainingDimensions] = task.remainingDimensions;
|
|
14433
|
+
const dimension = nextFilterType
|
|
14434
|
+
? dimensions.get(nextFilterType)
|
|
14435
|
+
: null;
|
|
14436
|
+
if (!dimension) {
|
|
14437
|
+
const bayAreaPartitionPlan = buildLocalAccountSearchCollectorBayAreaPartitions(checkpoint.sourceQueryUrl, task.filters);
|
|
14438
|
+
if (!bayAreaPartitionPlan) {
|
|
14439
|
+
const industryPartitionPlan = task.pass.startsWith("coverage-")
|
|
14440
|
+
? buildLocalAccountSearchCollectorIndustryPartitions(searchUrl)
|
|
14441
|
+
: null;
|
|
14442
|
+
if (industryPartitionPlan) {
|
|
14443
|
+
checkpoint.windows.push({
|
|
14444
|
+
id: task.id,
|
|
14445
|
+
pass: task.pass,
|
|
14446
|
+
total,
|
|
14447
|
+
searchUrl,
|
|
14448
|
+
splitOn: industryPartitionPlan.splitOn,
|
|
14449
|
+
});
|
|
14450
|
+
const industryTasks = [];
|
|
14451
|
+
for (const partition of industryPartitionPlan.partitions) {
|
|
14452
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, partition.searchUrl);
|
|
14453
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14454
|
+
checkpoint.duplicateQueries += 1;
|
|
14455
|
+
continue;
|
|
14456
|
+
}
|
|
14457
|
+
seenQueries.add(childQueryKey);
|
|
14458
|
+
industryTasks.push({
|
|
14459
|
+
kind: "node",
|
|
14460
|
+
id: `${task.id}-${industryPartitionPlan.splitOn}-${partition.idSuffix}`,
|
|
14461
|
+
pass: task.pass,
|
|
14462
|
+
filters: task.filters.map((filter) => ({
|
|
14463
|
+
...filter,
|
|
14464
|
+
values: filter.values.map((value) => ({ ...value })),
|
|
14465
|
+
})),
|
|
14466
|
+
remainingDimensions: [],
|
|
14467
|
+
searchUrlOverride: partition.searchUrl,
|
|
14468
|
+
keywordClauses: task.keywordClauses
|
|
14469
|
+
? [...task.keywordClauses]
|
|
14470
|
+
: undefined,
|
|
14471
|
+
keywordDepth: task.keywordDepth,
|
|
14472
|
+
});
|
|
14473
|
+
}
|
|
14474
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, industryTasks);
|
|
14475
|
+
}
|
|
14476
|
+
else {
|
|
14477
|
+
const keywordPartitionPlan = buildLocalAccountSearchCollectorKeywordPartitions(searchUrl, task);
|
|
14478
|
+
if (!keywordPartitionPlan) {
|
|
14479
|
+
const terminalPageLimit = Math.min(total, cap);
|
|
14480
|
+
const pageTasks = [];
|
|
14481
|
+
for (let pageStart = pageSize; pageStart < terminalPageLimit; pageStart += pageSize) {
|
|
14482
|
+
pageTasks.push({
|
|
14483
|
+
kind: "page",
|
|
14484
|
+
id: task.id,
|
|
14485
|
+
pass: task.pass,
|
|
14486
|
+
searchUrl,
|
|
14487
|
+
start: pageStart,
|
|
14488
|
+
total,
|
|
14489
|
+
});
|
|
14490
|
+
}
|
|
14491
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, pageTasks);
|
|
14492
|
+
checkpoint.failures.push({
|
|
14493
|
+
at: new Date().toISOString(),
|
|
14494
|
+
task,
|
|
14495
|
+
total,
|
|
14496
|
+
collectedResultWindow: terminalPageLimit,
|
|
14497
|
+
missingBeyondResultWindow: Math.max(0, total - terminalPageLimit),
|
|
14498
|
+
error: `Account Search slice remains above ${cap} with no split dimension left.`,
|
|
14499
|
+
});
|
|
14500
|
+
}
|
|
14501
|
+
else {
|
|
14502
|
+
checkpoint.windows.push({
|
|
14503
|
+
id: task.id,
|
|
14504
|
+
pass: task.pass,
|
|
14505
|
+
total,
|
|
14506
|
+
searchUrl,
|
|
14507
|
+
splitOn: keywordPartitionPlan.splitOn,
|
|
14508
|
+
});
|
|
14509
|
+
const keywordTasks = [];
|
|
14510
|
+
for (const partition of keywordPartitionPlan.partitions) {
|
|
14511
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, partition.searchUrl);
|
|
14512
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14513
|
+
checkpoint.duplicateQueries += 1;
|
|
14514
|
+
continue;
|
|
14515
|
+
}
|
|
14516
|
+
seenQueries.add(childQueryKey);
|
|
14517
|
+
keywordTasks.push({
|
|
14518
|
+
kind: "node",
|
|
14519
|
+
id: `${task.id}-${keywordPartitionPlan.splitOn}-${partition.idSuffix}`,
|
|
14520
|
+
pass: task.pass,
|
|
14521
|
+
filters: task.filters.map((filter) => ({
|
|
14522
|
+
...filter,
|
|
14523
|
+
values: filter.values.map((value) => ({ ...value })),
|
|
14524
|
+
})),
|
|
14525
|
+
remainingDimensions: [],
|
|
14526
|
+
searchUrlOverride: partition.searchUrl,
|
|
14527
|
+
keywordClauses: partition.keywordClauses,
|
|
14528
|
+
keywordDepth: partition.keywordDepth,
|
|
14529
|
+
});
|
|
14530
|
+
}
|
|
14531
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, keywordTasks);
|
|
14532
|
+
}
|
|
14533
|
+
}
|
|
14534
|
+
}
|
|
14535
|
+
else {
|
|
14536
|
+
checkpoint.windows.push({
|
|
14537
|
+
id: task.id,
|
|
14538
|
+
pass: task.pass,
|
|
14539
|
+
total,
|
|
14540
|
+
searchUrl,
|
|
14541
|
+
splitOn: bayAreaPartitionPlan.splitOn,
|
|
14542
|
+
});
|
|
14543
|
+
const regionTasks = [];
|
|
14544
|
+
bayAreaPartitionPlan.partitions.forEach((filters, index) => {
|
|
14545
|
+
const childSearchUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, filters).slicedQueryUrl;
|
|
14546
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, childSearchUrl);
|
|
14547
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14548
|
+
checkpoint.duplicateQueries += 1;
|
|
14549
|
+
return;
|
|
14550
|
+
}
|
|
14551
|
+
seenQueries.add(childQueryKey);
|
|
14552
|
+
regionTasks.push({
|
|
14553
|
+
kind: "node",
|
|
14554
|
+
id: `${task.id}-${bayAreaPartitionPlan.splitOn}-${index}`,
|
|
14555
|
+
pass: task.pass,
|
|
14556
|
+
filters,
|
|
14557
|
+
remainingDimensions: [],
|
|
14558
|
+
});
|
|
14559
|
+
});
|
|
14560
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, regionTasks);
|
|
14561
|
+
}
|
|
14562
|
+
}
|
|
14563
|
+
else {
|
|
14564
|
+
checkpoint.windows.push({
|
|
14565
|
+
id: task.id,
|
|
14566
|
+
pass: task.pass,
|
|
14567
|
+
total,
|
|
14568
|
+
searchUrl,
|
|
14569
|
+
splitOn: dimension.filterType,
|
|
14570
|
+
});
|
|
14571
|
+
const dimensionTasks = [];
|
|
14572
|
+
buildLocalAccountSearchCollectorDimensionPartitions(dimension).forEach((partition) => {
|
|
14573
|
+
const filters = [
|
|
14574
|
+
...task.filters,
|
|
14575
|
+
partition.filter,
|
|
14576
|
+
];
|
|
14577
|
+
const childSearchUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, filters).slicedQueryUrl;
|
|
14578
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, childSearchUrl);
|
|
14579
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14580
|
+
checkpoint.duplicateQueries += 1;
|
|
14581
|
+
return;
|
|
14582
|
+
}
|
|
14583
|
+
seenQueries.add(childQueryKey);
|
|
14584
|
+
dimensionTasks.push({
|
|
14585
|
+
kind: "node",
|
|
14586
|
+
id: `${task.id}-${dimension.filterType}-${partition.idSuffix}`,
|
|
14587
|
+
pass: task.pass,
|
|
14588
|
+
filters,
|
|
14589
|
+
remainingDimensions,
|
|
14590
|
+
});
|
|
14591
|
+
});
|
|
14592
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, dimensionTasks);
|
|
14593
|
+
}
|
|
14594
|
+
}
|
|
14595
|
+
}
|
|
14596
|
+
checkpoint.taskQueue.shift();
|
|
14597
|
+
checkpoint.seenQueries = [...seenQueries];
|
|
14598
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14599
|
+
if (checkpoint.taskQueue.length > 0 && requestBudgetAvailable()) {
|
|
14600
|
+
const delayMs = randomIntegerBetween(delayMinMs, delayMaxMs);
|
|
14601
|
+
if (delayMs > 0) {
|
|
14602
|
+
delayThisRunMs += delayMs;
|
|
14603
|
+
await delay(delayMs);
|
|
14604
|
+
}
|
|
14605
|
+
}
|
|
14606
|
+
}
|
|
14607
|
+
const payload = {
|
|
14608
|
+
status: stoppedByRateLimit
|
|
14609
|
+
? "rate_limited"
|
|
14610
|
+
: checkpoint.taskQueue.length === 0
|
|
14611
|
+
? "complete"
|
|
14612
|
+
: "checkpointed",
|
|
14613
|
+
mode: "account-local-collector",
|
|
14614
|
+
sourceQueryUrl: checkpoint.sourceQueryUrl,
|
|
14615
|
+
rootTotal: checkpoint.rootTotal,
|
|
14616
|
+
checkpointPath,
|
|
14617
|
+
rawOutputPath,
|
|
14618
|
+
requestsThisRun,
|
|
14619
|
+
delayThisRunMs,
|
|
14620
|
+
requests: checkpoint.requests,
|
|
14621
|
+
pagesWritten: checkpoint.pagesWritten,
|
|
14622
|
+
rowsWritten: checkpoint.rowsWritten,
|
|
14623
|
+
leaves: checkpoint.leaves.length,
|
|
14624
|
+
windows: checkpoint.windows.length,
|
|
14625
|
+
failures: checkpoint.failures.length,
|
|
14626
|
+
remainingTasks: checkpoint.taskQueue.length,
|
|
14627
|
+
distinctAccounts: distinctAccountKeys.size,
|
|
14628
|
+
completedByDistinctCount: checkpoint.completedByDistinctCount === true,
|
|
14629
|
+
rateLimited: checkpoint.rateLimited,
|
|
14630
|
+
browserRelayPort: browserRelay?.port ?? null,
|
|
14631
|
+
};
|
|
14632
|
+
if (options.out) {
|
|
14633
|
+
await writeJsonFile(options.out, payload);
|
|
14634
|
+
}
|
|
14635
|
+
printOutput(payload);
|
|
14636
|
+
}
|
|
14637
|
+
finally {
|
|
14638
|
+
await browserRelay?.close();
|
|
14639
|
+
}
|
|
14640
|
+
});
|
|
11573
14641
|
program
|
|
11574
14642
|
.command("salesnav:accounts:split")
|
|
11575
14643
|
.alias("accounts:split")
|
|
11576
14644
|
.description("Split a Sales Navigator account search into account slices below the result cap.")
|
|
11577
14645
|
.requiredOption("--query-url <url>", "LinkedIn Sales Navigator account search URL")
|
|
11578
14646
|
.option("--curl-file <path>", "Path to a saved account-search curl request used as the local request template")
|
|
11579
|
-
.option("--org-id <id>", "Salesprompter workspace org id for
|
|
14647
|
+
.option("--org-id <id>", "Salesprompter workspace org id for Supabase storage. Defaults to SALESPROMPTER_ORG_ID or saved auth session org.")
|
|
11580
14648
|
.option("--max-results-per-search <number>", "Maximum account results allowed for each final slice", "1000")
|
|
11581
14649
|
.option("--max-windowed-final-results <number>", "Bounded fallback fetch for an oversized final account slice when no split dimensions remain", "1000")
|
|
11582
14650
|
.option("--no-windowed-final-slices", "Do not fetch pageable rows for oversized final account slices after adaptive splitting is exhausted")
|
|
14651
|
+
.option("--include-location-mismatches", "Store account rows even when the visible Sales Navigator account location does not match the source REGION filter", false)
|
|
11583
14652
|
.option("--max-split-depth <number>", "Maximum number of adaptive account split dimensions to use", "3")
|
|
11584
14653
|
.option("--max-slices <number>", "Safety cap for total account slices probed in this invocation", "1000")
|
|
11585
14654
|
.option("--probe-count <number>", "Accounts to request while probing each slice count", "1")
|
|
@@ -11685,6 +14754,7 @@ program
|
|
|
11685
14754
|
.max(300000)
|
|
11686
14755
|
.parse(options.retryMaxDelayMs),
|
|
11687
14756
|
};
|
|
14757
|
+
const filterLocationMismatches = !Boolean(options.includeLocationMismatches);
|
|
11688
14758
|
const curlFile = typeof options.curlFile === "string" && options.curlFile.trim().length > 0
|
|
11689
14759
|
? options.curlFile.trim()
|
|
11690
14760
|
: null;
|
|
@@ -11745,7 +14815,9 @@ program
|
|
|
11745
14815
|
!new URL(templateRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
11746
14816
|
throw new Error("Account split --curl-file must contain a /sales-api/salesApiAccountSearch request.");
|
|
11747
14817
|
}
|
|
11748
|
-
const config = templateRequest
|
|
14818
|
+
const config = templateRequest
|
|
14819
|
+
? null
|
|
14820
|
+
: await readLinkedInAccountSearchConfig(queryUrl);
|
|
11749
14821
|
const buildAccountApiRequest = (slicedQueryUrl) => templateRequest
|
|
11750
14822
|
? buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(slicedQueryUrl, templateRequest, pageSize)
|
|
11751
14823
|
: buildSalesNavigatorAccountApiRequestFromSearchUrl(slicedQueryUrl, config, pageSize);
|
|
@@ -11898,6 +14970,9 @@ program
|
|
|
11898
14970
|
},
|
|
11899
14971
|
exportSlice: async (attempt) => {
|
|
11900
14972
|
const parsedRequest = buildAccountApiRequest(attempt.slicedQueryUrl);
|
|
14973
|
+
const accountLocationFilter = filterLocationMismatches
|
|
14974
|
+
? buildSalesNavigatorAccountLocationFilter(attempt.slicedQueryUrl)
|
|
14975
|
+
: null;
|
|
11901
14976
|
try {
|
|
11902
14977
|
const fetchResult = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
11903
14978
|
requestedAccounts,
|
|
@@ -11906,6 +14981,7 @@ program
|
|
|
11906
14981
|
pageDelayMaxMs,
|
|
11907
14982
|
maxAllowedResults: attempt.maxResultsPerSearch,
|
|
11908
14983
|
retry,
|
|
14984
|
+
accountLocationFilter,
|
|
11909
14985
|
});
|
|
11910
14986
|
const stop = fetchResult.pacing.stop;
|
|
11911
14987
|
const partialError = describeLocalSalesNavigatorAccountPartialError(fetchResult);
|
|
@@ -11926,6 +15002,8 @@ program
|
|
|
11926
15002
|
slicedQueryUrl: attempt.slicedQueryUrl,
|
|
11927
15003
|
pacing: fetchResult.pacing,
|
|
11928
15004
|
stop,
|
|
15005
|
+
accountLocationFilter,
|
|
15006
|
+
filteredAccounts: fetchResult.filteredAccounts,
|
|
11929
15007
|
},
|
|
11930
15008
|
});
|
|
11931
15009
|
const storedAccounts = await upsertSalesNavigatorAccounts({
|
|
@@ -11935,7 +15013,10 @@ program
|
|
|
11935
15013
|
sliceId,
|
|
11936
15014
|
accounts: fetchResult.accounts,
|
|
11937
15015
|
});
|
|
11938
|
-
storedAccountCount
|
|
15016
|
+
storedAccountCount = await countSalesNavigatorAccountsForRun({
|
|
15017
|
+
store,
|
|
15018
|
+
runId,
|
|
15019
|
+
});
|
|
11939
15020
|
if (fetchResult.pacing.rateLimit) {
|
|
11940
15021
|
throw new SalesNavigatorCrawlStopError(fetchResult.pacing.rateLimit.message, {
|
|
11941
15022
|
totalResults: fetchResult.totalResults,
|
|
@@ -12035,6 +15116,9 @@ program
|
|
|
12035
15116
|
if (canFetchWindowedFinalSlice) {
|
|
12036
15117
|
try {
|
|
12037
15118
|
const parsedRequest = buildAccountApiRequest(failure.attempt.slicedQueryUrl);
|
|
15119
|
+
const accountLocationFilter = filterLocationMismatches
|
|
15120
|
+
? buildSalesNavigatorAccountLocationFilter(failure.attempt.slicedQueryUrl)
|
|
15121
|
+
: null;
|
|
12038
15122
|
const fetchResult = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
12039
15123
|
requestedAccounts,
|
|
12040
15124
|
pageSize,
|
|
@@ -12043,6 +15127,7 @@ program
|
|
|
12043
15127
|
maxAllowedResults: maxWindowedFinalResults,
|
|
12044
15128
|
retry,
|
|
12045
15129
|
allowPartialPageErrors: true,
|
|
15130
|
+
accountLocationFilter,
|
|
12046
15131
|
});
|
|
12047
15132
|
const windowedError = describeLocalSalesNavigatorAccountPartialError(fetchResult);
|
|
12048
15133
|
const sliceId = await upsertSalesNavigatorAccountSearchSlice({
|
|
@@ -12064,6 +15149,8 @@ program
|
|
|
12064
15149
|
originalError: failure.error,
|
|
12065
15150
|
pacing: fetchResult.pacing,
|
|
12066
15151
|
stop: fetchResult.pacing.stop,
|
|
15152
|
+
accountLocationFilter,
|
|
15153
|
+
filteredAccounts: fetchResult.filteredAccounts,
|
|
12067
15154
|
},
|
|
12068
15155
|
});
|
|
12069
15156
|
const storedAccounts = await upsertSalesNavigatorAccounts({
|
|
@@ -12073,7 +15160,10 @@ program
|
|
|
12073
15160
|
sliceId,
|
|
12074
15161
|
accounts: fetchResult.accounts,
|
|
12075
15162
|
});
|
|
12076
|
-
storedAccountCount
|
|
15163
|
+
storedAccountCount = await countSalesNavigatorAccountsForRun({
|
|
15164
|
+
store,
|
|
15165
|
+
runId,
|
|
15166
|
+
});
|
|
12077
15167
|
windowedFinalSlices.push({
|
|
12078
15168
|
slicedQueryUrl: failure.attempt.slicedQueryUrl,
|
|
12079
15169
|
appliedFilters: failure.attempt.appliedFilters,
|
|
@@ -12316,10 +15406,12 @@ program
|
|
|
12316
15406
|
program
|
|
12317
15407
|
.command("salesnav:accounts:local-import")
|
|
12318
15408
|
.alias("accounts:local-import")
|
|
12319
|
-
.description("Run a saved Sales Navigator account request locally and store the accounts in
|
|
15409
|
+
.description("Run a saved Sales Navigator account request locally and store the accounts in Supabase.")
|
|
12320
15410
|
.option("--curl-file <path>", "Path to a saved account-search curl request copied from the Sales Navigator network panel")
|
|
12321
15411
|
.option("--query-url <url>", "Sales Navigator account search URL. Uses the locally captured extension session automatically.")
|
|
12322
|
-
.option("--
|
|
15412
|
+
.option("--raw-results-file <path>", "Line-delimited raw Sales Navigator account results collected for --query-url")
|
|
15413
|
+
.option("--reported-total <number>", "Reported total for completeness verification when using --raw-results-file")
|
|
15414
|
+
.option("--org-id <id>", "Salesprompter workspace org id for Supabase storage. Defaults to SALESPROMPTER_ORG_ID or saved auth session org.")
|
|
12323
15415
|
.option("--max-results-per-search <number>", "Maximum expected account results for this local request", "1000")
|
|
12324
15416
|
.option("--number-of-accounts <number>", "Total accounts to import, or 'all' for the full reported result set", "all")
|
|
12325
15417
|
.option("--page-size <number>", "Accounts requested per local Sales Navigator API page", "25")
|
|
@@ -12331,8 +15423,9 @@ program
|
|
|
12331
15423
|
.option("--retry-max-delay-ms <number>", "Maximum delay for full-jitter retry backoff", "30000")
|
|
12332
15424
|
.option("--slice-preset <name>", "Slice preset label stored with the import run", "local-account-search")
|
|
12333
15425
|
.option("--allow-partial-page-errors", "Persist rows fetched before a later non-rate-limit page error", false)
|
|
15426
|
+
.option("--include-location-mismatches", "Store account rows even when the visible Sales Navigator account location does not match the source REGION filter", false)
|
|
12334
15427
|
.option("--out <path>", "Optional local JSON output path")
|
|
12335
|
-
.option("--dry-run", "Run and normalize the local account request without writing to
|
|
15428
|
+
.option("--dry-run", "Run and normalize the local account request without writing to Supabase", false)
|
|
12336
15429
|
.action(async (options) => {
|
|
12337
15430
|
const maxResultsPerSearch = z.coerce
|
|
12338
15431
|
.number()
|
|
@@ -12398,31 +15491,76 @@ program
|
|
|
12398
15491
|
.max(300000)
|
|
12399
15492
|
.parse(options.retryMaxDelayMs),
|
|
12400
15493
|
};
|
|
15494
|
+
const filterLocationMismatches = !Boolean(options.includeLocationMismatches);
|
|
12401
15495
|
const curlFile = typeof options.curlFile === "string" && options.curlFile.trim().length > 0
|
|
12402
15496
|
? options.curlFile.trim()
|
|
12403
15497
|
: null;
|
|
12404
15498
|
const queryUrl = typeof options.queryUrl === "string" && options.queryUrl.trim().length > 0
|
|
12405
15499
|
? z.string().url().parse(options.queryUrl.trim())
|
|
12406
15500
|
: null;
|
|
12407
|
-
|
|
12408
|
-
|
|
15501
|
+
const rawResultsFile = typeof options.rawResultsFile === "string" &&
|
|
15502
|
+
options.rawResultsFile.trim().length > 0
|
|
15503
|
+
? options.rawResultsFile.trim()
|
|
15504
|
+
: null;
|
|
15505
|
+
const reportedTotal = options.reportedTotal == null
|
|
15506
|
+
? null
|
|
15507
|
+
: z.coerce
|
|
15508
|
+
.number()
|
|
15509
|
+
.int()
|
|
15510
|
+
.min(1)
|
|
15511
|
+
.max(1_000_000)
|
|
15512
|
+
.parse(options.reportedTotal);
|
|
15513
|
+
if (rawResultsFile) {
|
|
15514
|
+
if (curlFile || !queryUrl) {
|
|
15515
|
+
throw new Error("--raw-results-file requires --query-url and cannot be combined with --curl-file.");
|
|
15516
|
+
}
|
|
15517
|
+
if (numberOfAccountsOption !== "all") {
|
|
15518
|
+
throw new Error("--raw-results-file requires --number-of-accounts all so completeness can be verified.");
|
|
15519
|
+
}
|
|
12409
15520
|
}
|
|
12410
|
-
|
|
12411
|
-
|
|
12412
|
-
|
|
12413
|
-
if (
|
|
12414
|
-
throw new Error("
|
|
15521
|
+
else if (Boolean(curlFile) === Boolean(queryUrl)) {
|
|
15522
|
+
throw new Error("Provide exactly one of --curl-file or --query-url, or use --raw-results-file with --query-url.");
|
|
15523
|
+
}
|
|
15524
|
+
else if (reportedTotal != null) {
|
|
15525
|
+
throw new Error("--reported-total is only supported with --raw-results-file.");
|
|
15526
|
+
}
|
|
15527
|
+
let accountLocationFilter = filterLocationMismatches
|
|
15528
|
+
? buildSalesNavigatorAccountLocationFilter(queryUrl ?? "")
|
|
15529
|
+
: null;
|
|
15530
|
+
let rawInputStats = null;
|
|
15531
|
+
let localFetch;
|
|
15532
|
+
if (rawResultsFile) {
|
|
15533
|
+
const rawInput = await readLocalSalesNavigatorRawAccountInput({
|
|
15534
|
+
filePath: rawResultsFile,
|
|
15535
|
+
sourceQueryUrl: queryUrl,
|
|
15536
|
+
reportedTotal,
|
|
15537
|
+
accountLocationFilter,
|
|
15538
|
+
});
|
|
15539
|
+
rawInputStats = rawInput.stats;
|
|
15540
|
+
localFetch = rawInput.fetchResult;
|
|
15541
|
+
}
|
|
15542
|
+
else {
|
|
15543
|
+
const parsedRequest = curlFile
|
|
15544
|
+
? parseSalesNavigatorCurlRequest(await readFile(curlFile, "utf8"))
|
|
15545
|
+
: buildSalesNavigatorAccountApiRequestFromSearchUrl(queryUrl, await readLinkedInAccountSearchConfig(queryUrl), pageSize);
|
|
15546
|
+
if (!new URL(parsedRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
15547
|
+
throw new Error("Account local import requires a /sales-api/salesApiAccountSearch request.");
|
|
15548
|
+
}
|
|
15549
|
+
if (filterLocationMismatches && !accountLocationFilter) {
|
|
15550
|
+
accountLocationFilter = buildSalesNavigatorAccountLocationFilter(queryUrl ?? parsedRequest.url);
|
|
15551
|
+
}
|
|
15552
|
+
localFetch = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
15553
|
+
requestedAccounts,
|
|
15554
|
+
startOffset,
|
|
15555
|
+
pageSize,
|
|
15556
|
+
pageDelayMinMs,
|
|
15557
|
+
pageDelayMaxMs,
|
|
15558
|
+
maxAllowedResults: maxResultsPerSearch,
|
|
15559
|
+
retry,
|
|
15560
|
+
allowPartialPageErrors: Boolean(options.allowPartialPageErrors),
|
|
15561
|
+
accountLocationFilter,
|
|
15562
|
+
});
|
|
12415
15563
|
}
|
|
12416
|
-
const localFetch = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
12417
|
-
requestedAccounts,
|
|
12418
|
-
startOffset,
|
|
12419
|
-
pageSize,
|
|
12420
|
-
pageDelayMinMs,
|
|
12421
|
-
pageDelayMaxMs,
|
|
12422
|
-
maxAllowedResults: maxResultsPerSearch,
|
|
12423
|
-
retry,
|
|
12424
|
-
allowPartialPageErrors: Boolean(options.allowPartialPageErrors),
|
|
12425
|
-
});
|
|
12426
15564
|
const accounts = localFetch.accounts;
|
|
12427
15565
|
const totalResults = localFetch.totalResults;
|
|
12428
15566
|
const dryRun = Boolean(options.dryRun);
|
|
@@ -12432,6 +15570,9 @@ program
|
|
|
12432
15570
|
if (accounts.length === 0) {
|
|
12433
15571
|
throw new Error("The local Sales Navigator account response did not contain any importable account rows.");
|
|
12434
15572
|
}
|
|
15573
|
+
if (rawResultsFile && partialError) {
|
|
15574
|
+
throw new Error(`Raw Sales Navigator account input is incomplete and was not stored. ${partialError}`);
|
|
15575
|
+
}
|
|
12435
15576
|
const sourceQueryUrl = queryUrl ?? localFetch.sourceQueryUrl;
|
|
12436
15577
|
const basePayload = {
|
|
12437
15578
|
status: "ok",
|
|
@@ -12440,6 +15581,7 @@ program
|
|
|
12440
15581
|
sourceQueryUrl,
|
|
12441
15582
|
totalResults,
|
|
12442
15583
|
discovered: accounts.length,
|
|
15584
|
+
filteredAccounts: localFetch.filteredAccounts,
|
|
12443
15585
|
fetchedPages: localFetch.fetchedPages,
|
|
12444
15586
|
requestedAccounts: requestedAccounts ?? "all",
|
|
12445
15587
|
maxResultsPerSearch,
|
|
@@ -12449,7 +15591,8 @@ program
|
|
|
12449
15591
|
rateLimit: rateLimit ?? null,
|
|
12450
15592
|
pageError: pageError ?? null,
|
|
12451
15593
|
stop: localFetch.pacing.stop,
|
|
12452
|
-
|
|
15594
|
+
rawInput: rawInputStats,
|
|
15595
|
+
accounts: rawResultsFile && !dryRun ? undefined : accounts,
|
|
12453
15596
|
};
|
|
12454
15597
|
if (dryRun) {
|
|
12455
15598
|
if (options.out) {
|
|
@@ -12484,6 +15627,9 @@ program
|
|
|
12484
15627
|
rateLimit: rateLimit ?? null,
|
|
12485
15628
|
pageError: pageError ?? null,
|
|
12486
15629
|
stop: localFetch.pacing.stop,
|
|
15630
|
+
accountLocationFilter,
|
|
15631
|
+
filteredAccounts: localFetch.filteredAccounts,
|
|
15632
|
+
rawInput: rawInputStats,
|
|
12487
15633
|
},
|
|
12488
15634
|
});
|
|
12489
15635
|
const attempt = {
|
|
@@ -12516,6 +15662,9 @@ program
|
|
|
12516
15662
|
startOffset,
|
|
12517
15663
|
pacing: localFetch.pacing,
|
|
12518
15664
|
stop: localFetch.pacing.stop,
|
|
15665
|
+
accountLocationFilter,
|
|
15666
|
+
filteredAccounts: localFetch.filteredAccounts,
|
|
15667
|
+
rawInput: rawInputStats,
|
|
12519
15668
|
},
|
|
12520
15669
|
});
|
|
12521
15670
|
const storedAccounts = await upsertSalesNavigatorAccounts({
|
|
@@ -12876,6 +16025,7 @@ program
|
|
|
12876
16025
|
.option("--max-retries <number>", "Bounded retries for transient network/5xx failures. Rate limits still stop immediately.", "1")
|
|
12877
16026
|
.option("--retry-base-delay-ms <number>", "Base delay for full-jitter retry backoff", "2000")
|
|
12878
16027
|
.option("--retry-max-delay-ms <number>", "Maximum delay for full-jitter retry backoff", "30000")
|
|
16028
|
+
.option("--include-location-mismatches", "Store account rows even when the visible Sales Navigator account location does not match the source REGION filter", false)
|
|
12879
16029
|
.option("--out <path>", "Optional local JSON output path")
|
|
12880
16030
|
.action(async (options) => {
|
|
12881
16031
|
const runId = z.string().uuid().parse(options.runId);
|
|
@@ -12943,6 +16093,7 @@ program
|
|
|
12943
16093
|
.max(300000)
|
|
12944
16094
|
.parse(options.retryMaxDelayMs),
|
|
12945
16095
|
};
|
|
16096
|
+
const filterLocationMismatches = !Boolean(options.includeLocationMismatches);
|
|
12946
16097
|
const templateRequest = parseSalesNavigatorCurlRequest(await readFile(options.curlFile, "utf8"));
|
|
12947
16098
|
if (!new URL(templateRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
12948
16099
|
throw new Error("Account repair --curl-file must contain a /sales-api/salesApiAccountSearch request.");
|
|
@@ -12997,6 +16148,9 @@ program
|
|
|
12997
16148
|
? requestedAccountsOption
|
|
12998
16149
|
: Math.min(requestedAccountsOption, remainingFromReported);
|
|
12999
16150
|
const parsedRequest = buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(slice.slicedQueryUrl, templateRequest, pageSize);
|
|
16151
|
+
const accountLocationFilter = filterLocationMismatches
|
|
16152
|
+
? buildSalesNavigatorAccountLocationFilter(slice.slicedQueryUrl)
|
|
16153
|
+
: null;
|
|
13000
16154
|
const fetchResult = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
13001
16155
|
requestedAccounts,
|
|
13002
16156
|
startOffset,
|
|
@@ -13006,6 +16160,7 @@ program
|
|
|
13006
16160
|
maxAllowedResults: 100000,
|
|
13007
16161
|
retry,
|
|
13008
16162
|
allowPartialPageErrors: true,
|
|
16163
|
+
accountLocationFilter,
|
|
13009
16164
|
});
|
|
13010
16165
|
const rateLimit = fetchResult.pacing.rateLimit;
|
|
13011
16166
|
const pageError = fetchResult.pacing.pageError;
|
|
@@ -13037,6 +16192,8 @@ program
|
|
|
13037
16192
|
previousStoredAccounts: slice.storedAccounts,
|
|
13038
16193
|
startOffset,
|
|
13039
16194
|
fetchedAccounts: fetchResult.accounts.length,
|
|
16195
|
+
filteredAccounts: fetchResult.filteredAccounts,
|
|
16196
|
+
accountLocationFilter,
|
|
13040
16197
|
},
|
|
13041
16198
|
pacing: fetchResult.pacing,
|
|
13042
16199
|
stop: fetchResult.pacing.stop,
|