salesprompter-cli 0.1.43 → 0.1.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/cli.js +2587 -103
- package/dist/linkedin-session.js +103 -17
- package/dist/sales-navigator.js +42 -15
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { spawn } from "node:child_process";
|
|
3
3
|
import { access, appendFile, mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
|
4
|
+
import { createServer } from "node:http";
|
|
4
5
|
import { createRequire } from "node:module";
|
|
5
6
|
import os from "node:os";
|
|
6
7
|
import path from "node:path";
|
|
@@ -26,10 +27,10 @@ import { InstantlySyncProvider } from "./instantly.js";
|
|
|
26
27
|
import { backfillLinkedInCompanies } from "./linkedin-companies.js";
|
|
27
28
|
import { parseLinkedInCompanyPage } from "./linkedin-companies.js";
|
|
28
29
|
import { crawlLinkedInProductCategory } from "./linkedin-products.js";
|
|
29
|
-
import { claimValidatedSalesNavigatorSessionCookieForCli, createLinkedInSessionSupabaseClient, resolveConfiguredEnvValue } from "./linkedin-session.js";
|
|
30
|
+
import { claimLinkedInSessionCookieForCli, claimValidatedSalesNavigatorSessionCookieForCli, createLinkedInSessionSupabaseClient, recordLinkedInSessionCookieAudit, resolveConfiguredEnvValue } from "./linkedin-session.js";
|
|
30
31
|
import { buildLeadlistsFunnelQueries } from "./leadlists-funnel.js";
|
|
31
32
|
import { readJsonFile, splitCsv, writeJsonFile, writeTextFile } from "./io.js";
|
|
32
|
-
import { buildSalesNavigatorCrawlPreview, createSalesNavigatorCrawlSeed, DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS, DEFAULT_SALES_NAVIGATOR_CRAWL_DIMENSIONS, buildSalesNavigatorPeopleSearchUrl, buildSalesNavigatorPeopleSlice, deriveSalesNavigatorTitleQuerySeeds, executeSalesNavigatorAdaptiveCrawl, expandSalesNavigatorCrawlAttempt, SalesNavigatorCrawlStopError, SalesNavigatorSliceTooBroadError } from "./sales-navigator.js";
|
|
33
|
+
import { buildSalesNavigatorCrawlPreview, buildSalesNavigatorAccountQuery, createSalesNavigatorCrawlSeed, DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS, DEFAULT_SALES_NAVIGATOR_CRAWL_DIMENSIONS, buildSalesNavigatorPeopleSearchUrl, buildSalesNavigatorPeopleSlice, deriveSalesNavigatorTitleQuerySeeds, executeSalesNavigatorAdaptiveCrawl, expandSalesNavigatorCrawlAttempt, SalesNavigatorCrawlStopError, SalesNavigatorSliceTooBroadError } from "./sales-navigator.js";
|
|
33
34
|
import { buildSalesNavigatorHistoricalBackfillPlan, ensureSalesNavigatorPeopleCount, resolveSalesNavigatorHistoricalBackfillConfig, resolveSalesNavigatorHistoricalBackfillResumeState, resolveSalesNavigatorHistoricalBackfillOrgId, salesNavigatorHistoricalBackfillDefaults } from "./salesnav-backfill.js";
|
|
34
35
|
const require = createRequire(import.meta.url);
|
|
35
36
|
const { version: packageVersion } = require("../package.json");
|
|
@@ -123,6 +124,92 @@ const LinkedInCompanyBackfillStatusResponseSchema = z.object({
|
|
|
123
124
|
failureCode: z.string().nullable().optional(),
|
|
124
125
|
failureMessage: z.string().nullable().optional()
|
|
125
126
|
});
|
|
127
|
+
const AffiliateCampaignLaunchResponseSchema = z.object({
|
|
128
|
+
status: z.enum(["ok", "partial"]),
|
|
129
|
+
dryRun: z.boolean(),
|
|
130
|
+
product: z.object({
|
|
131
|
+
domain: z.string().min(1),
|
|
132
|
+
affiliateLink: z.string().url()
|
|
133
|
+
}),
|
|
134
|
+
audience: z.object({
|
|
135
|
+
sourceType: z.enum(["sales_navigator_people", "linkedin_content"]),
|
|
136
|
+
linkedInUrl: z.string().url(),
|
|
137
|
+
maxResults: z.number().int().positive(),
|
|
138
|
+
peoplePreview: z.array(z.record(z.string(), z.unknown())).optional(),
|
|
139
|
+
postsPreview: z.array(z.record(z.string(), z.unknown())).optional()
|
|
140
|
+
}),
|
|
141
|
+
extraction: z
|
|
142
|
+
.object({
|
|
143
|
+
provider: z.string().min(1),
|
|
144
|
+
monitorId: z.string().min(1).optional(),
|
|
145
|
+
agentId: z.string().min(1).optional(),
|
|
146
|
+
containerId: z.string().min(1).optional()
|
|
147
|
+
})
|
|
148
|
+
.passthrough()
|
|
149
|
+
.optional(),
|
|
150
|
+
previewUrl: z.string().nullable().optional(),
|
|
151
|
+
next: z.string().optional()
|
|
152
|
+
});
|
|
153
|
+
const AffiliateCampaignLocalResultSchema = z
|
|
154
|
+
.object({
|
|
155
|
+
profileUrl: z.string().min(1),
|
|
156
|
+
})
|
|
157
|
+
.passthrough();
|
|
158
|
+
const DatabaseTimestampSchema = z.string().min(1);
|
|
159
|
+
const AffiliateOutreachRunSchema = z.object({
|
|
160
|
+
id: z.string().uuid(),
|
|
161
|
+
audience_run_id: z.string().uuid(),
|
|
162
|
+
organization_id: z.string().min(1),
|
|
163
|
+
status: z.enum(["preparing", "ready", "active", "failed"]),
|
|
164
|
+
provider: z.literal("instantly"),
|
|
165
|
+
instantly_campaign_id: z.string().min(1).nullable(),
|
|
166
|
+
instantly_campaign_name: z.string().min(1).nullable(),
|
|
167
|
+
sequence: z.array(z.object({
|
|
168
|
+
step: z.number().int().positive(),
|
|
169
|
+
subject: z.string(),
|
|
170
|
+
body: z.string(),
|
|
171
|
+
rationale: z.string(),
|
|
172
|
+
delayDays: z.number().int().nonnegative(),
|
|
173
|
+
variants: z
|
|
174
|
+
.array(z.object({
|
|
175
|
+
subject: z.string(),
|
|
176
|
+
body: z.string()
|
|
177
|
+
}))
|
|
178
|
+
.min(1)
|
|
179
|
+
.max(3)
|
|
180
|
+
.optional()
|
|
181
|
+
})),
|
|
182
|
+
enriched_leads: z.array(z.object({
|
|
183
|
+
rowIndex: z.number().int().nonnegative(),
|
|
184
|
+
fullName: z.string().min(1),
|
|
185
|
+
firstName: z.string().min(1),
|
|
186
|
+
lastName: z.string().min(1),
|
|
187
|
+
jobTitle: z.string().nullable(),
|
|
188
|
+
companyName: z.string().min(1),
|
|
189
|
+
companyDomain: z.string().min(1),
|
|
190
|
+
location: z.string().nullable(),
|
|
191
|
+
profileUrl: z.string().nullable(),
|
|
192
|
+
email: z.string().email(),
|
|
193
|
+
emailScore: z.number().min(0).max(100),
|
|
194
|
+
acceptAll: z.boolean(),
|
|
195
|
+
emailSource: z.enum(["hunter", "phantombuster"]).optional()
|
|
196
|
+
})),
|
|
197
|
+
stats: z.record(z.string(), z.unknown()),
|
|
198
|
+
settings: z.record(z.string(), z.unknown()),
|
|
199
|
+
error_message: z.string().nullable(),
|
|
200
|
+
activated_at: DatabaseTimestampSchema.nullable(),
|
|
201
|
+
created_at: DatabaseTimestampSchema,
|
|
202
|
+
updated_at: DatabaseTimestampSchema
|
|
203
|
+
});
|
|
204
|
+
const AffiliateOutreachResponseSchema = z.object({
|
|
205
|
+
status: z.literal("ok"),
|
|
206
|
+
outreach: AffiliateOutreachRunSchema
|
|
207
|
+
});
|
|
208
|
+
const AffiliateOutreachEnrichStartResponseSchema = z.object({
|
|
209
|
+
status: z.literal("accepted"),
|
|
210
|
+
audienceRunId: z.string().uuid(),
|
|
211
|
+
workflowRunId: z.string().min(1)
|
|
212
|
+
});
|
|
126
213
|
const SalesNavigatorLocalImportResponseSchema = z.object({
|
|
127
214
|
status: z.literal("ok"),
|
|
128
215
|
runId: z.string().min(1),
|
|
@@ -398,7 +485,15 @@ const cliPacks = [
|
|
|
398
485
|
slug: "outreach",
|
|
399
486
|
title: "Outreach",
|
|
400
487
|
summary: "Prepare and sync qualified leads into downstream systems.",
|
|
401
|
-
commands: [
|
|
488
|
+
commands: [
|
|
489
|
+
"affiliate:run",
|
|
490
|
+
"affiliate:status",
|
|
491
|
+
"affiliate:enrich",
|
|
492
|
+
"affiliate:activate",
|
|
493
|
+
"affiliate:launch",
|
|
494
|
+
"sync:outreach",
|
|
495
|
+
"sync:crm"
|
|
496
|
+
],
|
|
402
497
|
installStatus: "included"
|
|
403
498
|
}
|
|
404
499
|
];
|
|
@@ -443,6 +538,11 @@ const helpVisibleCommandNames = new Set([
|
|
|
443
538
|
"leads:enrich",
|
|
444
539
|
"leads:score",
|
|
445
540
|
"leads:pipeline",
|
|
541
|
+
"affiliate:run",
|
|
542
|
+
"affiliate:status",
|
|
543
|
+
"affiliate:enrich",
|
|
544
|
+
"affiliate:activate",
|
|
545
|
+
"affiliate:launch",
|
|
446
546
|
"sync:crm",
|
|
447
547
|
"sync:outreach",
|
|
448
548
|
"linkedin-companies:backfill",
|
|
@@ -1300,9 +1400,74 @@ function normalizeLinkedInDirectLookupCookieHeader(cookie) {
|
|
|
1300
1400
|
}
|
|
1301
1401
|
return `li_at=${trimmed}`;
|
|
1302
1402
|
}
|
|
1403
|
+
function mergeLinkedInCookieHeader(cookieHeader, setCookieHeaders) {
|
|
1404
|
+
const cookies = new Map();
|
|
1405
|
+
for (const part of cookieHeader.split(";")) {
|
|
1406
|
+
const separatorIndex = part.indexOf("=");
|
|
1407
|
+
if (separatorIndex <= 0) {
|
|
1408
|
+
continue;
|
|
1409
|
+
}
|
|
1410
|
+
cookies.set(part.slice(0, separatorIndex).trim(), part.slice(separatorIndex + 1).trim());
|
|
1411
|
+
}
|
|
1412
|
+
for (const setCookieHeader of setCookieHeaders) {
|
|
1413
|
+
const firstPart = setCookieHeader.split(";", 1)[0] ?? "";
|
|
1414
|
+
const separatorIndex = firstPart.indexOf("=");
|
|
1415
|
+
if (separatorIndex <= 0) {
|
|
1416
|
+
continue;
|
|
1417
|
+
}
|
|
1418
|
+
cookies.set(firstPart.slice(0, separatorIndex).trim(), firstPart.slice(separatorIndex + 1).trim());
|
|
1419
|
+
}
|
|
1420
|
+
return [...cookies.entries()]
|
|
1421
|
+
.map(([name, value]) => `${name}=${value}`)
|
|
1422
|
+
.join("; ");
|
|
1423
|
+
}
|
|
1424
|
+
async function bootstrapLinkedInAccountSearchConfig(params) {
|
|
1425
|
+
const cookie = normalizeLinkedInDirectLookupCookieHeader(params.sessionCookie);
|
|
1426
|
+
const existingCsrfToken = params.csrfToken || deriveCsrfTokenFromCookie(cookie);
|
|
1427
|
+
if (existingCsrfToken) {
|
|
1428
|
+
return {
|
|
1429
|
+
csrfToken: existingCsrfToken,
|
|
1430
|
+
identity: params.identity,
|
|
1431
|
+
cookie,
|
|
1432
|
+
userAgent: params.userAgent,
|
|
1433
|
+
};
|
|
1434
|
+
}
|
|
1435
|
+
const response = await fetch(params.queryUrl, {
|
|
1436
|
+
redirect: "manual",
|
|
1437
|
+
headers: {
|
|
1438
|
+
accept: "text/html,application/xhtml+xml",
|
|
1439
|
+
cookie,
|
|
1440
|
+
"user-agent": params.userAgent,
|
|
1441
|
+
},
|
|
1442
|
+
});
|
|
1443
|
+
if (response.status === 429 || response.status === 999) {
|
|
1444
|
+
throw new Error("Sales Navigator is rate-limited while bootstrapping the Account Search API session.");
|
|
1445
|
+
}
|
|
1446
|
+
if (response.status >= 300 && response.status < 400) {
|
|
1447
|
+
await response.arrayBuffer();
|
|
1448
|
+
return null;
|
|
1449
|
+
}
|
|
1450
|
+
const headersWithSetCookie = response.headers;
|
|
1451
|
+
const setCookieHeaders = headersWithSetCookie.getSetCookie?.() ??
|
|
1452
|
+
(response.headers.get("set-cookie")
|
|
1453
|
+
? [response.headers.get("set-cookie")]
|
|
1454
|
+
: []);
|
|
1455
|
+
const refreshedCookie = mergeLinkedInCookieHeader(cookie, setCookieHeaders);
|
|
1456
|
+
await response.arrayBuffer();
|
|
1457
|
+
const csrfToken = deriveCsrfTokenFromCookie(refreshedCookie);
|
|
1458
|
+
if (!csrfToken) {
|
|
1459
|
+
return null;
|
|
1460
|
+
}
|
|
1461
|
+
return {
|
|
1462
|
+
csrfToken,
|
|
1463
|
+
identity: params.identity,
|
|
1464
|
+
cookie: refreshedCookie,
|
|
1465
|
+
userAgent: params.userAgent,
|
|
1466
|
+
};
|
|
1467
|
+
}
|
|
1303
1468
|
function parseLocalLinkedInExtensionTokenLog(content) {
|
|
1304
1469
|
const matches = [
|
|
1305
|
-
...content.matchAll(/\{"csrfToken":"([^"]+)","extractedFrom":"sales-api\/
|
|
1470
|
+
...content.matchAll(/\{"csrfToken":"([^"]+)","extractedFrom":"sales-api\/salesApi(?:Lead|Account)Search"[\s\S]*?"linkedInIdentity":"([^"]+)"[\s\S]*?"sessionCookie":"([\s\S]*?)","syncStatus":"(success|captured)"[\s\S]*?"userAgent":"([^"]+)"\}/g)
|
|
1306
1471
|
];
|
|
1307
1472
|
const last = matches.at(-1);
|
|
1308
1473
|
if (!last) {
|
|
@@ -1331,6 +1496,21 @@ async function readLocalLinkedInExtensionTokenLog(filePath) {
|
|
|
1331
1496
|
return null;
|
|
1332
1497
|
}
|
|
1333
1498
|
}
|
|
1499
|
+
async function sortPathsByMostRecentlyModified(paths) {
|
|
1500
|
+
const candidates = await Promise.all(paths.map(async (filePath) => {
|
|
1501
|
+
try {
|
|
1502
|
+
const fileStat = await stat(filePath);
|
|
1503
|
+
return { filePath, modifiedAtMs: fileStat.mtimeMs };
|
|
1504
|
+
}
|
|
1505
|
+
catch {
|
|
1506
|
+
return { filePath, modifiedAtMs: 0 };
|
|
1507
|
+
}
|
|
1508
|
+
}));
|
|
1509
|
+
return candidates
|
|
1510
|
+
.sort((left, right) => right.modifiedAtMs - left.modifiedAtMs ||
|
|
1511
|
+
right.filePath.localeCompare(left.filePath))
|
|
1512
|
+
.map((candidate) => candidate.filePath);
|
|
1513
|
+
}
|
|
1334
1514
|
async function listChromeExtensionTokenLogCandidates() {
|
|
1335
1515
|
const overrideFile = normalizeLookupWhitespace(process.env.SALESPROMPTER_LINKEDIN_EXTENSION_TOKENS_LOG_PATH);
|
|
1336
1516
|
if (overrideFile) {
|
|
@@ -1340,11 +1520,9 @@ async function listChromeExtensionTokenLogCandidates() {
|
|
|
1340
1520
|
if (overrideDir) {
|
|
1341
1521
|
try {
|
|
1342
1522
|
const files = await readdir(overrideDir);
|
|
1343
|
-
return files
|
|
1523
|
+
return sortPathsByMostRecentlyModified(files
|
|
1344
1524
|
.filter((file) => file.endsWith(".log") || file.endsWith(".ldb"))
|
|
1345
|
-
.map((file) => path.join(overrideDir, file))
|
|
1346
|
-
.sort()
|
|
1347
|
-
.reverse();
|
|
1525
|
+
.map((file) => path.join(overrideDir, file)));
|
|
1348
1526
|
}
|
|
1349
1527
|
catch {
|
|
1350
1528
|
return [];
|
|
@@ -1382,7 +1560,7 @@ async function listChromeExtensionTokenLogCandidates() {
|
|
|
1382
1560
|
continue;
|
|
1383
1561
|
}
|
|
1384
1562
|
for (const file of files) {
|
|
1385
|
-
if (!file.endsWith(".log")) {
|
|
1563
|
+
if (!file.endsWith(".log") && !file.endsWith(".ldb")) {
|
|
1386
1564
|
continue;
|
|
1387
1565
|
}
|
|
1388
1566
|
paths.push(path.join(extensionDir, file));
|
|
@@ -1390,7 +1568,7 @@ async function listChromeExtensionTokenLogCandidates() {
|
|
|
1390
1568
|
}
|
|
1391
1569
|
}
|
|
1392
1570
|
}
|
|
1393
|
-
return paths
|
|
1571
|
+
return sortPathsByMostRecentlyModified(paths);
|
|
1394
1572
|
}
|
|
1395
1573
|
async function readLocalLinkedInExtensionDirectLookupConfig() {
|
|
1396
1574
|
const candidates = await listChromeExtensionTokenLogCandidates();
|
|
@@ -1464,11 +1642,12 @@ async function readStoredLinkedInDirectLookupConfig() {
|
|
|
1464
1642
|
return null;
|
|
1465
1643
|
}
|
|
1466
1644
|
const lookupResult = await supabase
|
|
1467
|
-
.from("
|
|
1645
|
+
.from("salesprompter_linkedin_session_cookies")
|
|
1468
1646
|
.select("metadata")
|
|
1469
1647
|
.eq("session_cookie_sha256", claimed.sessionCookieSha256)
|
|
1470
1648
|
.maybeSingle();
|
|
1471
|
-
if (lookupResult.error &&
|
|
1649
|
+
if (lookupResult.error &&
|
|
1650
|
+
!/salesprompter_linkedin_session_cookies/.test(lookupResult.error.message)) {
|
|
1472
1651
|
throw new Error(`Failed to read stored LinkedIn session metadata: ${lookupResult.error.message}`);
|
|
1473
1652
|
}
|
|
1474
1653
|
const metadata = lookupResult.data?.metadata;
|
|
@@ -1488,7 +1667,195 @@ async function readStoredLinkedInDirectLookupConfig() {
|
|
|
1488
1667
|
userAgent
|
|
1489
1668
|
};
|
|
1490
1669
|
}
|
|
1670
|
+
async function readStoredLinkedInAccountSearchConfig(queryUrl) {
|
|
1671
|
+
const supabase = createLinkedInSessionSupabaseClient();
|
|
1672
|
+
if (!supabase) {
|
|
1673
|
+
return null;
|
|
1674
|
+
}
|
|
1675
|
+
let lastFailure = "No Account Search API-capable LinkedIn session was available.";
|
|
1676
|
+
const configuredAttempts = Number(process.env.SALESPROMPTER_ACCOUNT_SEARCH_SESSION_ATTEMPTS);
|
|
1677
|
+
const maxAttempts = Number.isFinite(configuredAttempts) && configuredAttempts > 0
|
|
1678
|
+
? Math.min(64, Math.trunc(configuredAttempts))
|
|
1679
|
+
: 24;
|
|
1680
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
1681
|
+
const claimed = await claimLinkedInSessionCookieForCli({
|
|
1682
|
+
source: "cli_account_search",
|
|
1683
|
+
supabase,
|
|
1684
|
+
});
|
|
1685
|
+
if (!claimed) {
|
|
1686
|
+
return null;
|
|
1687
|
+
}
|
|
1688
|
+
const lookupResult = await supabase
|
|
1689
|
+
.from("salesprompter_linkedin_session_cookies")
|
|
1690
|
+
.select("metadata")
|
|
1691
|
+
.eq("session_cookie_sha256", claimed.sessionCookieSha256)
|
|
1692
|
+
.maybeSingle();
|
|
1693
|
+
if (lookupResult.error) {
|
|
1694
|
+
throw new Error(`Failed to read stored LinkedIn Account Search metadata: ${lookupResult.error.message}`);
|
|
1695
|
+
}
|
|
1696
|
+
const metadata = lookupResult.data?.metadata;
|
|
1697
|
+
const identity = extractStoredLinkedInLookupValue(metadata, "linkedInIdentity");
|
|
1698
|
+
const csrfToken = extractStoredLinkedInLookupValue(metadata, "csrfToken") ||
|
|
1699
|
+
deriveCsrfTokenFromCookie(claimed.sessionCookie);
|
|
1700
|
+
const userAgent = process.env.SALESPROMPTER_LINKEDIN_USER_AGENT?.trim() ||
|
|
1701
|
+
extractStoredLinkedInLookupValue(metadata, "userAgent") ||
|
|
1702
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36";
|
|
1703
|
+
let config = null;
|
|
1704
|
+
try {
|
|
1705
|
+
config = await bootstrapLinkedInAccountSearchConfig({
|
|
1706
|
+
queryUrl,
|
|
1707
|
+
sessionCookie: claimed.sessionCookie,
|
|
1708
|
+
identity,
|
|
1709
|
+
csrfToken,
|
|
1710
|
+
userAgent,
|
|
1711
|
+
});
|
|
1712
|
+
}
|
|
1713
|
+
catch (error) {
|
|
1714
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1715
|
+
if (/rate-limit/i.test(message)) {
|
|
1716
|
+
throw error;
|
|
1717
|
+
}
|
|
1718
|
+
lastFailure = message;
|
|
1719
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1720
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1721
|
+
source: "cli_account_search_api_preflight",
|
|
1722
|
+
feedStatus: null,
|
|
1723
|
+
salesNavigatorStatus: "error",
|
|
1724
|
+
recruiterStatus: null,
|
|
1725
|
+
productClass: "sales_navigator",
|
|
1726
|
+
isActive: true,
|
|
1727
|
+
inactiveReason: null,
|
|
1728
|
+
validationError: message,
|
|
1729
|
+
details: {
|
|
1730
|
+
attemptNumber: attempt,
|
|
1731
|
+
capability: "account_search_api_bootstrap",
|
|
1732
|
+
},
|
|
1733
|
+
});
|
|
1734
|
+
continue;
|
|
1735
|
+
}
|
|
1736
|
+
if (!config) {
|
|
1737
|
+
lastFailure =
|
|
1738
|
+
"Stored LinkedIn session could not bootstrap Account Search credentials.";
|
|
1739
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1740
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1741
|
+
source: "cli_account_search_api_preflight",
|
|
1742
|
+
feedStatus: null,
|
|
1743
|
+
salesNavigatorStatus: "error",
|
|
1744
|
+
recruiterStatus: null,
|
|
1745
|
+
productClass: "sales_navigator",
|
|
1746
|
+
isActive: false,
|
|
1747
|
+
inactiveReason: "salesnav_account_search_bootstrap_failed",
|
|
1748
|
+
validationError: "salesnav_account_search_bootstrap_failed",
|
|
1749
|
+
details: {
|
|
1750
|
+
attemptNumber: attempt,
|
|
1751
|
+
hasStoredIdentity: Boolean(identity),
|
|
1752
|
+
hasStoredCsrfToken: Boolean(csrfToken),
|
|
1753
|
+
},
|
|
1754
|
+
});
|
|
1755
|
+
continue;
|
|
1756
|
+
}
|
|
1757
|
+
const probe = await probeLinkedInAccountSearchConfig(queryUrl, config);
|
|
1758
|
+
if (probe.status === "ok") {
|
|
1759
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1760
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1761
|
+
source: "cli_account_search_api_preflight",
|
|
1762
|
+
feedStatus: null,
|
|
1763
|
+
salesNavigatorStatus: "ok",
|
|
1764
|
+
recruiterStatus: null,
|
|
1765
|
+
productClass: "sales_navigator",
|
|
1766
|
+
isActive: true,
|
|
1767
|
+
inactiveReason: null,
|
|
1768
|
+
validationError: null,
|
|
1769
|
+
details: {
|
|
1770
|
+
attemptNumber: attempt,
|
|
1771
|
+
statusCode: probe.statusCode,
|
|
1772
|
+
capability: "account_search_api",
|
|
1773
|
+
},
|
|
1774
|
+
});
|
|
1775
|
+
return config;
|
|
1776
|
+
}
|
|
1777
|
+
lastFailure =
|
|
1778
|
+
probe.validationError ??
|
|
1779
|
+
`Sales Navigator Account Search preflight returned ${probe.status}.`;
|
|
1780
|
+
if (probe.status === "rate_limited") {
|
|
1781
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1782
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1783
|
+
source: "cli_account_search_api_preflight",
|
|
1784
|
+
feedStatus: null,
|
|
1785
|
+
salesNavigatorStatus: "rate_limited",
|
|
1786
|
+
recruiterStatus: null,
|
|
1787
|
+
productClass: "sales_navigator",
|
|
1788
|
+
isActive: true,
|
|
1789
|
+
inactiveReason: null,
|
|
1790
|
+
validationError: probe.validationError,
|
|
1791
|
+
details: {
|
|
1792
|
+
attemptNumber: attempt,
|
|
1793
|
+
statusCode: probe.statusCode,
|
|
1794
|
+
capability: "account_search_api",
|
|
1795
|
+
},
|
|
1796
|
+
});
|
|
1797
|
+
throw new Error("Sales Navigator Account Search API is rate-limited. The CLI stopped without retrying this session.");
|
|
1798
|
+
}
|
|
1799
|
+
if (probe.status === "sales_seat_required") {
|
|
1800
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1801
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1802
|
+
source: "cli_account_search_api_preflight",
|
|
1803
|
+
feedStatus: null,
|
|
1804
|
+
salesNavigatorStatus: "upsell",
|
|
1805
|
+
recruiterStatus: null,
|
|
1806
|
+
productClass: null,
|
|
1807
|
+
isActive: false,
|
|
1808
|
+
inactiveReason: "salesnav_account_search_seat_required",
|
|
1809
|
+
validationError: probe.validationError,
|
|
1810
|
+
details: {
|
|
1811
|
+
attemptNumber: attempt,
|
|
1812
|
+
statusCode: probe.statusCode,
|
|
1813
|
+
capability: "account_search_api",
|
|
1814
|
+
},
|
|
1815
|
+
});
|
|
1816
|
+
continue;
|
|
1817
|
+
}
|
|
1818
|
+
if (probe.status === "unauthorized" || probe.status === "forbidden") {
|
|
1819
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1820
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1821
|
+
source: "cli_account_search_api_preflight",
|
|
1822
|
+
feedStatus: null,
|
|
1823
|
+
salesNavigatorStatus: "logged_out",
|
|
1824
|
+
recruiterStatus: null,
|
|
1825
|
+
productClass: "sales_navigator",
|
|
1826
|
+
isActive: false,
|
|
1827
|
+
inactiveReason: "salesnav_account_search_unauthorized",
|
|
1828
|
+
validationError: probe.validationError,
|
|
1829
|
+
details: {
|
|
1830
|
+
attemptNumber: attempt,
|
|
1831
|
+
statusCode: probe.statusCode,
|
|
1832
|
+
capability: "account_search_api",
|
|
1833
|
+
},
|
|
1834
|
+
});
|
|
1835
|
+
continue;
|
|
1836
|
+
}
|
|
1837
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1838
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1839
|
+
source: "cli_account_search_api_preflight",
|
|
1840
|
+
feedStatus: null,
|
|
1841
|
+
salesNavigatorStatus: "error",
|
|
1842
|
+
recruiterStatus: null,
|
|
1843
|
+
productClass: "sales_navigator",
|
|
1844
|
+
isActive: true,
|
|
1845
|
+
inactiveReason: null,
|
|
1846
|
+
validationError: probe.validationError,
|
|
1847
|
+
details: {
|
|
1848
|
+
attemptNumber: attempt,
|
|
1849
|
+
statusCode: probe.statusCode,
|
|
1850
|
+
capability: "account_search_api",
|
|
1851
|
+
probeStatus: probe.status,
|
|
1852
|
+
},
|
|
1853
|
+
});
|
|
1854
|
+
}
|
|
1855
|
+
throw new Error(lastFailure);
|
|
1856
|
+
}
|
|
1491
1857
|
let cachedLinkedInDirectLookupConfig = null;
|
|
1858
|
+
let cachedLinkedInAccountSearchConfig = null;
|
|
1492
1859
|
async function readLinkedInDirectLookupConfig() {
|
|
1493
1860
|
if (cachedLinkedInDirectLookupConfig) {
|
|
1494
1861
|
return cachedLinkedInDirectLookupConfig;
|
|
@@ -1519,6 +1886,46 @@ async function readLinkedInDirectLookupConfig() {
|
|
|
1519
1886
|
}
|
|
1520
1887
|
throw new Error("Missing LinkedIn direct lookup session. Set LINKEDIN_CSRF_TOKEN, LINKEDIN_X_LI_IDENTITY, and LINKEDIN_SALES_NAV_COOKIE, or connect the Salesprompter Chrome extension to sync your LinkedIn session to salesprompter.ai.");
|
|
1521
1888
|
}
|
|
1889
|
+
async function readLinkedInAccountSearchConfig(queryUrl) {
|
|
1890
|
+
if (cachedLinkedInAccountSearchConfig) {
|
|
1891
|
+
return cachedLinkedInAccountSearchConfig;
|
|
1892
|
+
}
|
|
1893
|
+
const envConfig = readLinkedInDirectLookupEnvConfig();
|
|
1894
|
+
if (envConfig) {
|
|
1895
|
+
const probe = await probeLinkedInAccountSearchConfig(queryUrl, envConfig);
|
|
1896
|
+
assertLinkedInAccountSearchConfigProbe(probe);
|
|
1897
|
+
cachedLinkedInAccountSearchConfig = envConfig;
|
|
1898
|
+
return envConfig;
|
|
1899
|
+
}
|
|
1900
|
+
let storedError = null;
|
|
1901
|
+
try {
|
|
1902
|
+
const storedConfig = await readStoredLinkedInAccountSearchConfig(queryUrl);
|
|
1903
|
+
if (storedConfig) {
|
|
1904
|
+
cachedLinkedInAccountSearchConfig = storedConfig;
|
|
1905
|
+
return storedConfig;
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
catch (error) {
|
|
1909
|
+
storedError = error;
|
|
1910
|
+
if (isExplicitLinkedInSessionVaultEnabled() ||
|
|
1911
|
+
/rate-limit/i.test(error instanceof Error ? error.message : String(error))) {
|
|
1912
|
+
throw error;
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1915
|
+
if (!shouldDisableLinkedInDirectLookupAutodiscovery()) {
|
|
1916
|
+
const localExtensionConfig = await readLocalLinkedInExtensionDirectLookupConfig();
|
|
1917
|
+
if (localExtensionConfig) {
|
|
1918
|
+
const probe = await probeLinkedInAccountSearchConfig(queryUrl, localExtensionConfig);
|
|
1919
|
+
assertLinkedInAccountSearchConfigProbe(probe);
|
|
1920
|
+
cachedLinkedInAccountSearchConfig = localExtensionConfig;
|
|
1921
|
+
return localExtensionConfig;
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
1924
|
+
if (storedError) {
|
|
1925
|
+
throw storedError;
|
|
1926
|
+
}
|
|
1927
|
+
throw new Error("Missing LinkedIn Account Search API session. Connect the Salesprompter Chrome extension or configure the Salesprompter session vault.");
|
|
1928
|
+
}
|
|
1522
1929
|
function generateLinkedInSessionId() {
|
|
1523
1930
|
return Array.from({ length: 22 }, () => "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789".charAt(Math.floor(Math.random() * 62))).join("");
|
|
1524
1931
|
}
|
|
@@ -2198,6 +2605,203 @@ function normalizeLocalSalesNavigatorAccounts(responseBody, queryUrl) {
|
|
|
2198
2605
|
}
|
|
2199
2606
|
return accounts;
|
|
2200
2607
|
}
|
|
2608
|
+
async function readLocalSalesNavigatorRawAccountInput(params) {
|
|
2609
|
+
const filePath = path.resolve(params.filePath);
|
|
2610
|
+
const content = await readFile(filePath, "utf8");
|
|
2611
|
+
const accounts = [];
|
|
2612
|
+
const seenAccounts = new Set();
|
|
2613
|
+
const requestUrls = new Set();
|
|
2614
|
+
let parsedLines = 0;
|
|
2615
|
+
let parsedElements = 0;
|
|
2616
|
+
let invalidElements = 0;
|
|
2617
|
+
let duplicateAccounts = 0;
|
|
2618
|
+
let filteredAccounts = 0;
|
|
2619
|
+
const addElement = (element, requestUrl) => {
|
|
2620
|
+
parsedElements += 1;
|
|
2621
|
+
if (typeof element !== "object" || element === null || Array.isArray(element)) {
|
|
2622
|
+
invalidElements += 1;
|
|
2623
|
+
return;
|
|
2624
|
+
}
|
|
2625
|
+
const account = normalizeLocalSalesNavigatorAccount(element, requestUrl);
|
|
2626
|
+
if (!account) {
|
|
2627
|
+
invalidElements += 1;
|
|
2628
|
+
return;
|
|
2629
|
+
}
|
|
2630
|
+
if (seenAccounts.has(account.accountKey)) {
|
|
2631
|
+
duplicateAccounts += 1;
|
|
2632
|
+
return;
|
|
2633
|
+
}
|
|
2634
|
+
seenAccounts.add(account.accountKey);
|
|
2635
|
+
if (!salesNavigatorAccountMatchesLocationFilter(account, params.accountLocationFilter)) {
|
|
2636
|
+
filteredAccounts += 1;
|
|
2637
|
+
return;
|
|
2638
|
+
}
|
|
2639
|
+
accounts.push(account);
|
|
2640
|
+
};
|
|
2641
|
+
const lines = content.split(/\r?\n/);
|
|
2642
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
2643
|
+
const line = lines[index]?.trim() ?? "";
|
|
2644
|
+
if (!line) {
|
|
2645
|
+
continue;
|
|
2646
|
+
}
|
|
2647
|
+
parsedLines += 1;
|
|
2648
|
+
let value;
|
|
2649
|
+
try {
|
|
2650
|
+
value = JSON.parse(line);
|
|
2651
|
+
}
|
|
2652
|
+
catch (error) {
|
|
2653
|
+
throw new Error(`Raw Sales Navigator account input has invalid JSON on line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2654
|
+
}
|
|
2655
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
2656
|
+
invalidElements += 1;
|
|
2657
|
+
continue;
|
|
2658
|
+
}
|
|
2659
|
+
const record = value;
|
|
2660
|
+
const requestUrlCandidate = [
|
|
2661
|
+
record.queryUrl,
|
|
2662
|
+
record.requestUrl,
|
|
2663
|
+
record.slicedQueryUrl,
|
|
2664
|
+
].find((candidate) => typeof candidate === "string" && candidate.trim().length > 0);
|
|
2665
|
+
const requestUrl = requestUrlCandidate?.trim() ?? params.sourceQueryUrl;
|
|
2666
|
+
requestUrls.add(requestUrl);
|
|
2667
|
+
if (Array.isArray(record.elements)) {
|
|
2668
|
+
for (const element of record.elements) {
|
|
2669
|
+
addElement(element, requestUrl);
|
|
2670
|
+
}
|
|
2671
|
+
continue;
|
|
2672
|
+
}
|
|
2673
|
+
if ("element" in record) {
|
|
2674
|
+
addElement(record.element, requestUrl);
|
|
2675
|
+
continue;
|
|
2676
|
+
}
|
|
2677
|
+
if ("account" in record) {
|
|
2678
|
+
addElement(record.account, requestUrl);
|
|
2679
|
+
continue;
|
|
2680
|
+
}
|
|
2681
|
+
if (record.type === "metadata") {
|
|
2682
|
+
continue;
|
|
2683
|
+
}
|
|
2684
|
+
addElement(record, requestUrl);
|
|
2685
|
+
}
|
|
2686
|
+
const totalResults = params.reportedTotal ?? accounts.length;
|
|
2687
|
+
const missingFromReported = Math.max(0, totalResults - accounts.length);
|
|
2688
|
+
const complete = missingFromReported === 0;
|
|
2689
|
+
return {
|
|
2690
|
+
fetchResult: {
|
|
2691
|
+
accounts,
|
|
2692
|
+
filteredAccounts,
|
|
2693
|
+
totalResults,
|
|
2694
|
+
fetchedPages: requestUrls.size,
|
|
2695
|
+
sourceQueryUrl: params.sourceQueryUrl,
|
|
2696
|
+
pacing: {
|
|
2697
|
+
startOffset: 0,
|
|
2698
|
+
delayMinMs: 0,
|
|
2699
|
+
delayMaxMs: 0,
|
|
2700
|
+
totalDelayMs: 0,
|
|
2701
|
+
retryCount: 0,
|
|
2702
|
+
stop: {
|
|
2703
|
+
reason: complete
|
|
2704
|
+
? "reported_total_reached"
|
|
2705
|
+
: "raw_results_incomplete",
|
|
2706
|
+
startOffset: 0,
|
|
2707
|
+
pageSize: accounts.length,
|
|
2708
|
+
lastSuccessfulStart: accounts.length > 0 ? 0 : null,
|
|
2709
|
+
lastSuccessfulCount: accounts.length,
|
|
2710
|
+
lastSuccessfulAdded: accounts.length,
|
|
2711
|
+
stoppedAtStart: accounts.length,
|
|
2712
|
+
stoppedAtCount: 0,
|
|
2713
|
+
nextStart: accounts.length,
|
|
2714
|
+
fetchedAccounts: accounts.length,
|
|
2715
|
+
reportedTotal: totalResults,
|
|
2716
|
+
missingFromReported,
|
|
2717
|
+
},
|
|
2718
|
+
},
|
|
2719
|
+
},
|
|
2720
|
+
stats: {
|
|
2721
|
+
filePath,
|
|
2722
|
+
parsedLines,
|
|
2723
|
+
parsedElements,
|
|
2724
|
+
invalidElements,
|
|
2725
|
+
duplicateAccounts,
|
|
2726
|
+
distinctRequestUrls: requestUrls.size,
|
|
2727
|
+
},
|
|
2728
|
+
};
|
|
2729
|
+
}
|
|
2730
|
+
function decodeSalesNavigatorQueryTextValue(value) {
|
|
2731
|
+
let decoded = value.trim();
|
|
2732
|
+
for (let index = 0; index < 3; index += 1) {
|
|
2733
|
+
try {
|
|
2734
|
+
const next = decodeURIComponent(decoded);
|
|
2735
|
+
if (next === decoded) {
|
|
2736
|
+
break;
|
|
2737
|
+
}
|
|
2738
|
+
decoded = next;
|
|
2739
|
+
}
|
|
2740
|
+
catch {
|
|
2741
|
+
break;
|
|
2742
|
+
}
|
|
2743
|
+
}
|
|
2744
|
+
return decoded.replaceAll("+", " ").replace(/\s+/g, " ").trim();
|
|
2745
|
+
}
|
|
2746
|
+
function extractIncludedSalesNavigatorFilterTexts(searchUrl, filterType) {
|
|
2747
|
+
const decodedQuery = decodeSalesNavigatorQueryParam(searchUrl);
|
|
2748
|
+
if (!decodedQuery) {
|
|
2749
|
+
return [];
|
|
2750
|
+
}
|
|
2751
|
+
const values = [];
|
|
2752
|
+
const seen = new Set();
|
|
2753
|
+
const filterPattern = new RegExp(`type:${filterType},values:List\\((.*?)\\)\\)`, "g");
|
|
2754
|
+
for (const filterMatch of decodedQuery.matchAll(filterPattern)) {
|
|
2755
|
+
const block = filterMatch[1] ?? "";
|
|
2756
|
+
for (const valueMatch of block.matchAll(/text:([^,)]+),selectionType:INCLUDED/g)) {
|
|
2757
|
+
const text = decodeSalesNavigatorQueryTextValue(valueMatch[1] ?? "");
|
|
2758
|
+
const key = normalizeLooseMatchText(text);
|
|
2759
|
+
if (text && !seen.has(key)) {
|
|
2760
|
+
seen.add(key);
|
|
2761
|
+
values.push(text);
|
|
2762
|
+
}
|
|
2763
|
+
}
|
|
2764
|
+
}
|
|
2765
|
+
return values;
|
|
2766
|
+
}
|
|
2767
|
+
function buildSalesNavigatorAccountLocationFilter(searchUrl) {
|
|
2768
|
+
const regionTexts = extractIncludedSalesNavigatorFilterTexts(searchUrl, "REGION");
|
|
2769
|
+
const requiredTerms = regionTexts
|
|
2770
|
+
.flatMap((text) => normalizeLooseMatchText(text).split(" "))
|
|
2771
|
+
.filter((term) => term.length >= 3 &&
|
|
2772
|
+
![
|
|
2773
|
+
"area",
|
|
2774
|
+
"metro",
|
|
2775
|
+
"metropolitan",
|
|
2776
|
+
"region",
|
|
2777
|
+
"greater",
|
|
2778
|
+
"united",
|
|
2779
|
+
"states",
|
|
2780
|
+
"kingdom",
|
|
2781
|
+
"germany",
|
|
2782
|
+
"deutschland",
|
|
2783
|
+
"europe",
|
|
2784
|
+
].includes(term));
|
|
2785
|
+
if (regionTexts.length === 0 || requiredTerms.length === 0) {
|
|
2786
|
+
return null;
|
|
2787
|
+
}
|
|
2788
|
+
return {
|
|
2789
|
+
regionTexts,
|
|
2790
|
+
requiredTerms: [...new Set(requiredTerms)],
|
|
2791
|
+
};
|
|
2792
|
+
}
|
|
2793
|
+
function salesNavigatorAccountMatchesLocationFilter(account, filter) {
|
|
2794
|
+
if (!filter || filter.requiredTerms.length === 0) {
|
|
2795
|
+
return true;
|
|
2796
|
+
}
|
|
2797
|
+
const visibleLocation = typeof account.location === "string"
|
|
2798
|
+
? normalizeLooseMatchText(account.location)
|
|
2799
|
+
: "";
|
|
2800
|
+
if (!visibleLocation) {
|
|
2801
|
+
return true;
|
|
2802
|
+
}
|
|
2803
|
+
return filter.requiredTerms.some((term) => visibleLocation.includes(term));
|
|
2804
|
+
}
|
|
2201
2805
|
function buildLinkedInCompanyLookupVariants(params) {
|
|
2202
2806
|
const variants = [];
|
|
2203
2807
|
const seen = new Set();
|
|
@@ -5300,6 +5904,19 @@ function isMissingSupabaseTableError(code, message) {
|
|
|
5300
5904
|
const normalized = message.toLowerCase();
|
|
5301
5905
|
return code === "42P01" || normalized.includes("does not exist");
|
|
5302
5906
|
}
|
|
5907
|
+
function createSalesNavigatorSupabaseFetch(env = process.env) {
|
|
5908
|
+
const configuredTimeoutMs = Number(env.SALESPROMPTER_SALESNAV_SUPABASE_TIMEOUT_MS);
|
|
5909
|
+
const timeoutMs = Number.isFinite(configuredTimeoutMs) && configuredTimeoutMs >= 500
|
|
5910
|
+
? Math.min(30_000, Math.trunc(configuredTimeoutMs))
|
|
5911
|
+
: 2_500;
|
|
5912
|
+
return (input, init) => {
|
|
5913
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
5914
|
+
const signal = init?.signal
|
|
5915
|
+
? AbortSignal.any([init.signal, timeoutSignal])
|
|
5916
|
+
: timeoutSignal;
|
|
5917
|
+
return fetch(input, { ...init, signal });
|
|
5918
|
+
};
|
|
5919
|
+
}
|
|
5303
5920
|
async function createSalesNavigatorCrawlEventStore(options) {
|
|
5304
5921
|
const config = (() => {
|
|
5305
5922
|
try {
|
|
@@ -5317,7 +5934,10 @@ async function createSalesNavigatorCrawlEventStore(options) {
|
|
|
5317
5934
|
auth: {
|
|
5318
5935
|
persistSession: false,
|
|
5319
5936
|
autoRefreshToken: false
|
|
5320
|
-
}
|
|
5937
|
+
},
|
|
5938
|
+
global: {
|
|
5939
|
+
fetch: createSalesNavigatorSupabaseFetch(process.env),
|
|
5940
|
+
},
|
|
5321
5941
|
});
|
|
5322
5942
|
let writable = true;
|
|
5323
5943
|
return {
|
|
@@ -5341,6 +5961,7 @@ async function createSalesNavigatorCrawlEventStore(options) {
|
|
|
5341
5961
|
writable = false;
|
|
5342
5962
|
return;
|
|
5343
5963
|
}
|
|
5964
|
+
writable = false;
|
|
5344
5965
|
throw new Error(`Failed to write crawl event to Supabase: ${error.message}`);
|
|
5345
5966
|
},
|
|
5346
5967
|
listRecentForJob: async (jobId, limit) => {
|
|
@@ -5913,6 +6534,90 @@ async function fetchCliJson(session, request, schema) {
|
|
|
5913
6534
|
return schema.parse(parsed);
|
|
5914
6535
|
});
|
|
5915
6536
|
}
|
|
6537
|
+
async function launchAffiliateCampaignViaApp(session, payload) {
|
|
6538
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-campaigns`, {
|
|
6539
|
+
method: "POST",
|
|
6540
|
+
headers: {
|
|
6541
|
+
"Content-Type": "application/json",
|
|
6542
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6543
|
+
},
|
|
6544
|
+
body: JSON.stringify(payload)
|
|
6545
|
+
}), AffiliateCampaignLaunchResponseSchema);
|
|
6546
|
+
return value;
|
|
6547
|
+
}
|
|
6548
|
+
async function prepareAffiliateOutreachViaApp(session, payload) {
|
|
6549
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach`, {
|
|
6550
|
+
method: "POST",
|
|
6551
|
+
headers: {
|
|
6552
|
+
"Content-Type": "application/json",
|
|
6553
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6554
|
+
},
|
|
6555
|
+
body: JSON.stringify(payload)
|
|
6556
|
+
}), AffiliateOutreachResponseSchema);
|
|
6557
|
+
return value.outreach;
|
|
6558
|
+
}
|
|
6559
|
+
async function getAffiliateOutreachViaApp(session, audienceRunId) {
|
|
6560
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach/${encodeURIComponent(audienceRunId)}`, {
|
|
6561
|
+
headers: {
|
|
6562
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6563
|
+
}
|
|
6564
|
+
}), AffiliateOutreachResponseSchema);
|
|
6565
|
+
return value.outreach;
|
|
6566
|
+
}
|
|
6567
|
+
async function activateAffiliateOutreachViaApp(session, audienceRunId) {
|
|
6568
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach/${encodeURIComponent(audienceRunId)}/activate`, {
|
|
6569
|
+
method: "POST",
|
|
6570
|
+
headers: {
|
|
6571
|
+
"Content-Type": "application/json",
|
|
6572
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6573
|
+
},
|
|
6574
|
+
body: JSON.stringify({})
|
|
6575
|
+
}), AffiliateOutreachResponseSchema);
|
|
6576
|
+
return value.outreach;
|
|
6577
|
+
}
|
|
6578
|
+
async function regenerateAffiliateOutreachSequenceViaApp(session, audienceRunId) {
|
|
6579
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach/${encodeURIComponent(audienceRunId)}/sequence`, {
|
|
6580
|
+
method: "POST",
|
|
6581
|
+
headers: {
|
|
6582
|
+
"Content-Type": "application/json",
|
|
6583
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6584
|
+
},
|
|
6585
|
+
body: JSON.stringify({})
|
|
6586
|
+
}), AffiliateOutreachResponseSchema);
|
|
6587
|
+
return value.outreach;
|
|
6588
|
+
}
|
|
6589
|
+
async function enrichAffiliateOutreachViaApp(session, audienceRunId) {
|
|
6590
|
+
const before = await getAffiliateOutreachViaApp(session, audienceRunId);
|
|
6591
|
+
const previousAttemptedAt = typeof before.stats.latestTopUp === "object" && before.stats.latestTopUp !== null
|
|
6592
|
+
? String(before.stats.latestTopUp.attemptedAt ?? "")
|
|
6593
|
+
: "";
|
|
6594
|
+
await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach/${encodeURIComponent(audienceRunId)}/enrich`, {
|
|
6595
|
+
method: "POST",
|
|
6596
|
+
headers: {
|
|
6597
|
+
"Content-Type": "application/json",
|
|
6598
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6599
|
+
},
|
|
6600
|
+
body: JSON.stringify({})
|
|
6601
|
+
}), AffiliateOutreachEnrichStartResponseSchema);
|
|
6602
|
+
const deadline = Date.now() + 10 * 60 * 1000;
|
|
6603
|
+
const configuredPollMs = Number(process.env.SALESPROMPTER_AFFILIATE_ENRICH_POLL_MS);
|
|
6604
|
+
const pollMs = Number.isFinite(configuredPollMs) && configuredPollMs > 0
|
|
6605
|
+
? Math.max(10, Math.trunc(configuredPollMs))
|
|
6606
|
+
: 5_000;
|
|
6607
|
+
while (Date.now() < deadline) {
|
|
6608
|
+
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
|
6609
|
+
const current = await getAffiliateOutreachViaApp(session, audienceRunId);
|
|
6610
|
+
const latestTopUp = typeof current.stats.latestTopUp === "object" &&
|
|
6611
|
+
current.stats.latestTopUp !== null
|
|
6612
|
+
? current.stats.latestTopUp
|
|
6613
|
+
: null;
|
|
6614
|
+
const attemptedAt = String(latestTopUp?.attemptedAt ?? "");
|
|
6615
|
+
if (attemptedAt && attemptedAt !== previousAttemptedAt) {
|
|
6616
|
+
return current;
|
|
6617
|
+
}
|
|
6618
|
+
}
|
|
6619
|
+
throw new Error("Email recovery is still running after 10 minutes. Check affiliate:status and retry later.");
|
|
6620
|
+
}
|
|
5916
6621
|
async function uploadLinkedInProductsCatalog(session, payload, batchSize = 100, traceId) {
|
|
5917
6622
|
let imported = 0;
|
|
5918
6623
|
let upserted = 0;
|
|
@@ -6068,12 +6773,15 @@ function normalizeSalesNavigatorApiQueryValue(rawQuery) {
|
|
|
6068
6773
|
: withRecentSearchId.replace(/^\(/, "(spellCorrectionEnabled:true,");
|
|
6069
6774
|
}
|
|
6070
6775
|
function buildSalesNavigatorApiRequestFromSearchUrl(searchUrl, config, count) {
|
|
6776
|
+
return {
|
|
6777
|
+
url: buildSalesNavigatorLeadApiUrlFromSearchUrl(searchUrl, count),
|
|
6778
|
+
headers: buildLinkedInSalesNavigatorLocalImportHeaders(config),
|
|
6779
|
+
};
|
|
6780
|
+
}
|
|
6781
|
+
function buildSalesNavigatorLeadApiUrlFromSearchUrl(searchUrl, count) {
|
|
6071
6782
|
const source = new URL(searchUrl);
|
|
6072
6783
|
if (source.pathname.includes("/sales-api/salesApiLeadSearch")) {
|
|
6073
|
-
return
|
|
6074
|
-
url: source.toString(),
|
|
6075
|
-
headers: buildLinkedInSalesNavigatorLocalImportHeaders(config),
|
|
6076
|
-
};
|
|
6784
|
+
return source.toString();
|
|
6077
6785
|
}
|
|
6078
6786
|
if (!source.pathname.includes("/sales/search/people")) {
|
|
6079
6787
|
throw new Error("Automatic Sales Navigator import requires a /sales/search/people URL.");
|
|
@@ -6092,10 +6800,7 @@ function buildSalesNavigatorApiRequestFromSearchUrl(searchUrl, config, count) {
|
|
|
6092
6800
|
`?q=searchQuery&query=${apiQuery}&start=0&count=${normalizedCount}` +
|
|
6093
6801
|
`&trackingParam=(sessionId:${rawSessionId})` +
|
|
6094
6802
|
"&decorationId=com.linkedin.sales.deco.desktop.searchv2.LeadSearchResult-14";
|
|
6095
|
-
return
|
|
6096
|
-
url: apiUrl,
|
|
6097
|
-
headers: buildLinkedInSalesNavigatorLocalImportHeaders(config),
|
|
6098
|
-
};
|
|
6803
|
+
return apiUrl;
|
|
6099
6804
|
}
|
|
6100
6805
|
function extractSalesNavigatorDecorationId(requestUrl, fallbackDecorationId) {
|
|
6101
6806
|
try {
|
|
@@ -6122,15 +6827,12 @@ function buildSalesNavigatorAccountApiUrlFromSearchUrl(searchUrl, count, options
|
|
|
6122
6827
|
const baseUrl = compactOptionalText(options.baseUrl) ??
|
|
6123
6828
|
compactOptionalText(process.env.SALESPROMPTER_LINKEDIN_SALES_API_BASE_URL) ??
|
|
6124
6829
|
`${source.protocol}//${source.host}`;
|
|
6125
|
-
const
|
|
6126
|
-
encodeURIComponent(generateLinkedInSessionId());
|
|
6127
|
-
const apiQuery = normalizeSalesNavigatorApiQueryValue(rawQuery);
|
|
6830
|
+
const apiQuery = decodeRawSalesNavigatorQueryParam(rawQuery);
|
|
6128
6831
|
const normalizedCount = Math.max(1, Math.min(100, Math.trunc(count)));
|
|
6129
6832
|
const decorationId = compactOptionalText(options.decorationId) ??
|
|
6130
|
-
"com.linkedin.sales.deco.desktop.searchv2.AccountSearchResult-
|
|
6833
|
+
"com.linkedin.sales.deco.desktop.searchv2.AccountSearchResult-4";
|
|
6131
6834
|
return (`${baseUrl.replace(/\/+$/, "")}/sales-api/salesApiAccountSearch` +
|
|
6132
6835
|
`?q=searchQuery&query=${apiQuery}&start=0&count=${normalizedCount}` +
|
|
6133
|
-
`&trackingParam=(sessionId:${rawSessionId})` +
|
|
6134
6836
|
`&decorationId=${decorationId}`);
|
|
6135
6837
|
}
|
|
6136
6838
|
function buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, count) {
|
|
@@ -6146,17 +6848,127 @@ function buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, co
|
|
|
6146
6848
|
headers: buildLinkedInSalesNavigatorAccountSearchHeaders(config),
|
|
6147
6849
|
};
|
|
6148
6850
|
}
|
|
6149
|
-
function
|
|
6150
|
-
const
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6155
|
-
|
|
6156
|
-
|
|
6157
|
-
|
|
6158
|
-
|
|
6159
|
-
|
|
6851
|
+
async function probeLinkedInAccountSearchConfig(searchUrl, config) {
|
|
6852
|
+
const request = buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, 1);
|
|
6853
|
+
const controller = new AbortController();
|
|
6854
|
+
const timeout = setTimeout(() => controller.abort(), 8_000);
|
|
6855
|
+
try {
|
|
6856
|
+
const response = await fetch(request.url, {
|
|
6857
|
+
headers: request.headers,
|
|
6858
|
+
signal: controller.signal,
|
|
6859
|
+
});
|
|
6860
|
+
if (response.status === 401) {
|
|
6861
|
+
return {
|
|
6862
|
+
status: "unauthorized",
|
|
6863
|
+
statusCode: response.status,
|
|
6864
|
+
validationError: "salesnav_account_search_http_401",
|
|
6865
|
+
};
|
|
6866
|
+
}
|
|
6867
|
+
if (response.status === 403) {
|
|
6868
|
+
const body = await response.text();
|
|
6869
|
+
let errorCode = "";
|
|
6870
|
+
try {
|
|
6871
|
+
const parsed = JSON.parse(body);
|
|
6872
|
+
errorCode =
|
|
6873
|
+
typeof parsed.code === "string" ? parsed.code.trim() : "";
|
|
6874
|
+
}
|
|
6875
|
+
catch {
|
|
6876
|
+
errorCode = "";
|
|
6877
|
+
}
|
|
6878
|
+
if (errorCode === "SALES_SEAT_REQUIRED") {
|
|
6879
|
+
return {
|
|
6880
|
+
status: "sales_seat_required",
|
|
6881
|
+
statusCode: response.status,
|
|
6882
|
+
validationError: "salesnav_account_search_sales_seat_required",
|
|
6883
|
+
};
|
|
6884
|
+
}
|
|
6885
|
+
return {
|
|
6886
|
+
status: "forbidden",
|
|
6887
|
+
statusCode: response.status,
|
|
6888
|
+
validationError: "salesnav_account_search_http_403",
|
|
6889
|
+
};
|
|
6890
|
+
}
|
|
6891
|
+
if (response.status === 429 || response.status === 999) {
|
|
6892
|
+
return {
|
|
6893
|
+
status: "rate_limited",
|
|
6894
|
+
statusCode: response.status,
|
|
6895
|
+
validationError: "salesnav_account_search_rate_limited",
|
|
6896
|
+
};
|
|
6897
|
+
}
|
|
6898
|
+
if (!response.ok) {
|
|
6899
|
+
return {
|
|
6900
|
+
status: "error",
|
|
6901
|
+
statusCode: response.status,
|
|
6902
|
+
validationError: `salesnav_account_search_http_${response.status}`,
|
|
6903
|
+
};
|
|
6904
|
+
}
|
|
6905
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
6906
|
+
const body = await response.text();
|
|
6907
|
+
if (!/json/i.test(contentType) && /^\s*</.test(body)) {
|
|
6908
|
+
return {
|
|
6909
|
+
status: "invalid_response",
|
|
6910
|
+
statusCode: response.status,
|
|
6911
|
+
validationError: "salesnav_account_search_returned_html",
|
|
6912
|
+
};
|
|
6913
|
+
}
|
|
6914
|
+
try {
|
|
6915
|
+
const parsed = JSON.parse(body);
|
|
6916
|
+
if (typeof parsed !== "object" || parsed === null) {
|
|
6917
|
+
throw new Error("response is not an object");
|
|
6918
|
+
}
|
|
6919
|
+
}
|
|
6920
|
+
catch {
|
|
6921
|
+
return {
|
|
6922
|
+
status: "invalid_response",
|
|
6923
|
+
statusCode: response.status,
|
|
6924
|
+
validationError: "salesnav_account_search_invalid_json",
|
|
6925
|
+
};
|
|
6926
|
+
}
|
|
6927
|
+
return {
|
|
6928
|
+
status: "ok",
|
|
6929
|
+
statusCode: response.status,
|
|
6930
|
+
validationError: null,
|
|
6931
|
+
};
|
|
6932
|
+
}
|
|
6933
|
+
catch (error) {
|
|
6934
|
+
return {
|
|
6935
|
+
status: "error",
|
|
6936
|
+
statusCode: null,
|
|
6937
|
+
validationError: error instanceof Error && error.name === "AbortError"
|
|
6938
|
+
? "salesnav_account_search_timeout"
|
|
6939
|
+
: `salesnav_account_search_request_failed:${error instanceof Error ? error.message : String(error)}`,
|
|
6940
|
+
};
|
|
6941
|
+
}
|
|
6942
|
+
finally {
|
|
6943
|
+
clearTimeout(timeout);
|
|
6944
|
+
}
|
|
6945
|
+
}
|
|
6946
|
+
function assertLinkedInAccountSearchConfigProbe(probe) {
|
|
6947
|
+
if (probe.status === "ok") {
|
|
6948
|
+
return;
|
|
6949
|
+
}
|
|
6950
|
+
if (probe.status === "rate_limited") {
|
|
6951
|
+
throw new Error("Sales Navigator Account Search API is rate-limited. The CLI stopped without retrying the session.");
|
|
6952
|
+
}
|
|
6953
|
+
if (probe.status === "unauthorized" || probe.status === "forbidden") {
|
|
6954
|
+
throw new Error(`The configured LinkedIn session cannot call the Sales Navigator Account Search API (HTTP ${probe.statusCode}). Refresh the Salesprompter LinkedIn session before retrying.`);
|
|
6955
|
+
}
|
|
6956
|
+
if (probe.status === "sales_seat_required") {
|
|
6957
|
+
throw new Error("The configured LinkedIn session is logged in but does not have a Sales Navigator seat.");
|
|
6958
|
+
}
|
|
6959
|
+
throw new Error(`Sales Navigator Account Search API preflight failed: ${probe.validationError ?? probe.status}`);
|
|
6960
|
+
}
|
|
6961
|
+
function buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(searchUrl, templateRequest, count) {
|
|
6962
|
+
const templateUrl = new URL(templateRequest.url);
|
|
6963
|
+
return {
|
|
6964
|
+
url: buildSalesNavigatorAccountApiUrlFromSearchUrl(searchUrl, count, {
|
|
6965
|
+
baseUrl: `${templateUrl.protocol}//${templateUrl.host}`,
|
|
6966
|
+
decorationId: extractSalesNavigatorDecorationId(templateRequest.url, "com.linkedin.sales.deco.desktop.searchv2.AccountSearchResult-4"),
|
|
6967
|
+
}),
|
|
6968
|
+
headers: { ...templateRequest.headers },
|
|
6969
|
+
};
|
|
6970
|
+
}
|
|
6971
|
+
const SALES_NAVIGATOR_PROFILE_LANGUAGE_VALUES = new Map([
|
|
6160
6972
|
["de", { id: "de", text: "German", selectionType: "INCLUDED" }],
|
|
6161
6973
|
["german", { id: "de", text: "German", selectionType: "INCLUDED" }],
|
|
6162
6974
|
["en", { id: "en", text: "English", selectionType: "INCLUDED" }],
|
|
@@ -6218,12 +7030,17 @@ const SALES_NAVIGATOR_COMPANY_HEADCOUNT_VALUES = new Map([
|
|
|
6218
7030
|
["11-50", { id: "C", text: "11-50", selectionType: "INCLUDED" }],
|
|
6219
7031
|
["51-200", { id: "D", text: "51-200", selectionType: "INCLUDED" }],
|
|
6220
7032
|
["201-500", { id: "E", text: "201-500", selectionType: "INCLUDED" }],
|
|
6221
|
-
["501-1000", { id: "F", text: "501-
|
|
6222
|
-
["
|
|
6223
|
-
["
|
|
6224
|
-
["
|
|
6225
|
-
["10,000
|
|
6226
|
-
["10000
|
|
7033
|
+
["501-1000", { id: "F", text: "501-1,000", selectionType: "INCLUDED" }],
|
|
7034
|
+
["501-1,000", { id: "F", text: "501-1,000", selectionType: "INCLUDED" }],
|
|
7035
|
+
["1001-5000", { id: "G", text: "1,001-5,000", selectionType: "INCLUDED" }],
|
|
7036
|
+
["1,001-5,000", { id: "G", text: "1,001-5,000", selectionType: "INCLUDED" }],
|
|
7037
|
+
["5001-10,000", { id: "H", text: "5,001-10,000", selectionType: "INCLUDED" }],
|
|
7038
|
+
["5001-10000", { id: "H", text: "5,001-10,000", selectionType: "INCLUDED" }],
|
|
7039
|
+
["5,001-10,000", { id: "H", text: "5,001-10,000", selectionType: "INCLUDED" }],
|
|
7040
|
+
["10,000+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
7041
|
+
["10000+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
7042
|
+
["10,001+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
7043
|
+
["10001+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
6227
7044
|
]);
|
|
6228
7045
|
const SALES_NAVIGATOR_TENURE_VALUES = new Map([
|
|
6229
7046
|
[
|
|
@@ -6447,14 +7264,17 @@ function withSalesNavigatorPaging(url, start, count) {
|
|
|
6447
7264
|
return `${base}?${params.join("&")}${hash}`;
|
|
6448
7265
|
}
|
|
6449
7266
|
function buildLinkedInSalesNavigatorLocalImportHeaders(config) {
|
|
6450
|
-
|
|
7267
|
+
const headers = {
|
|
6451
7268
|
accept: "*/*",
|
|
6452
7269
|
"csrf-token": config.csrfToken,
|
|
6453
7270
|
cookie: config.cookie,
|
|
6454
7271
|
"user-agent": config.userAgent,
|
|
6455
|
-
"x-li-identity": config.identity,
|
|
6456
7272
|
"x-restli-protocol-version": "2.0.0",
|
|
6457
7273
|
};
|
|
7274
|
+
if (config.identity) {
|
|
7275
|
+
headers["x-li-identity"] = config.identity;
|
|
7276
|
+
}
|
|
7277
|
+
return headers;
|
|
6458
7278
|
}
|
|
6459
7279
|
function buildLinkedInSalesNavigatorAccountSearchHeaders(config) {
|
|
6460
7280
|
return {
|
|
@@ -6745,10 +7565,18 @@ function extractLocalSalesNavigatorTotalResults(responseBody) {
|
|
|
6745
7565
|
return null;
|
|
6746
7566
|
}
|
|
6747
7567
|
const record = responseBody;
|
|
7568
|
+
const data = typeof record.data === "object" &&
|
|
7569
|
+
record.data !== null &&
|
|
7570
|
+
!Array.isArray(record.data)
|
|
7571
|
+
? record.data
|
|
7572
|
+
: null;
|
|
6748
7573
|
const candidates = [
|
|
6749
7574
|
record.paging,
|
|
7575
|
+
data?.paging,
|
|
6750
7576
|
record.metadata,
|
|
7577
|
+
data?.metadata,
|
|
6751
7578
|
record.searchResultsMetadata,
|
|
7579
|
+
data,
|
|
6752
7580
|
record,
|
|
6753
7581
|
];
|
|
6754
7582
|
for (const candidate of candidates) {
|
|
@@ -6850,6 +7678,781 @@ async function fetchLocalSalesNavigatorRequest(parsedRequest, retryOptions) {
|
|
|
6850
7678
|
}
|
|
6851
7679
|
}
|
|
6852
7680
|
}
|
|
7681
|
+
async function readLocalAccountSearchRelayBody(request) {
|
|
7682
|
+
const chunks = [];
|
|
7683
|
+
let size = 0;
|
|
7684
|
+
for await (const chunk of request) {
|
|
7685
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
7686
|
+
size += buffer.length;
|
|
7687
|
+
if (size > 50 * 1024 * 1024) {
|
|
7688
|
+
throw new Error("Browser relay response exceeded 50 MB.");
|
|
7689
|
+
}
|
|
7690
|
+
chunks.push(buffer);
|
|
7691
|
+
}
|
|
7692
|
+
const text = Buffer.concat(chunks).toString("utf8");
|
|
7693
|
+
return text ? JSON.parse(text) : null;
|
|
7694
|
+
}
|
|
7695
|
+
function writeLocalAccountSearchRelayJson(response, statusCode, payload) {
|
|
7696
|
+
response.writeHead(statusCode, {
|
|
7697
|
+
"content-type": "application/json; charset=utf-8",
|
|
7698
|
+
"cache-control": "no-store",
|
|
7699
|
+
"access-control-allow-origin": "https://www.linkedin.com",
|
|
7700
|
+
"access-control-allow-methods": "GET, POST, OPTIONS",
|
|
7701
|
+
"access-control-allow-headers": "content-type",
|
|
7702
|
+
"access-control-allow-private-network": "true",
|
|
7703
|
+
});
|
|
7704
|
+
response.end(`${JSON.stringify(payload)}\n`);
|
|
7705
|
+
}
|
|
7706
|
+
async function createLocalAccountSearchBrowserRelay(requestedPort) {
|
|
7707
|
+
let pending = null;
|
|
7708
|
+
let requestSequence = 0;
|
|
7709
|
+
const server = createServer(async (request, response) => {
|
|
7710
|
+
try {
|
|
7711
|
+
const requestUrl = new URL(request.url ?? "/", `http://127.0.0.1:${requestedPort}`);
|
|
7712
|
+
if (request.method === "OPTIONS") {
|
|
7713
|
+
writeLocalAccountSearchRelayJson(response, 204, null);
|
|
7714
|
+
return;
|
|
7715
|
+
}
|
|
7716
|
+
if (request.method === "GET" && requestUrl.pathname === "/health") {
|
|
7717
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7718
|
+
status: "ok",
|
|
7719
|
+
pending: Boolean(pending),
|
|
7720
|
+
});
|
|
7721
|
+
return;
|
|
7722
|
+
}
|
|
7723
|
+
if (request.method === "GET" && requestUrl.pathname === "/next") {
|
|
7724
|
+
writeLocalAccountSearchRelayJson(response, 200, pending
|
|
7725
|
+
? {
|
|
7726
|
+
status: "task",
|
|
7727
|
+
requestId: pending.requestId,
|
|
7728
|
+
url: pending.url,
|
|
7729
|
+
}
|
|
7730
|
+
: { status: "idle" });
|
|
7731
|
+
return;
|
|
7732
|
+
}
|
|
7733
|
+
if (request.method === "POST" && requestUrl.pathname === "/result") {
|
|
7734
|
+
if (!pending) {
|
|
7735
|
+
writeLocalAccountSearchRelayJson(response, 409, {
|
|
7736
|
+
status: "error",
|
|
7737
|
+
error: "No Account Search relay task is pending.",
|
|
7738
|
+
});
|
|
7739
|
+
return;
|
|
7740
|
+
}
|
|
7741
|
+
const value = await readLocalAccountSearchRelayBody(request);
|
|
7742
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
7743
|
+
throw new Error("Browser relay result must be a JSON object.");
|
|
7744
|
+
}
|
|
7745
|
+
const record = value;
|
|
7746
|
+
if (record.requestId !== pending.requestId) {
|
|
7747
|
+
writeLocalAccountSearchRelayJson(response, 409, {
|
|
7748
|
+
status: "error",
|
|
7749
|
+
error: "Browser relay request id does not match the pending task.",
|
|
7750
|
+
});
|
|
7751
|
+
return;
|
|
7752
|
+
}
|
|
7753
|
+
const status = Number(record.status);
|
|
7754
|
+
const retryAfter = typeof record.retryAfter === "string" ? record.retryAfter : null;
|
|
7755
|
+
const body = record.body;
|
|
7756
|
+
const bodyPreview = JSON.stringify(body ?? null).slice(0, 300);
|
|
7757
|
+
const current = pending;
|
|
7758
|
+
pending = null;
|
|
7759
|
+
if (isSalesNavigatorRateLimitStatus(status, bodyPreview)) {
|
|
7760
|
+
current.reject(new LocalSalesNavigatorRateLimitError(status, bodyPreview, parseRetryAfterHeader(retryAfter)));
|
|
7761
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7762
|
+
status: "rate_limited",
|
|
7763
|
+
});
|
|
7764
|
+
return;
|
|
7765
|
+
}
|
|
7766
|
+
if (!Number.isFinite(status) || status < 200 || status >= 300) {
|
|
7767
|
+
current.reject(new Error(`Sales Navigator browser relay failed with HTTP ${status}: ${bodyPreview}`));
|
|
7768
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7769
|
+
status: "failed",
|
|
7770
|
+
});
|
|
7771
|
+
return;
|
|
7772
|
+
}
|
|
7773
|
+
current.resolve({ body, retryCount: 0, retryDelayMs: 0 });
|
|
7774
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7775
|
+
status: "accepted",
|
|
7776
|
+
});
|
|
7777
|
+
return;
|
|
7778
|
+
}
|
|
7779
|
+
writeLocalAccountSearchRelayJson(response, 404, {
|
|
7780
|
+
status: "error",
|
|
7781
|
+
error: "Not found",
|
|
7782
|
+
});
|
|
7783
|
+
}
|
|
7784
|
+
catch (error) {
|
|
7785
|
+
writeLocalAccountSearchRelayJson(response, 500, {
|
|
7786
|
+
status: "error",
|
|
7787
|
+
error: error instanceof Error ? error.message : String(error),
|
|
7788
|
+
});
|
|
7789
|
+
}
|
|
7790
|
+
});
|
|
7791
|
+
await new Promise((resolve, reject) => {
|
|
7792
|
+
server.once("error", reject);
|
|
7793
|
+
server.listen(requestedPort, "127.0.0.1", () => resolve());
|
|
7794
|
+
});
|
|
7795
|
+
const address = server.address();
|
|
7796
|
+
if (!address || typeof address === "string") {
|
|
7797
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
7798
|
+
throw new Error("Account Search browser relay failed to bind.");
|
|
7799
|
+
}
|
|
7800
|
+
return {
|
|
7801
|
+
port: address.port,
|
|
7802
|
+
request(url) {
|
|
7803
|
+
if (pending) {
|
|
7804
|
+
throw new Error("Account Search browser relay already has a pending task.");
|
|
7805
|
+
}
|
|
7806
|
+
const parsedUrl = new URL(url);
|
|
7807
|
+
if (!/(^|\.)linkedin\.com$/i.test(parsedUrl.hostname) ||
|
|
7808
|
+
!(parsedUrl.pathname.includes("/sales-api/salesApiAccountSearch") ||
|
|
7809
|
+
parsedUrl.pathname.includes("/sales-api/salesApiLeadSearch"))) {
|
|
7810
|
+
throw new Error("Browser relay only accepts LinkedIn Account or Lead Search API URLs.");
|
|
7811
|
+
}
|
|
7812
|
+
requestSequence += 1;
|
|
7813
|
+
return new Promise((resolve, reject) => {
|
|
7814
|
+
pending = {
|
|
7815
|
+
requestId: `sales-nav-search-${requestSequence}`,
|
|
7816
|
+
url,
|
|
7817
|
+
resolve,
|
|
7818
|
+
reject,
|
|
7819
|
+
};
|
|
7820
|
+
});
|
|
7821
|
+
},
|
|
7822
|
+
async close() {
|
|
7823
|
+
if (pending) {
|
|
7824
|
+
pending.reject(new Error("Account Search browser relay closed."));
|
|
7825
|
+
pending = null;
|
|
7826
|
+
}
|
|
7827
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
7828
|
+
},
|
|
7829
|
+
};
|
|
7830
|
+
}
|
|
7831
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_RESULT_WINDOW = 1600;
|
|
7832
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION = 3;
|
|
7833
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS = "coverage-complete";
|
|
7834
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_PASSES = [
|
|
7835
|
+
{
|
|
7836
|
+
name: LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS,
|
|
7837
|
+
filterTypes: [
|
|
7838
|
+
"COMPANY_HEADCOUNT",
|
|
7839
|
+
"NUM_OF_FOLLOWERS",
|
|
7840
|
+
],
|
|
7841
|
+
},
|
|
7842
|
+
{
|
|
7843
|
+
name: "headcount-first",
|
|
7844
|
+
filterTypes: [
|
|
7845
|
+
"COMPANY_HEADCOUNT",
|
|
7846
|
+
"NUM_OF_FOLLOWERS",
|
|
7847
|
+
"ANNUAL_REVENUE",
|
|
7848
|
+
"COMPANY_HEADCOUNT_GROWTH",
|
|
7849
|
+
],
|
|
7850
|
+
},
|
|
7851
|
+
{
|
|
7852
|
+
name: "followers-first",
|
|
7853
|
+
filterTypes: [
|
|
7854
|
+
"NUM_OF_FOLLOWERS",
|
|
7855
|
+
"COMPANY_HEADCOUNT",
|
|
7856
|
+
"ANNUAL_REVENUE",
|
|
7857
|
+
"COMPANY_HEADCOUNT_GROWTH",
|
|
7858
|
+
],
|
|
7859
|
+
},
|
|
7860
|
+
{
|
|
7861
|
+
name: "revenue-first",
|
|
7862
|
+
filterTypes: [
|
|
7863
|
+
"ANNUAL_REVENUE",
|
|
7864
|
+
"COMPANY_HEADCOUNT",
|
|
7865
|
+
"NUM_OF_FOLLOWERS",
|
|
7866
|
+
"COMPANY_HEADCOUNT_GROWTH",
|
|
7867
|
+
],
|
|
7868
|
+
},
|
|
7869
|
+
];
|
|
7870
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_KEYWORD_TOKENS = [
|
|
7871
|
+
"software",
|
|
7872
|
+
"ai",
|
|
7873
|
+
"technology",
|
|
7874
|
+
"data",
|
|
7875
|
+
"platform",
|
|
7876
|
+
"services",
|
|
7877
|
+
"solutions",
|
|
7878
|
+
"digital",
|
|
7879
|
+
"cloud",
|
|
7880
|
+
"business",
|
|
7881
|
+
"internet",
|
|
7882
|
+
"online",
|
|
7883
|
+
"app",
|
|
7884
|
+
"systems",
|
|
7885
|
+
"company",
|
|
7886
|
+
"management",
|
|
7887
|
+
"development",
|
|
7888
|
+
"network",
|
|
7889
|
+
"security",
|
|
7890
|
+
"automation",
|
|
7891
|
+
"marketing",
|
|
7892
|
+
"financial",
|
|
7893
|
+
"media",
|
|
7894
|
+
"consulting",
|
|
7895
|
+
"health",
|
|
7896
|
+
"mobile",
|
|
7897
|
+
"enterprise",
|
|
7898
|
+
"analytics",
|
|
7899
|
+
"engineering",
|
|
7900
|
+
"global",
|
|
7901
|
+
"design",
|
|
7902
|
+
"web",
|
|
7903
|
+
];
|
|
7904
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_GROWTH_DIMENSION = {
|
|
7905
|
+
key: "company-headcount-growth",
|
|
7906
|
+
filterType: "COMPANY_HEADCOUNT_GROWTH",
|
|
7907
|
+
combinableValues: false,
|
|
7908
|
+
values: [
|
|
7909
|
+
{ text: "-1,000,000% to -1%", rangeValue: { min: -1_000_000, max: -1 } },
|
|
7910
|
+
{ text: "0%", rangeValue: { min: 0, max: 0 } },
|
|
7911
|
+
{ text: "1% to 10%", rangeValue: { min: 1, max: 10 } },
|
|
7912
|
+
{ text: "11% to 25%", rangeValue: { min: 11, max: 25 } },
|
|
7913
|
+
{ text: "26% to 50%", rangeValue: { min: 26, max: 50 } },
|
|
7914
|
+
{ text: "51% to 100%", rangeValue: { min: 51, max: 100 } },
|
|
7915
|
+
{ text: "101%+", rangeValue: { min: 101, max: 1_000_000 } },
|
|
7916
|
+
],
|
|
7917
|
+
};
|
|
7918
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_PARENT_REGION = {
|
|
7919
|
+
id: "90000084",
|
|
7920
|
+
text: "San Francisco Bay Area",
|
|
7921
|
+
selectionType: "INCLUDED",
|
|
7922
|
+
};
|
|
7923
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES = [
|
|
7924
|
+
{
|
|
7925
|
+
id: "102839053",
|
|
7926
|
+
text: "Alameda County, California, United States",
|
|
7927
|
+
selectionType: "INCLUDED",
|
|
7928
|
+
},
|
|
7929
|
+
{
|
|
7930
|
+
id: "102476922",
|
|
7931
|
+
text: "Contra Costa County, California, United States",
|
|
7932
|
+
selectionType: "INCLUDED",
|
|
7933
|
+
},
|
|
7934
|
+
{
|
|
7935
|
+
id: "104929029",
|
|
7936
|
+
text: "Marin County, California, United States",
|
|
7937
|
+
selectionType: "INCLUDED",
|
|
7938
|
+
},
|
|
7939
|
+
{
|
|
7940
|
+
id: "105546787",
|
|
7941
|
+
text: "Napa County, California, United States",
|
|
7942
|
+
selectionType: "INCLUDED",
|
|
7943
|
+
},
|
|
7944
|
+
{
|
|
7945
|
+
id: "100901743",
|
|
7946
|
+
text: "San Francisco County, California, United States",
|
|
7947
|
+
selectionType: "INCLUDED",
|
|
7948
|
+
},
|
|
7949
|
+
{
|
|
7950
|
+
id: "104739033",
|
|
7951
|
+
text: "San Mateo County, California, United States",
|
|
7952
|
+
selectionType: "INCLUDED",
|
|
7953
|
+
},
|
|
7954
|
+
{
|
|
7955
|
+
id: "105416315",
|
|
7956
|
+
text: "Santa Clara County, California, United States",
|
|
7957
|
+
selectionType: "INCLUDED",
|
|
7958
|
+
},
|
|
7959
|
+
{
|
|
7960
|
+
id: "103079873",
|
|
7961
|
+
text: "Solano County, California, United States",
|
|
7962
|
+
selectionType: "INCLUDED",
|
|
7963
|
+
},
|
|
7964
|
+
{
|
|
7965
|
+
id: "101079764",
|
|
7966
|
+
text: "Sonoma County, California, United States",
|
|
7967
|
+
selectionType: "INCLUDED",
|
|
7968
|
+
},
|
|
7969
|
+
];
|
|
7970
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_SAN_FRANCISCO_CITY = {
|
|
7971
|
+
id: "102277331",
|
|
7972
|
+
text: "San Francisco, California, United States",
|
|
7973
|
+
selectionType: "INCLUDED",
|
|
7974
|
+
};
|
|
7975
|
+
function isLocalAccountSearchCollectorNodeTask(value) {
|
|
7976
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
7977
|
+
return false;
|
|
7978
|
+
}
|
|
7979
|
+
const task = value;
|
|
7980
|
+
return (task.kind === "node" &&
|
|
7981
|
+
typeof task.id === "string" &&
|
|
7982
|
+
typeof task.pass === "string" &&
|
|
7983
|
+
Array.isArray(task.filters) &&
|
|
7984
|
+
Array.isArray(task.remainingDimensions));
|
|
7985
|
+
}
|
|
7986
|
+
function localAccountSearchCollectorHasFilterType(filters, filterType) {
|
|
7987
|
+
return filters.some((filter) => filter.type === filterType);
|
|
7988
|
+
}
|
|
7989
|
+
function buildLocalAccountSearchCollectorDimensionPartitions(dimension) {
|
|
7990
|
+
const partitions = dimension.values.map((value, index) => ({
|
|
7991
|
+
idSuffix: String(index),
|
|
7992
|
+
filter: {
|
|
7993
|
+
type: dimension.filterType,
|
|
7994
|
+
values: [{ ...value }],
|
|
7995
|
+
},
|
|
7996
|
+
}));
|
|
7997
|
+
const residualValues = dimension.values
|
|
7998
|
+
.filter((value) => typeof value.id === "string" && value.id.length > 0)
|
|
7999
|
+
.map((value) => ({
|
|
8000
|
+
...value,
|
|
8001
|
+
selectionType: "EXCLUDED",
|
|
8002
|
+
}));
|
|
8003
|
+
if (dimension.supportsResidualExclusion !== false &&
|
|
8004
|
+
residualValues.length === dimension.values.length &&
|
|
8005
|
+
residualValues.length > 0) {
|
|
8006
|
+
partitions.push({
|
|
8007
|
+
idSuffix: "residual",
|
|
8008
|
+
filter: {
|
|
8009
|
+
type: dimension.filterType,
|
|
8010
|
+
values: residualValues,
|
|
8011
|
+
},
|
|
8012
|
+
});
|
|
8013
|
+
}
|
|
8014
|
+
return partitions;
|
|
8015
|
+
}
|
|
8016
|
+
function withLocalAccountSearchCollectorKeywords(searchUrl, keywordExpression) {
|
|
8017
|
+
const url = new URL(searchUrl);
|
|
8018
|
+
const query = url.searchParams.get("query");
|
|
8019
|
+
if (!query) {
|
|
8020
|
+
return null;
|
|
8021
|
+
}
|
|
8022
|
+
let nextQuery;
|
|
8023
|
+
if (/^\(keywords:[\s\S]*?,filters:List\(/.test(query)) {
|
|
8024
|
+
nextQuery = query.replace(/^\(keywords:[\s\S]*?,filters:List\(/, `(keywords:${keywordExpression},filters:List(`);
|
|
8025
|
+
}
|
|
8026
|
+
else if (query.startsWith("(filters:List(")) {
|
|
8027
|
+
nextQuery = `(keywords:${keywordExpression},${query.slice(1)}`;
|
|
8028
|
+
}
|
|
8029
|
+
else {
|
|
8030
|
+
return null;
|
|
8031
|
+
}
|
|
8032
|
+
url.searchParams.set("query", nextQuery);
|
|
8033
|
+
return url.toString();
|
|
8034
|
+
}
|
|
8035
|
+
function buildLocalAccountSearchCollectorKeywordPartitions(searchUrl, task) {
|
|
8036
|
+
const keywordDepth = Math.max(0, Math.trunc(task.keywordDepth ?? 0));
|
|
8037
|
+
const token = LOCAL_ACCOUNT_SEARCH_COLLECTOR_KEYWORD_TOKENS[keywordDepth];
|
|
8038
|
+
if (!token) {
|
|
8039
|
+
return null;
|
|
8040
|
+
}
|
|
8041
|
+
const keywordClauses = Array.isArray(task.keywordClauses)
|
|
8042
|
+
? task.keywordClauses.filter((clause) => clause.trim().length > 0)
|
|
8043
|
+
: [];
|
|
8044
|
+
const branches = [token, `NOT ${token}`].map((clause) => {
|
|
8045
|
+
const nextClauses = [...keywordClauses, clause];
|
|
8046
|
+
return {
|
|
8047
|
+
clause,
|
|
8048
|
+
nextClauses,
|
|
8049
|
+
searchUrl: withLocalAccountSearchCollectorKeywords(searchUrl, nextClauses.join(" AND ")),
|
|
8050
|
+
};
|
|
8051
|
+
});
|
|
8052
|
+
if (branches.some((branch) => !branch.searchUrl)) {
|
|
8053
|
+
return null;
|
|
8054
|
+
}
|
|
8055
|
+
return {
|
|
8056
|
+
splitOn: `KEYWORDS_${token}`,
|
|
8057
|
+
partitions: branches.map((branch, index) => ({
|
|
8058
|
+
idSuffix: index === 0 ? token : `not-${token}`,
|
|
8059
|
+
searchUrl: branch.searchUrl,
|
|
8060
|
+
keywordClauses: branch.nextClauses,
|
|
8061
|
+
keywordDepth: keywordDepth + 1,
|
|
8062
|
+
})),
|
|
8063
|
+
};
|
|
8064
|
+
}
|
|
8065
|
+
function buildLocalAccountSearchCollectorIndustryPartitions(searchUrl) {
|
|
8066
|
+
const url = new URL(searchUrl);
|
|
8067
|
+
const query = url.searchParams.get("query");
|
|
8068
|
+
if (!query) {
|
|
8069
|
+
return null;
|
|
8070
|
+
}
|
|
8071
|
+
const parentIndustryPattern = /\(type:INDUSTRY,values:List\(\(id:6,text:Technology%2C%20Information%20and%20Internet,selectionType:INCLUDED\)\)\)/;
|
|
8072
|
+
if (!parentIndustryPattern.test(query)) {
|
|
8073
|
+
return null;
|
|
8074
|
+
}
|
|
8075
|
+
const industryBlocks = [
|
|
8076
|
+
{
|
|
8077
|
+
idSuffix: "software-development",
|
|
8078
|
+
block: "(type:INDUSTRY,values:List((id:4,text:Software%20Development,selectionType:INCLUDED)))",
|
|
8079
|
+
},
|
|
8080
|
+
{
|
|
8081
|
+
idSuffix: "technology-internet-residual",
|
|
8082
|
+
block: "(type:INDUSTRY,values:List((id:6,text:Technology%2C%20Information%20and%20Internet,selectionType:INCLUDED),(id:4,text:Software%20Development,selectionType:EXCLUDED)))",
|
|
8083
|
+
},
|
|
8084
|
+
];
|
|
8085
|
+
return {
|
|
8086
|
+
splitOn: "INDUSTRY_SOFTWARE_DEVELOPMENT",
|
|
8087
|
+
partitions: industryBlocks.map((partition) => {
|
|
8088
|
+
const childUrl = new URL(url);
|
|
8089
|
+
childUrl.searchParams.set("query", query.replace(parentIndustryPattern, partition.block));
|
|
8090
|
+
return {
|
|
8091
|
+
idSuffix: partition.idSuffix,
|
|
8092
|
+
searchUrl: childUrl.toString(),
|
|
8093
|
+
};
|
|
8094
|
+
}),
|
|
8095
|
+
};
|
|
8096
|
+
}
|
|
8097
|
+
function isSanFranciscoBayAreaAccountSearch(sourceQueryUrl) {
|
|
8098
|
+
const query = new URL(sourceQueryUrl).searchParams.get("query") ?? "";
|
|
8099
|
+
return (query.includes("type:REGION") &&
|
|
8100
|
+
query.includes(`id:${LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_PARENT_REGION.id}`));
|
|
8101
|
+
}
|
|
8102
|
+
function buildLocalAccountSearchCollectorBayAreaPartitions(sourceQueryUrl, filters) {
|
|
8103
|
+
if (!isSanFranciscoBayAreaAccountSearch(sourceQueryUrl)) {
|
|
8104
|
+
return null;
|
|
8105
|
+
}
|
|
8106
|
+
const regionFilter = filters.find((filter) => filter.type === "REGION");
|
|
8107
|
+
if (!regionFilter) {
|
|
8108
|
+
const countyPartitions = LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES.map((county) => [
|
|
8109
|
+
...filters,
|
|
8110
|
+
{ type: "REGION", values: [{ ...county }] },
|
|
8111
|
+
]);
|
|
8112
|
+
const residualPartition = [
|
|
8113
|
+
...filters,
|
|
8114
|
+
{
|
|
8115
|
+
type: "REGION",
|
|
8116
|
+
values: [
|
|
8117
|
+
{ ...LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_PARENT_REGION },
|
|
8118
|
+
...LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES.map((county) => ({
|
|
8119
|
+
...county,
|
|
8120
|
+
selectionType: "EXCLUDED",
|
|
8121
|
+
})),
|
|
8122
|
+
],
|
|
8123
|
+
},
|
|
8124
|
+
];
|
|
8125
|
+
return {
|
|
8126
|
+
splitOn: "REGION_BAY_AREA_COUNTY",
|
|
8127
|
+
partitions: [...countyPartitions, residualPartition],
|
|
8128
|
+
};
|
|
8129
|
+
}
|
|
8130
|
+
const includedRegions = regionFilter.values.filter((value) => (value.selectionType ?? "INCLUDED") === "INCLUDED");
|
|
8131
|
+
const excludedRegions = regionFilter.values.filter((value) => value.selectionType === "EXCLUDED");
|
|
8132
|
+
const sanFranciscoCounty = LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES.find((county) => county.id === "100901743");
|
|
8133
|
+
if (sanFranciscoCounty &&
|
|
8134
|
+
includedRegions.length === 1 &&
|
|
8135
|
+
includedRegions[0]?.id === sanFranciscoCounty.id &&
|
|
8136
|
+
excludedRegions.length === 0) {
|
|
8137
|
+
return {
|
|
8138
|
+
splitOn: "REGION_SAN_FRANCISCO_CITY",
|
|
8139
|
+
partitions: [
|
|
8140
|
+
[
|
|
8141
|
+
...filters.filter((filter) => filter.type !== "REGION"),
|
|
8142
|
+
{
|
|
8143
|
+
type: "REGION",
|
|
8144
|
+
values: [{ ...LOCAL_ACCOUNT_SEARCH_COLLECTOR_SAN_FRANCISCO_CITY }],
|
|
8145
|
+
},
|
|
8146
|
+
],
|
|
8147
|
+
[
|
|
8148
|
+
...filters.filter((filter) => filter.type !== "REGION"),
|
|
8149
|
+
{
|
|
8150
|
+
type: "REGION",
|
|
8151
|
+
values: [
|
|
8152
|
+
{ ...sanFranciscoCounty },
|
|
8153
|
+
{
|
|
8154
|
+
...LOCAL_ACCOUNT_SEARCH_COLLECTOR_SAN_FRANCISCO_CITY,
|
|
8155
|
+
selectionType: "EXCLUDED",
|
|
8156
|
+
},
|
|
8157
|
+
],
|
|
8158
|
+
},
|
|
8159
|
+
],
|
|
8160
|
+
],
|
|
8161
|
+
};
|
|
8162
|
+
}
|
|
8163
|
+
return null;
|
|
8164
|
+
}
|
|
8165
|
+
function migrateLocalAccountSearchCollectorCheckpoint(checkpoint) {
|
|
8166
|
+
let changed = false;
|
|
8167
|
+
const growthFilterType = LOCAL_ACCOUNT_SEARCH_COLLECTOR_GROWTH_DIMENSION.filterType;
|
|
8168
|
+
const needsCoverageMigration = checkpoint.coveragePassVersion !==
|
|
8169
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION;
|
|
8170
|
+
if (needsCoverageMigration) {
|
|
8171
|
+
checkpoint.taskQueue = checkpoint.taskQueue.filter((task) => !task.pass.startsWith("coverage-"));
|
|
8172
|
+
checkpoint.seenQueries = checkpoint.seenQueries.filter((queryKey) => !queryKey.startsWith("coverage-"));
|
|
8173
|
+
checkpoint.coveragePassSeeded = false;
|
|
8174
|
+
checkpoint.coveragePassVersion =
|
|
8175
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION;
|
|
8176
|
+
changed = true;
|
|
8177
|
+
}
|
|
8178
|
+
for (const task of checkpoint.taskQueue) {
|
|
8179
|
+
if (task.kind === "node" && task.pass.startsWith("coverage-")) {
|
|
8180
|
+
const remainingDimensions = task.remainingDimensions.filter((filterType) => filterType !== growthFilterType);
|
|
8181
|
+
if (remainingDimensions.length !== task.remainingDimensions.length) {
|
|
8182
|
+
task.remainingDimensions = remainingDimensions;
|
|
8183
|
+
changed = true;
|
|
8184
|
+
}
|
|
8185
|
+
continue;
|
|
8186
|
+
}
|
|
8187
|
+
if (task.kind === "node" &&
|
|
8188
|
+
!localAccountSearchCollectorHasFilterType(task.filters, growthFilterType) &&
|
|
8189
|
+
!task.remainingDimensions.includes(growthFilterType)) {
|
|
8190
|
+
task.remainingDimensions.push(growthFilterType);
|
|
8191
|
+
changed = true;
|
|
8192
|
+
}
|
|
8193
|
+
}
|
|
8194
|
+
const queuedNodeQueries = new Set(checkpoint.taskQueue
|
|
8195
|
+
.filter(isLocalAccountSearchCollectorNodeTask)
|
|
8196
|
+
.map((task) => localAccountSearchCollectorQueryKey(buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, task.filters).slicedQueryUrl)));
|
|
8197
|
+
const remainingFailures = [];
|
|
8198
|
+
for (const failure of checkpoint.failures) {
|
|
8199
|
+
const error = typeof failure.error === "string" ? failure.error : "";
|
|
8200
|
+
const task = failure.task;
|
|
8201
|
+
const isTerminalSlice = error.includes("with no split dimension left");
|
|
8202
|
+
if (!isTerminalSlice || !isLocalAccountSearchCollectorNodeTask(task)) {
|
|
8203
|
+
remainingFailures.push(failure);
|
|
8204
|
+
continue;
|
|
8205
|
+
}
|
|
8206
|
+
const hasGrowthFilter = localAccountSearchCollectorHasFilterType(task.filters, growthFilterType);
|
|
8207
|
+
const canUseBayAreaFallback = Boolean(buildLocalAccountSearchCollectorBayAreaPartitions(checkpoint.sourceQueryUrl, task.filters));
|
|
8208
|
+
if (hasGrowthFilter && !canUseBayAreaFallback) {
|
|
8209
|
+
remainingFailures.push(failure);
|
|
8210
|
+
continue;
|
|
8211
|
+
}
|
|
8212
|
+
const remainingDimensions = [...task.remainingDimensions];
|
|
8213
|
+
if (!hasGrowthFilter && !remainingDimensions.includes(growthFilterType)) {
|
|
8214
|
+
remainingDimensions.push(growthFilterType);
|
|
8215
|
+
}
|
|
8216
|
+
const queryKey = localAccountSearchCollectorQueryKey(buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, task.filters).slicedQueryUrl);
|
|
8217
|
+
if (!queuedNodeQueries.has(queryKey)) {
|
|
8218
|
+
checkpoint.taskQueue.push({
|
|
8219
|
+
...task,
|
|
8220
|
+
filters: task.filters.map((filter) => ({
|
|
8221
|
+
...filter,
|
|
8222
|
+
values: filter.values.map((value) => ({ ...value })),
|
|
8223
|
+
})),
|
|
8224
|
+
remainingDimensions,
|
|
8225
|
+
});
|
|
8226
|
+
queuedNodeQueries.add(queryKey);
|
|
8227
|
+
}
|
|
8228
|
+
changed = true;
|
|
8229
|
+
}
|
|
8230
|
+
if (remainingFailures.length !== checkpoint.failures.length) {
|
|
8231
|
+
checkpoint.failures = remainingFailures;
|
|
8232
|
+
}
|
|
8233
|
+
const isFreshCheckpoint = checkpoint.taskQueue.length === 0 && checkpoint.seenQueries.length === 0;
|
|
8234
|
+
if (!isFreshCheckpoint &&
|
|
8235
|
+
needsCoverageMigration &&
|
|
8236
|
+
(checkpoint.rootTotal ?? 0) > checkpoint.cap) {
|
|
8237
|
+
seedLocalAccountSearchCollectorPassTasks(checkpoint, LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS, true);
|
|
8238
|
+
checkpoint.coveragePassSeeded = true;
|
|
8239
|
+
changed = true;
|
|
8240
|
+
}
|
|
8241
|
+
const coverageTasks = checkpoint.taskQueue.filter((task) => task.pass.startsWith("coverage-"));
|
|
8242
|
+
if (coverageTasks.length > 0 &&
|
|
8243
|
+
checkpoint.taskQueue.some((task, index) => task.pass.startsWith("coverage-") && index >= coverageTasks.length)) {
|
|
8244
|
+
checkpoint.taskQueue = [
|
|
8245
|
+
...coverageTasks,
|
|
8246
|
+
...checkpoint.taskQueue.filter((task) => !task.pass.startsWith("coverage-")),
|
|
8247
|
+
];
|
|
8248
|
+
changed = true;
|
|
8249
|
+
}
|
|
8250
|
+
return changed;
|
|
8251
|
+
}
|
|
8252
|
+
function parseLocalAccountSearchCollectorCheckpoint(value) {
|
|
8253
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
8254
|
+
throw new Error("Account Search checkpoint must be a JSON object.");
|
|
8255
|
+
}
|
|
8256
|
+
const checkpoint = value;
|
|
8257
|
+
if (checkpoint.version !== 1 ||
|
|
8258
|
+
typeof checkpoint.sourceQueryUrl !== "string" ||
|
|
8259
|
+
!Array.isArray(checkpoint.taskQueue) ||
|
|
8260
|
+
!Array.isArray(checkpoint.leaves) ||
|
|
8261
|
+
!Array.isArray(checkpoint.windows) ||
|
|
8262
|
+
!Array.isArray(checkpoint.failures)) {
|
|
8263
|
+
throw new Error("Unsupported or invalid Account Search checkpoint.");
|
|
8264
|
+
}
|
|
8265
|
+
const seenQueryValues = Array.isArray(checkpoint.seenQueries)
|
|
8266
|
+
? checkpoint.seenQueries
|
|
8267
|
+
: checkpoint.seenQueries && typeof checkpoint.seenQueries === "object"
|
|
8268
|
+
? Object.keys(checkpoint.seenQueries)
|
|
8269
|
+
: [];
|
|
8270
|
+
return {
|
|
8271
|
+
...checkpoint,
|
|
8272
|
+
seenQueries: seenQueryValues.map(localAccountSearchCollectorQueryKey),
|
|
8273
|
+
};
|
|
8274
|
+
}
|
|
8275
|
+
function localAccountSearchCollectorQueryKey(searchUrlOrQuery) {
|
|
8276
|
+
try {
|
|
8277
|
+
const query = new URL(searchUrlOrQuery).searchParams.get("query");
|
|
8278
|
+
return query ?? searchUrlOrQuery;
|
|
8279
|
+
}
|
|
8280
|
+
catch {
|
|
8281
|
+
return searchUrlOrQuery;
|
|
8282
|
+
}
|
|
8283
|
+
}
|
|
8284
|
+
function localAccountSearchCollectorSeenKey(pass, searchUrlOrQuery) {
|
|
8285
|
+
const queryKey = localAccountSearchCollectorQueryKey(searchUrlOrQuery);
|
|
8286
|
+
return pass.startsWith("coverage-") ? `${pass}:${queryKey}` : queryKey;
|
|
8287
|
+
}
|
|
8288
|
+
function createLocalAccountSearchCollectorCheckpoint(params) {
|
|
8289
|
+
const now = new Date().toISOString();
|
|
8290
|
+
return {
|
|
8291
|
+
version: 1,
|
|
8292
|
+
sourceQueryUrl: params.sourceQueryUrl,
|
|
8293
|
+
rootTotal: params.rootTotal,
|
|
8294
|
+
cap: params.cap,
|
|
8295
|
+
pageSize: params.pageSize,
|
|
8296
|
+
delayMinMs: params.delayMinMs,
|
|
8297
|
+
delayMaxMs: params.delayMaxMs,
|
|
8298
|
+
requests: 0,
|
|
8299
|
+
pagesWritten: 0,
|
|
8300
|
+
rowsWritten: 0,
|
|
8301
|
+
startedAt: now,
|
|
8302
|
+
updatedAt: now,
|
|
8303
|
+
taskQueue: [],
|
|
8304
|
+
seenQueries: [],
|
|
8305
|
+
duplicateQueries: 0,
|
|
8306
|
+
leaves: [],
|
|
8307
|
+
windows: [],
|
|
8308
|
+
failures: [],
|
|
8309
|
+
rateLimited: null,
|
|
8310
|
+
coveragePassSeeded: false,
|
|
8311
|
+
coveragePassVersion: LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION,
|
|
8312
|
+
distinctAccounts: 0,
|
|
8313
|
+
completedByDistinctCount: false,
|
|
8314
|
+
};
|
|
8315
|
+
}
|
|
8316
|
+
function seedLocalAccountSearchCollectorTasks(checkpoint) {
|
|
8317
|
+
for (const pass of LOCAL_ACCOUNT_SEARCH_COLLECTOR_PASSES) {
|
|
8318
|
+
seedLocalAccountSearchCollectorPassTasks(checkpoint, pass.name, false);
|
|
8319
|
+
}
|
|
8320
|
+
checkpoint.coveragePassSeeded = true;
|
|
8321
|
+
checkpoint.coveragePassVersion =
|
|
8322
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION;
|
|
8323
|
+
}
|
|
8324
|
+
function seedLocalAccountSearchCollectorPassTasks(checkpoint, passName, prepend) {
|
|
8325
|
+
const dimensions = new Map(DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS.map((dimension) => [
|
|
8326
|
+
dimension.filterType,
|
|
8327
|
+
dimension,
|
|
8328
|
+
]));
|
|
8329
|
+
const seenQueries = new Set(checkpoint.seenQueries);
|
|
8330
|
+
const pass = LOCAL_ACCOUNT_SEARCH_COLLECTOR_PASSES.find((candidate) => candidate.name === passName);
|
|
8331
|
+
if (!pass) {
|
|
8332
|
+
return false;
|
|
8333
|
+
}
|
|
8334
|
+
const [firstFilterType, ...remainingDimensions] = pass.filterTypes;
|
|
8335
|
+
const dimension = dimensions.get(firstFilterType);
|
|
8336
|
+
if (!dimension) {
|
|
8337
|
+
throw new Error(`Missing Account Search split dimension ${firstFilterType}.`);
|
|
8338
|
+
}
|
|
8339
|
+
const tasks = [];
|
|
8340
|
+
for (const partition of buildLocalAccountSearchCollectorDimensionPartitions(dimension)) {
|
|
8341
|
+
const filters = [partition.filter];
|
|
8342
|
+
const searchUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, filters).slicedQueryUrl;
|
|
8343
|
+
const queryKey = localAccountSearchCollectorSeenKey(pass.name, searchUrl);
|
|
8344
|
+
if (seenQueries.has(queryKey)) {
|
|
8345
|
+
continue;
|
|
8346
|
+
}
|
|
8347
|
+
seenQueries.add(queryKey);
|
|
8348
|
+
tasks.push({
|
|
8349
|
+
kind: "node",
|
|
8350
|
+
id: `${pass.name}-${partition.idSuffix}`,
|
|
8351
|
+
pass: pass.name,
|
|
8352
|
+
filters,
|
|
8353
|
+
remainingDimensions: [...remainingDimensions],
|
|
8354
|
+
});
|
|
8355
|
+
}
|
|
8356
|
+
if (prepend) {
|
|
8357
|
+
checkpoint.taskQueue.unshift(...tasks);
|
|
8358
|
+
}
|
|
8359
|
+
else {
|
|
8360
|
+
checkpoint.taskQueue.push(...tasks);
|
|
8361
|
+
}
|
|
8362
|
+
checkpoint.seenQueries = [...seenQueries];
|
|
8363
|
+
return tasks.length > 0;
|
|
8364
|
+
}
|
|
8365
|
+
function enqueueLocalAccountSearchCollectorTasks(checkpoint, parentTask, tasks) {
|
|
8366
|
+
if (tasks.length === 0) {
|
|
8367
|
+
return;
|
|
8368
|
+
}
|
|
8369
|
+
if (parentTask.pass.startsWith("coverage-")) {
|
|
8370
|
+
checkpoint.taskQueue.splice(1, 0, ...tasks);
|
|
8371
|
+
return;
|
|
8372
|
+
}
|
|
8373
|
+
checkpoint.taskQueue.push(...tasks);
|
|
8374
|
+
}
|
|
8375
|
+
function extractLocalSalesNavigatorElements(body) {
|
|
8376
|
+
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
8377
|
+
return [];
|
|
8378
|
+
}
|
|
8379
|
+
const record = body;
|
|
8380
|
+
const elements = record.elements;
|
|
8381
|
+
if (Array.isArray(elements)) {
|
|
8382
|
+
return elements;
|
|
8383
|
+
}
|
|
8384
|
+
const data = typeof record.data === "object" &&
|
|
8385
|
+
record.data !== null &&
|
|
8386
|
+
!Array.isArray(record.data)
|
|
8387
|
+
? record.data
|
|
8388
|
+
: null;
|
|
8389
|
+
const nestedElements = data?.elements;
|
|
8390
|
+
if (Array.isArray(nestedElements)) {
|
|
8391
|
+
return nestedElements;
|
|
8392
|
+
}
|
|
8393
|
+
const references = data?.["*elements"];
|
|
8394
|
+
const included = record.included;
|
|
8395
|
+
if (!Array.isArray(references) || !Array.isArray(included)) {
|
|
8396
|
+
return [];
|
|
8397
|
+
}
|
|
8398
|
+
const referencedUrns = new Set(references.filter((reference) => typeof reference === "string" && reference.length > 0));
|
|
8399
|
+
return included.filter((value) => {
|
|
8400
|
+
if (typeof value !== "object" ||
|
|
8401
|
+
value === null ||
|
|
8402
|
+
Array.isArray(value)) {
|
|
8403
|
+
return false;
|
|
8404
|
+
}
|
|
8405
|
+
const entityUrn = value.entityUrn;
|
|
8406
|
+
return typeof entityUrn === "string" && referencedUrns.has(entityUrn);
|
|
8407
|
+
});
|
|
8408
|
+
}
|
|
8409
|
+
async function appendLocalAccountSearchRawElements(params) {
|
|
8410
|
+
if (params.elements.length === 0) {
|
|
8411
|
+
return;
|
|
8412
|
+
}
|
|
8413
|
+
await mkdir(path.dirname(params.rawOutputPath), { recursive: true });
|
|
8414
|
+
await appendFile(params.rawOutputPath, `${params.elements
|
|
8415
|
+
.map((element) => JSON.stringify({ queryUrl: params.searchUrl, element }))
|
|
8416
|
+
.join("\n")}\n`, "utf8");
|
|
8417
|
+
}
|
|
8418
|
+
function addLocalAccountSearchElementKey(accountKeys, element, queryUrl) {
|
|
8419
|
+
if (typeof element !== "object" || element === null || Array.isArray(element)) {
|
|
8420
|
+
return;
|
|
8421
|
+
}
|
|
8422
|
+
const companyId = extractLinkedInSalesCompanyIdFromUrn(element.entityUrn);
|
|
8423
|
+
if (companyId) {
|
|
8424
|
+
accountKeys.add(`company-id:${companyId}`);
|
|
8425
|
+
return;
|
|
8426
|
+
}
|
|
8427
|
+
const account = normalizeLocalSalesNavigatorAccount(element, queryUrl);
|
|
8428
|
+
if (account) {
|
|
8429
|
+
accountKeys.add(account.accountKey);
|
|
8430
|
+
}
|
|
8431
|
+
}
|
|
8432
|
+
async function loadLocalAccountSearchRawAccountKeys(rawOutputPath) {
|
|
8433
|
+
const accountKeys = new Set();
|
|
8434
|
+
if (!(await fileExists(rawOutputPath))) {
|
|
8435
|
+
return accountKeys;
|
|
8436
|
+
}
|
|
8437
|
+
const raw = await readFile(rawOutputPath, "utf8");
|
|
8438
|
+
for (const line of raw.split(/\r?\n/)) {
|
|
8439
|
+
if (!line.trim()) {
|
|
8440
|
+
continue;
|
|
8441
|
+
}
|
|
8442
|
+
try {
|
|
8443
|
+
const row = JSON.parse(line);
|
|
8444
|
+
addLocalAccountSearchElementKey(accountKeys, row.element, typeof row.queryUrl === "string" ? row.queryUrl : "");
|
|
8445
|
+
}
|
|
8446
|
+
catch {
|
|
8447
|
+
// The import command reports malformed raw rows with line-level context.
|
|
8448
|
+
}
|
|
8449
|
+
}
|
|
8450
|
+
return accountKeys;
|
|
8451
|
+
}
|
|
8452
|
+
async function saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint) {
|
|
8453
|
+
checkpoint.updatedAt = new Date().toISOString();
|
|
8454
|
+
await writeJsonFile(checkpointPath, checkpoint);
|
|
8455
|
+
}
|
|
6853
8456
|
async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
|
|
6854
8457
|
const people = [];
|
|
6855
8458
|
const seen = new Set();
|
|
@@ -6873,7 +8476,9 @@ async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
|
|
|
6873
8476
|
...parsedRequest,
|
|
6874
8477
|
url: withSalesNavigatorPaging(parsedRequest.url, start, currentCount),
|
|
6875
8478
|
};
|
|
6876
|
-
const response =
|
|
8479
|
+
const response = options.executeRequest
|
|
8480
|
+
? await options.executeRequest(pageRequest)
|
|
8481
|
+
: await fetchLocalSalesNavigatorRequest(pageRequest, options.retry);
|
|
6877
8482
|
retryCount += response.retryCount;
|
|
6878
8483
|
totalDelayMs += response.retryDelayMs;
|
|
6879
8484
|
const responseBody = response.body;
|
|
@@ -6919,6 +8524,7 @@ async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
|
|
|
6919
8524
|
async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
6920
8525
|
const accounts = [];
|
|
6921
8526
|
const seen = new Set();
|
|
8527
|
+
let filteredAccounts = 0;
|
|
6922
8528
|
let totalResults = null;
|
|
6923
8529
|
let fetchedPages = 0;
|
|
6924
8530
|
let totalDelayMs = 0;
|
|
@@ -6993,13 +8599,20 @@ async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
|
6993
8599
|
}
|
|
6994
8600
|
const pageAccounts = normalizeLocalSalesNavigatorAccounts(responseBody, pageRequest.url);
|
|
6995
8601
|
let added = 0;
|
|
8602
|
+
let addedOrFiltered = 0;
|
|
6996
8603
|
for (const account of pageAccounts) {
|
|
6997
8604
|
if (seen.has(account.accountKey)) {
|
|
6998
8605
|
continue;
|
|
6999
8606
|
}
|
|
7000
|
-
seen.add(account.accountKey);
|
|
8607
|
+
seen.add(account.accountKey);
|
|
8608
|
+
if (!salesNavigatorAccountMatchesLocationFilter(account, options.accountLocationFilter)) {
|
|
8609
|
+
filteredAccounts += 1;
|
|
8610
|
+
addedOrFiltered += 1;
|
|
8611
|
+
continue;
|
|
8612
|
+
}
|
|
7001
8613
|
accounts.push(account);
|
|
7002
8614
|
added += 1;
|
|
8615
|
+
addedOrFiltered += 1;
|
|
7003
8616
|
if (requestedAccounts != null && accounts.length >= requestedAccounts) {
|
|
7004
8617
|
break;
|
|
7005
8618
|
}
|
|
@@ -7024,7 +8637,7 @@ async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
|
7024
8637
|
nextStart = start;
|
|
7025
8638
|
break;
|
|
7026
8639
|
}
|
|
7027
|
-
if (
|
|
8640
|
+
if (addedOrFiltered === 0) {
|
|
7028
8641
|
stopReason = "duplicate_page";
|
|
7029
8642
|
stoppedAtStart = start;
|
|
7030
8643
|
stoppedAtCount = currentCount;
|
|
@@ -7048,6 +8661,7 @@ async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
|
7048
8661
|
const missingFromReported = totalResults == null ? null : Math.max(0, totalResults - fetchedThrough);
|
|
7049
8662
|
return {
|
|
7050
8663
|
accounts,
|
|
8664
|
+
filteredAccounts,
|
|
7051
8665
|
totalResults,
|
|
7052
8666
|
fetchedPages,
|
|
7053
8667
|
sourceQueryUrl: parsedRequest.url,
|
|
@@ -7129,6 +8743,13 @@ async function importLocalSalesNavigatorPeopleViaApp(session, payload) {
|
|
|
7129
8743
|
}, SalesNavigatorLocalImportResponseSchema);
|
|
7130
8744
|
return value;
|
|
7131
8745
|
}
|
|
8746
|
+
const SALESPROMPTER_SUPABASE_TABLES = {
|
|
8747
|
+
accountSearchRuns: "salesprompter_linkedin_sales_nav_account_search_runs",
|
|
8748
|
+
accountSearchSlices: "salesprompter_linkedin_sales_nav_account_search_slices",
|
|
8749
|
+
accountSearchAccounts: "salesprompter_linkedin_sales_nav_accounts",
|
|
8750
|
+
accountRunMemberships: "salesprompter_linkedin_sales_nav_account_run_memberships",
|
|
8751
|
+
linkedinCompaniesProcessed: "salesprompter_linkedin_companies_processed",
|
|
8752
|
+
};
|
|
7132
8753
|
async function createSalesNavigatorAccountSearchStore() {
|
|
7133
8754
|
const databaseUrl = process.env.SALESPROMPTER_DATABASE_URL?.trim() ||
|
|
7134
8755
|
process.env.DATABASE_URL?.trim() ||
|
|
@@ -7159,7 +8780,7 @@ async function closeSalesNavigatorAccountSearchStore(store) {
|
|
|
7159
8780
|
}
|
|
7160
8781
|
async function createSalesNavigatorAccountSearchRun(params) {
|
|
7161
8782
|
if (params.store.kind === "postgres") {
|
|
7162
|
-
const result = await params.store.client.query(`INSERT INTO public.
|
|
8783
|
+
const result = await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_account_search_runs (
|
|
7163
8784
|
org_id, source_query_url, slice_preset, max_results_per_search, page_size, status, raw_payload
|
|
7164
8785
|
)
|
|
7165
8786
|
VALUES ($1, $2, $3, $4, $5, 'running', $6::jsonb)
|
|
@@ -7173,12 +8794,12 @@ async function createSalesNavigatorAccountSearchRun(params) {
|
|
|
7173
8794
|
]);
|
|
7174
8795
|
const id = result.rows[0]?.id ?? "";
|
|
7175
8796
|
if (!id) {
|
|
7176
|
-
throw new Error("Failed to create account search run in
|
|
8797
|
+
throw new Error("Failed to create account search run in Supabase: missing run id.");
|
|
7177
8798
|
}
|
|
7178
8799
|
return id;
|
|
7179
8800
|
}
|
|
7180
8801
|
const { data, error } = await params.store.client
|
|
7181
|
-
.from(
|
|
8802
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchRuns)
|
|
7182
8803
|
.insert({
|
|
7183
8804
|
org_id: params.orgId,
|
|
7184
8805
|
source_query_url: params.sourceQueryUrl,
|
|
@@ -7191,17 +8812,17 @@ async function createSalesNavigatorAccountSearchRun(params) {
|
|
|
7191
8812
|
.select("id")
|
|
7192
8813
|
.single();
|
|
7193
8814
|
if (error) {
|
|
7194
|
-
throw new Error(`Failed to create account search run in
|
|
8815
|
+
throw new Error(`Failed to create account search run in Supabase: ${error.message}`);
|
|
7195
8816
|
}
|
|
7196
8817
|
const id = typeof data?.id === "string" ? data.id : "";
|
|
7197
8818
|
if (!id) {
|
|
7198
|
-
throw new Error("Failed to create account search run in
|
|
8819
|
+
throw new Error("Failed to create account search run in Supabase: missing run id.");
|
|
7199
8820
|
}
|
|
7200
8821
|
return id;
|
|
7201
8822
|
}
|
|
7202
8823
|
async function updateSalesNavigatorAccountSearchRun(params) {
|
|
7203
8824
|
if (params.store.kind === "postgres") {
|
|
7204
|
-
await params.store.client.query(`UPDATE public.
|
|
8825
|
+
await params.store.client.query(`UPDATE public.salesprompter_linkedin_sales_nav_account_search_runs
|
|
7205
8826
|
SET status = $2,
|
|
7206
8827
|
total_slices = $3,
|
|
7207
8828
|
stored_accounts = $4,
|
|
@@ -7234,12 +8855,12 @@ async function updateSalesNavigatorAccountSearchRun(params) {
|
|
|
7234
8855
|
let mergedRawPayload = null;
|
|
7235
8856
|
if (params.rawPayload) {
|
|
7236
8857
|
const { data: existing, error: selectError } = await params.store.client
|
|
7237
|
-
.from(
|
|
8858
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchRuns)
|
|
7238
8859
|
.select("raw_payload")
|
|
7239
8860
|
.eq("id", params.runId)
|
|
7240
8861
|
.maybeSingle();
|
|
7241
8862
|
if (selectError) {
|
|
7242
|
-
throw new Error(`Failed to read account search run payload in
|
|
8863
|
+
throw new Error(`Failed to read account search run payload in Supabase: ${selectError.message}`);
|
|
7243
8864
|
}
|
|
7244
8865
|
const existingPayload = typeof existing?.raw_payload === "object" &&
|
|
7245
8866
|
existing.raw_payload !== null &&
|
|
@@ -7252,7 +8873,7 @@ async function updateSalesNavigatorAccountSearchRun(params) {
|
|
|
7252
8873
|
};
|
|
7253
8874
|
}
|
|
7254
8875
|
const { error } = await params.store.client
|
|
7255
|
-
.from(
|
|
8876
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchRuns)
|
|
7256
8877
|
.update({
|
|
7257
8878
|
status: params.status,
|
|
7258
8879
|
total_slices: params.totalSlices,
|
|
@@ -7272,12 +8893,12 @@ async function updateSalesNavigatorAccountSearchRun(params) {
|
|
|
7272
8893
|
})
|
|
7273
8894
|
.eq("id", params.runId);
|
|
7274
8895
|
if (error) {
|
|
7275
|
-
throw new Error(`Failed to update account search run in
|
|
8896
|
+
throw new Error(`Failed to update account search run in Supabase: ${error.message}`);
|
|
7276
8897
|
}
|
|
7277
8898
|
}
|
|
7278
8899
|
async function upsertSalesNavigatorAccountSearchSlice(params) {
|
|
7279
8900
|
if (params.store.kind === "postgres") {
|
|
7280
|
-
const result = await params.store.client.query(`INSERT INTO public.
|
|
8901
|
+
const result = await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_account_search_slices (
|
|
7281
8902
|
run_id, org_id, sliced_query_url, applied_filters, split_trail, depth, status,
|
|
7282
8903
|
total_results, fetched_pages, stored_accounts, retry_count, retry_delay_ms, last_error, raw_payload
|
|
7283
8904
|
)
|
|
@@ -7313,12 +8934,12 @@ async function upsertSalesNavigatorAccountSearchSlice(params) {
|
|
|
7313
8934
|
]);
|
|
7314
8935
|
const id = result.rows[0]?.id ?? "";
|
|
7315
8936
|
if (!id) {
|
|
7316
|
-
throw new Error("Failed to upsert account search slice in
|
|
8937
|
+
throw new Error("Failed to upsert account search slice in Supabase: missing slice id.");
|
|
7317
8938
|
}
|
|
7318
8939
|
return id;
|
|
7319
8940
|
}
|
|
7320
8941
|
const { data, error } = await params.store.client
|
|
7321
|
-
.from(
|
|
8942
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchSlices)
|
|
7322
8943
|
.upsert({
|
|
7323
8944
|
run_id: params.runId,
|
|
7324
8945
|
org_id: params.orgId,
|
|
@@ -7338,11 +8959,11 @@ async function upsertSalesNavigatorAccountSearchSlice(params) {
|
|
|
7338
8959
|
.select("id")
|
|
7339
8960
|
.single();
|
|
7340
8961
|
if (error) {
|
|
7341
|
-
throw new Error(`Failed to upsert account search slice in
|
|
8962
|
+
throw new Error(`Failed to upsert account search slice in Supabase: ${error.message}`);
|
|
7342
8963
|
}
|
|
7343
8964
|
const id = typeof data?.id === "string" ? data.id : "";
|
|
7344
8965
|
if (!id) {
|
|
7345
|
-
throw new Error("Failed to upsert account search slice in
|
|
8966
|
+
throw new Error("Failed to upsert account search slice in Supabase: missing slice id.");
|
|
7346
8967
|
}
|
|
7347
8968
|
return id;
|
|
7348
8969
|
}
|
|
@@ -7821,7 +9442,7 @@ async function upsertLocalLinkedInCompanyScrape(row) {
|
|
|
7821
9442
|
let committed = false;
|
|
7822
9443
|
try {
|
|
7823
9444
|
await client.query("BEGIN");
|
|
7824
|
-
await client.query(`INSERT INTO public.
|
|
9445
|
+
await client.query(`INSERT INTO public.salesprompter_linkedin_companies_processed (
|
|
7825
9446
|
error, query, timestamp, name, description, follower_count, website, domain, industry,
|
|
7826
9447
|
company_size, founded, company_address, main_company_id, linkedin_id, sales_navigator_link,
|
|
7827
9448
|
employees_on_linked_in, company_name, total_employee_count, company_url, headquarters
|
|
@@ -8027,7 +9648,7 @@ async function upsertSalesNavigatorAccounts(params) {
|
|
|
8027
9648
|
}));
|
|
8028
9649
|
if (params.store.kind === "postgres") {
|
|
8029
9650
|
for (const row of rows) {
|
|
8030
|
-
await params.store.client.query(`INSERT INTO public.
|
|
9651
|
+
await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_accounts (
|
|
8031
9652
|
org_id, last_run_id, last_slice_id, account_key, sales_nav_company_url,
|
|
8032
9653
|
linkedin_company_url, company_id, company_name, industry, location, headquarters,
|
|
8033
9654
|
employee_count, follower_count, company_type, description, website, search_query_url,
|
|
@@ -8076,17 +9697,62 @@ async function upsertSalesNavigatorAccounts(params) {
|
|
|
8076
9697
|
row.scraped_at,
|
|
8077
9698
|
JSON.stringify(row.raw_payload),
|
|
8078
9699
|
]);
|
|
9700
|
+
await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_account_run_memberships (
|
|
9701
|
+
run_id, org_id, account_key, last_slice_id
|
|
9702
|
+
)
|
|
9703
|
+
VALUES ($1, $2, $3, $4)
|
|
9704
|
+
ON CONFLICT (run_id, account_key)
|
|
9705
|
+
DO UPDATE SET
|
|
9706
|
+
last_slice_id = excluded.last_slice_id,
|
|
9707
|
+
updated_at = now()`, [
|
|
9708
|
+
row.last_run_id,
|
|
9709
|
+
row.org_id,
|
|
9710
|
+
row.account_key,
|
|
9711
|
+
row.last_slice_id,
|
|
9712
|
+
]);
|
|
8079
9713
|
}
|
|
8080
9714
|
return rows.length;
|
|
8081
9715
|
}
|
|
8082
|
-
const
|
|
8083
|
-
|
|
8084
|
-
.
|
|
8085
|
-
|
|
8086
|
-
|
|
9716
|
+
const supabaseBatchSize = 250;
|
|
9717
|
+
for (let index = 0; index < rows.length; index += supabaseBatchSize) {
|
|
9718
|
+
const batch = rows.slice(index, index + supabaseBatchSize);
|
|
9719
|
+
const { error } = await params.store.client
|
|
9720
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchAccounts)
|
|
9721
|
+
.upsert(batch, { onConflict: "org_id,account_key" });
|
|
9722
|
+
if (error) {
|
|
9723
|
+
throw new Error(`Failed to upsert Sales Navigator accounts in Supabase: ${error.message}`);
|
|
9724
|
+
}
|
|
9725
|
+
const { error: membershipError } = await params.store.client
|
|
9726
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountRunMemberships)
|
|
9727
|
+
.upsert(batch.map((row) => ({
|
|
9728
|
+
run_id: row.last_run_id,
|
|
9729
|
+
org_id: row.org_id,
|
|
9730
|
+
account_key: row.account_key,
|
|
9731
|
+
last_slice_id: row.last_slice_id,
|
|
9732
|
+
updated_at: new Date().toISOString(),
|
|
9733
|
+
})), { onConflict: "run_id,account_key" });
|
|
9734
|
+
if (membershipError) {
|
|
9735
|
+
throw new Error(`Failed to record Sales Navigator account run membership in Supabase: ${membershipError.message}`);
|
|
9736
|
+
}
|
|
8087
9737
|
}
|
|
8088
9738
|
return rows.length;
|
|
8089
9739
|
}
|
|
9740
|
+
async function countSalesNavigatorAccountsForRun(params) {
|
|
9741
|
+
if (params.store.kind === "postgres") {
|
|
9742
|
+
const result = await params.store.client.query(`SELECT count(*)::text AS count
|
|
9743
|
+
FROM public.salesprompter_linkedin_sales_nav_account_run_memberships
|
|
9744
|
+
WHERE run_id = $1`, [params.runId]);
|
|
9745
|
+
return Number(result.rows[0]?.count ?? 0);
|
|
9746
|
+
}
|
|
9747
|
+
const { count, error } = await params.store.client
|
|
9748
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountRunMemberships)
|
|
9749
|
+
.select("account_key", { count: "exact", head: true })
|
|
9750
|
+
.eq("run_id", params.runId);
|
|
9751
|
+
if (error) {
|
|
9752
|
+
throw new Error(`Failed to count Sales Navigator account run membership in Supabase: ${error.message}`);
|
|
9753
|
+
}
|
|
9754
|
+
return count ?? 0;
|
|
9755
|
+
}
|
|
8090
9756
|
function asJsonObject(value) {
|
|
8091
9757
|
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
8092
9758
|
? value
|
|
@@ -8145,8 +9811,8 @@ async function listSalesNavigatorAccountSearchRepairSlices(params) {
|
|
|
8145
9811
|
runs.slice_preset,
|
|
8146
9812
|
runs.max_results_per_search,
|
|
8147
9813
|
runs.page_size
|
|
8148
|
-
FROM public.
|
|
8149
|
-
INNER JOIN public.
|
|
9814
|
+
FROM public.salesprompter_linkedin_sales_nav_account_search_slices AS slices
|
|
9815
|
+
INNER JOIN public.salesprompter_linkedin_sales_nav_account_search_runs AS runs
|
|
8150
9816
|
ON runs.id = slices.run_id
|
|
8151
9817
|
WHERE slices.run_id = $1
|
|
8152
9818
|
AND slices.status IN ('failed', 'unresolved')
|
|
@@ -8178,32 +9844,52 @@ async function listSalesNavigatorAccountSearchRepairSlices(params) {
|
|
|
8178
9844
|
}));
|
|
8179
9845
|
}
|
|
8180
9846
|
async function summarizeSalesNavigatorAccountSearchRunFromSlices(params) {
|
|
8181
|
-
if (params.store.kind
|
|
8182
|
-
|
|
9847
|
+
if (params.store.kind === "supabase") {
|
|
9848
|
+
const { data, error } = await params.store.client
|
|
9849
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchSlices)
|
|
9850
|
+
.select("status,last_error,updated_at")
|
|
9851
|
+
.eq("run_id", params.runId);
|
|
9852
|
+
if (error) {
|
|
9853
|
+
throw new Error(`Failed to summarize Sales Navigator account slices in Supabase: ${error.message}`);
|
|
9854
|
+
}
|
|
9855
|
+
const rows = (data ?? []);
|
|
9856
|
+
const finalRows = rows.filter((row) => row.status !== "split");
|
|
9857
|
+
const failedRows = rows.filter((row) => row.status === "failed");
|
|
9858
|
+
const unresolvedRows = rows.filter((row) => row.status === "unresolved");
|
|
9859
|
+
const latestProblem = [...failedRows, ...unresolvedRows].sort((a, b) => Date.parse(b.updated_at) - Date.parse(a.updated_at))[0];
|
|
9860
|
+
return {
|
|
9861
|
+
totalSlices: finalRows.length,
|
|
9862
|
+
storedAccounts: await countSalesNavigatorAccountsForRun(params),
|
|
9863
|
+
attemptedSlices: finalRows.length,
|
|
9864
|
+
splitEvents: rows.filter((row) => row.status === "split").length,
|
|
9865
|
+
failedSlices: failedRows.length,
|
|
9866
|
+
unresolvedSlices: unresolvedRows.length,
|
|
9867
|
+
truncated: failedRows.length > 0 || unresolvedRows.length > 0,
|
|
9868
|
+
lastError: latestProblem?.last_error ?? null,
|
|
9869
|
+
};
|
|
8183
9870
|
}
|
|
8184
9871
|
const result = await params.store.client.query(`SELECT
|
|
8185
9872
|
count(*) FILTER (WHERE status <> 'split')::int AS total_slices,
|
|
8186
|
-
COALESCE(sum(stored_accounts) FILTER (WHERE status <> 'split'), 0)::int AS stored_accounts,
|
|
8187
9873
|
count(*) FILTER (WHERE status <> 'split')::int AS attempted_slices,
|
|
8188
9874
|
count(*) FILTER (WHERE status = 'split')::int AS split_events,
|
|
8189
9875
|
count(*) FILTER (WHERE status = 'failed')::int AS failed_slices,
|
|
8190
9876
|
count(*) FILTER (WHERE status = 'unresolved')::int AS unresolved_slices,
|
|
8191
9877
|
(
|
|
8192
9878
|
SELECT last_error
|
|
8193
|
-
FROM public.
|
|
9879
|
+
FROM public.salesprompter_linkedin_sales_nav_account_search_slices
|
|
8194
9880
|
WHERE run_id = $1
|
|
8195
9881
|
AND status IN ('failed', 'unresolved')
|
|
8196
9882
|
ORDER BY updated_at DESC
|
|
8197
9883
|
LIMIT 1
|
|
8198
9884
|
) AS last_error
|
|
8199
|
-
FROM public.
|
|
9885
|
+
FROM public.salesprompter_linkedin_sales_nav_account_search_slices
|
|
8200
9886
|
WHERE run_id = $1`, [params.runId]);
|
|
8201
9887
|
const row = result.rows[0] ?? {};
|
|
8202
9888
|
const failedSlices = Number(row.failed_slices ?? 0);
|
|
8203
9889
|
const unresolvedSlices = Number(row.unresolved_slices ?? 0);
|
|
8204
9890
|
return {
|
|
8205
9891
|
totalSlices: Number(row.total_slices ?? 0),
|
|
8206
|
-
storedAccounts:
|
|
9892
|
+
storedAccounts: await countSalesNavigatorAccountsForRun(params),
|
|
8207
9893
|
attemptedSlices: Number(row.attempted_slices ?? 0),
|
|
8208
9894
|
splitEvents: Number(row.split_events ?? 0),
|
|
8209
9895
|
failedSlices,
|
|
@@ -11499,6 +13185,199 @@ program
|
|
|
11499
13185
|
}
|
|
11500
13186
|
});
|
|
11501
13187
|
});
|
|
13188
|
+
async function runAffiliateLaunchCommand(options) {
|
|
13189
|
+
const affiliateLink = z.string().url().parse(options.affiliateLink);
|
|
13190
|
+
const linkedInUrl = z.string().url().parse(options.linkedinUrl);
|
|
13191
|
+
const maxResults = z.coerce.number().int().min(1).max(1000).parse(options.maxResults);
|
|
13192
|
+
const session = await requireAuthSession();
|
|
13193
|
+
const isSalesNavigatorPeopleSearch = new URL(linkedInUrl).pathname.includes("/sales/search/people");
|
|
13194
|
+
let localResults;
|
|
13195
|
+
let localCollection;
|
|
13196
|
+
if (!options.dryRun && isSalesNavigatorPeopleSearch) {
|
|
13197
|
+
const pageSize = z.coerce.number().int().min(1).max(100).parse(options.pageSize);
|
|
13198
|
+
const pageDelayMinMs = z.coerce.number().int().min(0).parse(options.pageDelayMinMs);
|
|
13199
|
+
const pageDelayMaxMs = z.coerce.number().int().min(pageDelayMinMs).parse(options.pageDelayMaxMs);
|
|
13200
|
+
const browserRelayPort = options.browserRelayPort == null
|
|
13201
|
+
? null
|
|
13202
|
+
: z.coerce.number().int().min(0).max(65535).parse(options.browserRelayPort);
|
|
13203
|
+
if (options.curlFile && browserRelayPort != null) {
|
|
13204
|
+
throw new Error("Use either --curl-file or --browser-relay-port, not both.");
|
|
13205
|
+
}
|
|
13206
|
+
const parsedRequest = options.curlFile
|
|
13207
|
+
? parseSalesNavigatorCurlRequest(await readFile(path.resolve(String(options.curlFile)), "utf8"))
|
|
13208
|
+
: browserRelayPort != null
|
|
13209
|
+
? {
|
|
13210
|
+
url: buildSalesNavigatorLeadApiUrlFromSearchUrl(linkedInUrl, pageSize),
|
|
13211
|
+
headers: {},
|
|
13212
|
+
}
|
|
13213
|
+
: buildSalesNavigatorApiRequestFromSearchUrl(linkedInUrl, await readLinkedInDirectLookupConfig(), pageSize);
|
|
13214
|
+
if (!new URL(parsedRequest.url).pathname.includes("/sales-api/salesApiLeadSearch")) {
|
|
13215
|
+
throw new Error("Affiliate --curl-file must contain a /sales-api/salesApiLeadSearch request.");
|
|
13216
|
+
}
|
|
13217
|
+
const browserRelay = browserRelayPort == null
|
|
13218
|
+
? null
|
|
13219
|
+
: await createLocalAccountSearchBrowserRelay(browserRelayPort);
|
|
13220
|
+
let collected;
|
|
13221
|
+
try {
|
|
13222
|
+
collected = await fetchAllLocalSalesNavigatorPeople(parsedRequest, {
|
|
13223
|
+
requestedProfiles: maxResults,
|
|
13224
|
+
pageSize,
|
|
13225
|
+
pageDelayMinMs,
|
|
13226
|
+
pageDelayMaxMs,
|
|
13227
|
+
retry: {
|
|
13228
|
+
maxRetries: 2,
|
|
13229
|
+
retryBaseDelayMs: 2_000,
|
|
13230
|
+
retryMaxDelayMs: 10_000,
|
|
13231
|
+
},
|
|
13232
|
+
executeRequest: browserRelay
|
|
13233
|
+
? (request) => browserRelay.request(request.url)
|
|
13234
|
+
: undefined,
|
|
13235
|
+
});
|
|
13236
|
+
}
|
|
13237
|
+
finally {
|
|
13238
|
+
await browserRelay?.close();
|
|
13239
|
+
}
|
|
13240
|
+
localResults = collected.people;
|
|
13241
|
+
localCollection = {
|
|
13242
|
+
totalResults: collected.totalResults,
|
|
13243
|
+
fetchedPages: collected.fetchedPages,
|
|
13244
|
+
pacing: collected.pacing,
|
|
13245
|
+
};
|
|
13246
|
+
}
|
|
13247
|
+
const payload = await launchAffiliateCampaignViaApp(session, {
|
|
13248
|
+
affiliateLink,
|
|
13249
|
+
linkedInUrl,
|
|
13250
|
+
maxResults,
|
|
13251
|
+
dryRun: Boolean(options.dryRun),
|
|
13252
|
+
localResults,
|
|
13253
|
+
localCollection,
|
|
13254
|
+
});
|
|
13255
|
+
if (options.out) {
|
|
13256
|
+
await writeJsonFile(options.out, payload);
|
|
13257
|
+
}
|
|
13258
|
+
return { session, payload };
|
|
13259
|
+
}
|
|
13260
|
+
function addAffiliateAudienceOptions(command) {
|
|
13261
|
+
return command
|
|
13262
|
+
.requiredOption("--affiliate-link <url>", "Affiliate link to promote")
|
|
13263
|
+
.requiredOption("--linkedin-url <url>", "Sales Navigator people or LinkedIn content search URL")
|
|
13264
|
+
.option("--max-results <number>", "Maximum audience results", "1000")
|
|
13265
|
+
.option("--curl-file <path>", "Optional copied Sales Navigator Lead Search curl request")
|
|
13266
|
+
.option("--browser-relay-port <number>", "Use the signed-in browser through a loopback relay")
|
|
13267
|
+
.option("--page-size <number>", "Direct Sales Navigator page size", "100")
|
|
13268
|
+
.option("--page-delay-min-ms <number>", "Minimum delay between direct pages", "5000")
|
|
13269
|
+
.option("--page-delay-max-ms <number>", "Maximum delay between direct pages", "8000")
|
|
13270
|
+
.option("--dry-run", "Validate and preview without creating remote resources", false)
|
|
13271
|
+
.option("--out <path>", "Optional local JSON output path");
|
|
13272
|
+
}
|
|
13273
|
+
addAffiliateAudienceOptions(program
|
|
13274
|
+
.command("affiliate:launch")
|
|
13275
|
+
.description("Build an affiliate audience from an affiliate link and LinkedIn search URL."))
|
|
13276
|
+
.action(async (options) => {
|
|
13277
|
+
const { payload } = await runAffiliateLaunchCommand(options);
|
|
13278
|
+
printOutput(payload);
|
|
13279
|
+
});
|
|
13280
|
+
addAffiliateAudienceOptions(program
|
|
13281
|
+
.command("affiliate:run")
|
|
13282
|
+
.description("Build an affiliate audience and prepare a reviewed Instantly campaign end to end."))
|
|
13283
|
+
.option("--campaign-name <name>", "Optional Instantly campaign name")
|
|
13284
|
+
.option("--language <language>", "Sequence language", "English")
|
|
13285
|
+
.option("--steps <number>", "Number of sequence emails", "3")
|
|
13286
|
+
.option("--min-email-score <number>", "Minimum Hunter confidence score", "80")
|
|
13287
|
+
.option("--allow-accept-all", "Include catch-all domains that meet the score threshold", false)
|
|
13288
|
+
.option("--daily-limit <number>", "Maximum new leads contacted per day", "25")
|
|
13289
|
+
.option("--timezone <timezone>", "Instantly sending timezone", "America/Detroit")
|
|
13290
|
+
.option("--send-from <time>", "Weekday sending window start (HH:MM)", "09:00")
|
|
13291
|
+
.option("--send-to <time>", "Weekday sending window end (HH:MM)", "16:00")
|
|
13292
|
+
.option("--activate", "Activate the prepared Instantly campaign after review data is returned", false)
|
|
13293
|
+
.action(async (options) => {
|
|
13294
|
+
const launched = await runAffiliateLaunchCommand(options);
|
|
13295
|
+
if (options.dryRun) {
|
|
13296
|
+
printOutput({
|
|
13297
|
+
...launched.payload,
|
|
13298
|
+
outreach: {
|
|
13299
|
+
status: "dry-run",
|
|
13300
|
+
provider: "instantly",
|
|
13301
|
+
activation: "not_requested"
|
|
13302
|
+
}
|
|
13303
|
+
});
|
|
13304
|
+
return;
|
|
13305
|
+
}
|
|
13306
|
+
const runId = launched.payload.extraction &&
|
|
13307
|
+
typeof launched.payload.extraction.runId === "string"
|
|
13308
|
+
? launched.payload.extraction.runId
|
|
13309
|
+
: null;
|
|
13310
|
+
if (!runId) {
|
|
13311
|
+
throw new Error("The audience provider did not return a durable run id. End-to-end outreach requires a Sales Navigator people search.");
|
|
13312
|
+
}
|
|
13313
|
+
const outreach = await prepareAffiliateOutreachViaApp(launched.session, {
|
|
13314
|
+
audienceRunId: z.string().uuid().parse(runId),
|
|
13315
|
+
campaignName: options.campaignName?.trim() || undefined,
|
|
13316
|
+
language: z.string().trim().min(1).max(40).parse(options.language),
|
|
13317
|
+
numberOfSteps: z.coerce.number().int().min(1).max(6).parse(options.steps),
|
|
13318
|
+
minEmailScore: z.coerce.number().int().min(0).max(100).parse(options.minEmailScore),
|
|
13319
|
+
allowAcceptAll: Boolean(options.allowAcceptAll),
|
|
13320
|
+
dailyLimit: z.coerce.number().int().min(1).max(1000).parse(options.dailyLimit),
|
|
13321
|
+
timezone: z.string().trim().min(1).max(100).parse(options.timezone),
|
|
13322
|
+
sendFrom: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/).parse(options.sendFrom),
|
|
13323
|
+
sendTo: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/).parse(options.sendTo)
|
|
13324
|
+
});
|
|
13325
|
+
const finalOutreach = options.activate
|
|
13326
|
+
? await activateAffiliateOutreachViaApp(launched.session, runId)
|
|
13327
|
+
: outreach;
|
|
13328
|
+
printOutput({
|
|
13329
|
+
status: "ok",
|
|
13330
|
+
audience: launched.payload,
|
|
13331
|
+
outreach: finalOutreach,
|
|
13332
|
+
next: finalOutreach.status === "ready"
|
|
13333
|
+
? `Review the sequence, then run: salesprompter affiliate:activate ${runId}`
|
|
13334
|
+
: "The Instantly campaign is active."
|
|
13335
|
+
});
|
|
13336
|
+
});
|
|
13337
|
+
program
|
|
13338
|
+
.command("affiliate:status <run-id>")
|
|
13339
|
+
.description("Show enrichment, sequence, Instantly, and activation status for an affiliate run.")
|
|
13340
|
+
.action(async (runId) => {
|
|
13341
|
+
const session = await requireAuthSession();
|
|
13342
|
+
const outreach = await getAffiliateOutreachViaApp(session, z.string().uuid().parse(runId));
|
|
13343
|
+
printOutput({ status: "ok", outreach });
|
|
13344
|
+
});
|
|
13345
|
+
program
|
|
13346
|
+
.command("affiliate:regenerate-sequence <run-id>")
|
|
13347
|
+
.description("Regenerate sequence variants and spintax, then sync them to Instantly.")
|
|
13348
|
+
.action(async (runId) => {
|
|
13349
|
+
const session = await requireAuthSession();
|
|
13350
|
+
const outreach = await regenerateAffiliateOutreachSequenceViaApp(session, z.string().uuid().parse(runId));
|
|
13351
|
+
printOutput({
|
|
13352
|
+
status: "ok",
|
|
13353
|
+
outreach,
|
|
13354
|
+
message: "Sequence variants and spintax were regenerated and synced to Instantly."
|
|
13355
|
+
});
|
|
13356
|
+
});
|
|
13357
|
+
program
|
|
13358
|
+
.command("affiliate:enrich <run-id>")
|
|
13359
|
+
.description("Find additional verified emails and add only new leads to the campaign.")
|
|
13360
|
+
.action(async (runId) => {
|
|
13361
|
+
const session = await requireAuthSession();
|
|
13362
|
+
const outreach = await enrichAffiliateOutreachViaApp(session, z.string().uuid().parse(runId));
|
|
13363
|
+
printOutput({
|
|
13364
|
+
status: "ok",
|
|
13365
|
+
outreach,
|
|
13366
|
+
message: "Email recovery finished and new verified leads were added."
|
|
13367
|
+
});
|
|
13368
|
+
});
|
|
13369
|
+
program
|
|
13370
|
+
.command("affiliate:activate <run-id>")
|
|
13371
|
+
.description("Activate a reviewed Instantly affiliate campaign.")
|
|
13372
|
+
.action(async (runId) => {
|
|
13373
|
+
const session = await requireAuthSession();
|
|
13374
|
+
const outreach = await activateAffiliateOutreachViaApp(session, z.string().uuid().parse(runId));
|
|
13375
|
+
printOutput({
|
|
13376
|
+
status: "ok",
|
|
13377
|
+
outreach,
|
|
13378
|
+
message: "The Instantly campaign is active."
|
|
13379
|
+
});
|
|
13380
|
+
});
|
|
11502
13381
|
program
|
|
11503
13382
|
.command("sync:crm")
|
|
11504
13383
|
.description("Dry-run sync scored leads into a CRM target.")
|
|
@@ -12274,16 +14153,533 @@ program
|
|
|
12274
14153
|
await writeJsonFile(samplesPath, samples);
|
|
12275
14154
|
printOutput(payload);
|
|
12276
14155
|
});
|
|
14156
|
+
program
|
|
14157
|
+
.command("salesnav:accounts:collect")
|
|
14158
|
+
.alias("accounts:collect")
|
|
14159
|
+
.description("Collect an exhaustive Sales Navigator account search locally with a durable checkpoint.")
|
|
14160
|
+
.requiredOption("--query-url <url>", "LinkedIn Sales Navigator account search URL")
|
|
14161
|
+
.requiredOption("--checkpoint <path>", "Checkpoint file used to resume after rate limits or interruptions")
|
|
14162
|
+
.requiredOption("--raw-out <path>", "Line-delimited raw account results used by accounts:local-import")
|
|
14163
|
+
.option("--curl-file <path>", "Path to a saved account-search curl request used as the local request template")
|
|
14164
|
+
.option("--browser-relay-port <number>", "Use the signed-in local browser as Account Search transport through this loopback port")
|
|
14165
|
+
.option("--reported-total <number>", "Known root result total. Skips the initial one-request root count probe.")
|
|
14166
|
+
.option("--max-results-per-search <number>", "Maximum pageable results per final filter slice", String(LOCAL_ACCOUNT_SEARCH_COLLECTOR_RESULT_WINDOW))
|
|
14167
|
+
.option("--page-size <number>", "Accounts requested per Account Search API request", "100")
|
|
14168
|
+
.option("--delay-min-ms <number>", "Minimum randomized delay between Account Search API requests", "5000")
|
|
14169
|
+
.option("--delay-max-ms <number>", "Maximum randomized delay between Account Search API requests", "8000")
|
|
14170
|
+
.option("--max-requests <number>", "Maximum API requests in this invocation, or 'all'", "all")
|
|
14171
|
+
.option("--max-retries <number>", "Retries for transient network/5xx failures. Rate limits stop immediately.", "1")
|
|
14172
|
+
.option("--retry-base-delay-ms <number>", "Base delay for full-jitter retry backoff", "2000")
|
|
14173
|
+
.option("--retry-max-delay-ms <number>", "Maximum delay for full-jitter retry backoff", "30000")
|
|
14174
|
+
.option("--out <path>", "Optional JSON summary output path")
|
|
14175
|
+
.action(async (options) => {
|
|
14176
|
+
const queryUrl = z.string().url().parse(options.queryUrl);
|
|
14177
|
+
const checkpointPath = path.resolve(z.string().min(1).parse(options.checkpoint));
|
|
14178
|
+
const rawOutputPath = path.resolve(z.string().min(1).parse(options.rawOut));
|
|
14179
|
+
const cap = z.coerce
|
|
14180
|
+
.number()
|
|
14181
|
+
.int()
|
|
14182
|
+
.min(1)
|
|
14183
|
+
.max(LOCAL_ACCOUNT_SEARCH_COLLECTOR_RESULT_WINDOW)
|
|
14184
|
+
.parse(options.maxResultsPerSearch);
|
|
14185
|
+
const pageSize = z.coerce
|
|
14186
|
+
.number()
|
|
14187
|
+
.int()
|
|
14188
|
+
.min(1)
|
|
14189
|
+
.max(100)
|
|
14190
|
+
.parse(options.pageSize);
|
|
14191
|
+
const delayMinMs = z.coerce
|
|
14192
|
+
.number()
|
|
14193
|
+
.int()
|
|
14194
|
+
.min(0)
|
|
14195
|
+
.max(300000)
|
|
14196
|
+
.parse(options.delayMinMs);
|
|
14197
|
+
const delayMaxMs = z.coerce
|
|
14198
|
+
.number()
|
|
14199
|
+
.int()
|
|
14200
|
+
.min(0)
|
|
14201
|
+
.max(300000)
|
|
14202
|
+
.parse(options.delayMaxMs);
|
|
14203
|
+
if (delayMaxMs < delayMinMs) {
|
|
14204
|
+
throw new Error("--delay-max-ms must be greater than or equal to --delay-min-ms.");
|
|
14205
|
+
}
|
|
14206
|
+
const maxRequestsOption = String(options.maxRequests ?? "all")
|
|
14207
|
+
.trim()
|
|
14208
|
+
.toLowerCase();
|
|
14209
|
+
const maxRequests = maxRequestsOption === "all"
|
|
14210
|
+
? null
|
|
14211
|
+
: z.coerce.number().int().min(1).max(100000).parse(options.maxRequests);
|
|
14212
|
+
const reportedTotal = options.reportedTotal == null
|
|
14213
|
+
? null
|
|
14214
|
+
: z.coerce
|
|
14215
|
+
.number()
|
|
14216
|
+
.int()
|
|
14217
|
+
.min(1)
|
|
14218
|
+
.max(1_000_000)
|
|
14219
|
+
.parse(options.reportedTotal);
|
|
14220
|
+
const retry = {
|
|
14221
|
+
maxRetries: z.coerce
|
|
14222
|
+
.number()
|
|
14223
|
+
.int()
|
|
14224
|
+
.min(0)
|
|
14225
|
+
.max(5)
|
|
14226
|
+
.parse(options.maxRetries),
|
|
14227
|
+
retryBaseDelayMs: z.coerce
|
|
14228
|
+
.number()
|
|
14229
|
+
.int()
|
|
14230
|
+
.min(0)
|
|
14231
|
+
.max(300000)
|
|
14232
|
+
.parse(options.retryBaseDelayMs),
|
|
14233
|
+
retryMaxDelayMs: z.coerce
|
|
14234
|
+
.number()
|
|
14235
|
+
.int()
|
|
14236
|
+
.min(0)
|
|
14237
|
+
.max(300000)
|
|
14238
|
+
.parse(options.retryMaxDelayMs),
|
|
14239
|
+
};
|
|
14240
|
+
const normalizedSourceQueryUrl = buildSalesNavigatorAccountQuery(queryUrl, []).sourceQueryUrl;
|
|
14241
|
+
let checkpoint;
|
|
14242
|
+
if (await fileExists(checkpointPath)) {
|
|
14243
|
+
checkpoint = parseLocalAccountSearchCollectorCheckpoint(JSON.parse(await readFile(checkpointPath, "utf8")));
|
|
14244
|
+
const normalizedCheckpointUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, []).sourceQueryUrl;
|
|
14245
|
+
if (normalizedCheckpointUrl !== normalizedSourceQueryUrl) {
|
|
14246
|
+
throw new Error("The checkpoint belongs to a different Sales Navigator account search.");
|
|
14247
|
+
}
|
|
14248
|
+
if (checkpoint.pageSize !== pageSize || checkpoint.cap > cap) {
|
|
14249
|
+
throw new Error(`Checkpoint paging does not match this invocation. Use --max-results-per-search ${checkpoint.cap} --page-size ${checkpoint.pageSize}.`);
|
|
14250
|
+
}
|
|
14251
|
+
if (checkpoint.cap < cap) {
|
|
14252
|
+
checkpoint.cap = cap;
|
|
14253
|
+
}
|
|
14254
|
+
checkpoint.delayMinMs = delayMinMs;
|
|
14255
|
+
checkpoint.delayMaxMs = delayMaxMs;
|
|
14256
|
+
if (reportedTotal != null && checkpoint.rootTotal == null) {
|
|
14257
|
+
checkpoint.rootTotal = reportedTotal;
|
|
14258
|
+
}
|
|
14259
|
+
}
|
|
14260
|
+
else {
|
|
14261
|
+
checkpoint = createLocalAccountSearchCollectorCheckpoint({
|
|
14262
|
+
sourceQueryUrl: queryUrl,
|
|
14263
|
+
rootTotal: reportedTotal,
|
|
14264
|
+
cap,
|
|
14265
|
+
pageSize,
|
|
14266
|
+
delayMinMs,
|
|
14267
|
+
delayMaxMs,
|
|
14268
|
+
});
|
|
14269
|
+
}
|
|
14270
|
+
if (migrateLocalAccountSearchCollectorCheckpoint(checkpoint)) {
|
|
14271
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14272
|
+
}
|
|
14273
|
+
const curlFile = typeof options.curlFile === "string" && options.curlFile.trim().length > 0
|
|
14274
|
+
? options.curlFile.trim()
|
|
14275
|
+
: null;
|
|
14276
|
+
const browserRelayPort = options.browserRelayPort == null
|
|
14277
|
+
? null
|
|
14278
|
+
: z.coerce
|
|
14279
|
+
.number()
|
|
14280
|
+
.int()
|
|
14281
|
+
.min(1)
|
|
14282
|
+
.max(65535)
|
|
14283
|
+
.parse(options.browserRelayPort);
|
|
14284
|
+
if (curlFile && browserRelayPort != null) {
|
|
14285
|
+
throw new Error("Use either --curl-file or --browser-relay-port, not both.");
|
|
14286
|
+
}
|
|
14287
|
+
const templateRequest = curlFile
|
|
14288
|
+
? parseSalesNavigatorCurlRequest(await readFile(curlFile, "utf8"))
|
|
14289
|
+
: null;
|
|
14290
|
+
if (templateRequest &&
|
|
14291
|
+
!new URL(templateRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
14292
|
+
throw new Error("Account collection --curl-file must contain a /sales-api/salesApiAccountSearch request.");
|
|
14293
|
+
}
|
|
14294
|
+
const config = templateRequest || browserRelayPort != null
|
|
14295
|
+
? null
|
|
14296
|
+
: await readLinkedInAccountSearchConfig(queryUrl);
|
|
14297
|
+
const browserRelay = browserRelayPort == null
|
|
14298
|
+
? null
|
|
14299
|
+
: await createLocalAccountSearchBrowserRelay(browserRelayPort);
|
|
14300
|
+
const buildRequest = (searchUrl, start, count) => {
|
|
14301
|
+
const request = browserRelay
|
|
14302
|
+
? {
|
|
14303
|
+
url: buildSalesNavigatorAccountApiUrlFromSearchUrl(searchUrl, pageSize),
|
|
14304
|
+
headers: {},
|
|
14305
|
+
}
|
|
14306
|
+
: templateRequest
|
|
14307
|
+
? buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(searchUrl, templateRequest, pageSize)
|
|
14308
|
+
: buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, pageSize);
|
|
14309
|
+
return {
|
|
14310
|
+
...request,
|
|
14311
|
+
url: withSalesNavigatorPaging(request.url, start, count),
|
|
14312
|
+
};
|
|
14313
|
+
};
|
|
14314
|
+
const executeRequest = (request) => browserRelay
|
|
14315
|
+
? browserRelay.request(request.url)
|
|
14316
|
+
: fetchLocalSalesNavigatorRequest(request, retry);
|
|
14317
|
+
try {
|
|
14318
|
+
let requestsThisRun = 0;
|
|
14319
|
+
let delayThisRunMs = 0;
|
|
14320
|
+
const distinctAccountKeys = await loadLocalAccountSearchRawAccountKeys(rawOutputPath);
|
|
14321
|
+
checkpoint.distinctAccounts = distinctAccountKeys.size;
|
|
14322
|
+
checkpoint.completedByDistinctCount =
|
|
14323
|
+
checkpoint.rootTotal != null &&
|
|
14324
|
+
distinctAccountKeys.size >= checkpoint.rootTotal;
|
|
14325
|
+
if (checkpoint.completedByDistinctCount) {
|
|
14326
|
+
checkpoint.taskQueue = [];
|
|
14327
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14328
|
+
}
|
|
14329
|
+
const requestBudgetAvailable = () => maxRequests == null || requestsThisRun < maxRequests;
|
|
14330
|
+
const stopForRateLimit = async (error, task) => {
|
|
14331
|
+
checkpoint.rateLimited = {
|
|
14332
|
+
at: new Date().toISOString(),
|
|
14333
|
+
retryAfter: error.retryAfterMs,
|
|
14334
|
+
status: error.status,
|
|
14335
|
+
task,
|
|
14336
|
+
};
|
|
14337
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14338
|
+
};
|
|
14339
|
+
if (checkpoint.rootTotal == null && requestBudgetAvailable()) {
|
|
14340
|
+
try {
|
|
14341
|
+
const rootResponse = await executeRequest(buildRequest(queryUrl, 0, 1));
|
|
14342
|
+
requestsThisRun += 1;
|
|
14343
|
+
checkpoint.requests += 1;
|
|
14344
|
+
checkpoint.rootTotal = extractLocalSalesNavigatorTotalResults(rootResponse.body);
|
|
14345
|
+
checkpoint.rateLimited = null;
|
|
14346
|
+
if (checkpoint.rootTotal == null) {
|
|
14347
|
+
throw new Error("Sales Navigator Account Search root response did not include a result total.");
|
|
14348
|
+
}
|
|
14349
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14350
|
+
}
|
|
14351
|
+
catch (error) {
|
|
14352
|
+
if (error instanceof LocalSalesNavigatorRateLimitError) {
|
|
14353
|
+
await stopForRateLimit(error, { kind: "root" });
|
|
14354
|
+
const payload = {
|
|
14355
|
+
status: "rate_limited",
|
|
14356
|
+
mode: "account-local-collector",
|
|
14357
|
+
checkpointPath,
|
|
14358
|
+
rawOutputPath,
|
|
14359
|
+
requestsThisRun,
|
|
14360
|
+
requests: checkpoint.requests,
|
|
14361
|
+
remainingTasks: checkpoint.taskQueue.length,
|
|
14362
|
+
rateLimited: checkpoint.rateLimited,
|
|
14363
|
+
};
|
|
14364
|
+
if (options.out) {
|
|
14365
|
+
await writeJsonFile(options.out, payload);
|
|
14366
|
+
}
|
|
14367
|
+
printOutput(payload);
|
|
14368
|
+
return;
|
|
14369
|
+
}
|
|
14370
|
+
throw error;
|
|
14371
|
+
}
|
|
14372
|
+
}
|
|
14373
|
+
if (checkpoint.rootTotal != null &&
|
|
14374
|
+
checkpoint.taskQueue.length === 0 &&
|
|
14375
|
+
checkpoint.seenQueries.length === 0) {
|
|
14376
|
+
seedLocalAccountSearchCollectorTasks(checkpoint);
|
|
14377
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14378
|
+
}
|
|
14379
|
+
const dimensions = new Map([
|
|
14380
|
+
...DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS,
|
|
14381
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_GROWTH_DIMENSION,
|
|
14382
|
+
].map((dimension) => [dimension.filterType, dimension]));
|
|
14383
|
+
const seenQueries = new Set(checkpoint.seenQueries);
|
|
14384
|
+
let stoppedByRateLimit = false;
|
|
14385
|
+
while (checkpoint.taskQueue.length > 0 && requestBudgetAvailable()) {
|
|
14386
|
+
const task = checkpoint.taskQueue[0];
|
|
14387
|
+
const searchUrl = task.kind === "node"
|
|
14388
|
+
? task.searchUrlOverride ??
|
|
14389
|
+
buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, task.filters).slicedQueryUrl
|
|
14390
|
+
: task.searchUrl;
|
|
14391
|
+
const start = task.kind === "page" ? task.start : 0;
|
|
14392
|
+
let response;
|
|
14393
|
+
try {
|
|
14394
|
+
response = await executeRequest(buildRequest(searchUrl, start, pageSize));
|
|
14395
|
+
}
|
|
14396
|
+
catch (error) {
|
|
14397
|
+
if (error instanceof LocalSalesNavigatorRateLimitError) {
|
|
14398
|
+
stoppedByRateLimit = true;
|
|
14399
|
+
await stopForRateLimit(error, task);
|
|
14400
|
+
break;
|
|
14401
|
+
}
|
|
14402
|
+
checkpoint.failures.push({
|
|
14403
|
+
at: new Date().toISOString(),
|
|
14404
|
+
task,
|
|
14405
|
+
error: error instanceof Error ? error.message : String(error),
|
|
14406
|
+
});
|
|
14407
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14408
|
+
throw error;
|
|
14409
|
+
}
|
|
14410
|
+
requestsThisRun += 1;
|
|
14411
|
+
checkpoint.requests += 1;
|
|
14412
|
+
checkpoint.rateLimited = null;
|
|
14413
|
+
const elements = extractLocalSalesNavigatorElements(response.body);
|
|
14414
|
+
await appendLocalAccountSearchRawElements({
|
|
14415
|
+
rawOutputPath,
|
|
14416
|
+
searchUrl,
|
|
14417
|
+
elements,
|
|
14418
|
+
});
|
|
14419
|
+
for (const element of elements) {
|
|
14420
|
+
addLocalAccountSearchElementKey(distinctAccountKeys, element, searchUrl);
|
|
14421
|
+
}
|
|
14422
|
+
checkpoint.distinctAccounts = distinctAccountKeys.size;
|
|
14423
|
+
checkpoint.pagesWritten += 1;
|
|
14424
|
+
checkpoint.rowsWritten += elements.length;
|
|
14425
|
+
if (checkpoint.rootTotal != null &&
|
|
14426
|
+
distinctAccountKeys.size >= checkpoint.rootTotal) {
|
|
14427
|
+
checkpoint.taskQueue = [];
|
|
14428
|
+
checkpoint.completedByDistinctCount = true;
|
|
14429
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14430
|
+
break;
|
|
14431
|
+
}
|
|
14432
|
+
if (task.kind === "node") {
|
|
14433
|
+
const total = extractLocalSalesNavigatorTotalResults(response.body);
|
|
14434
|
+
if (total == null) {
|
|
14435
|
+
checkpoint.failures.push({
|
|
14436
|
+
at: new Date().toISOString(),
|
|
14437
|
+
task,
|
|
14438
|
+
error: "Account Search response did not include a result total.",
|
|
14439
|
+
});
|
|
14440
|
+
}
|
|
14441
|
+
else if (total <= cap) {
|
|
14442
|
+
checkpoint.leaves.push({
|
|
14443
|
+
id: task.id,
|
|
14444
|
+
pass: task.pass,
|
|
14445
|
+
total,
|
|
14446
|
+
searchUrl,
|
|
14447
|
+
filters: task.filters,
|
|
14448
|
+
});
|
|
14449
|
+
const pageTasks = [];
|
|
14450
|
+
for (let pageStart = pageSize; pageStart < total; pageStart += pageSize) {
|
|
14451
|
+
pageTasks.push({
|
|
14452
|
+
kind: "page",
|
|
14453
|
+
id: task.id,
|
|
14454
|
+
pass: task.pass,
|
|
14455
|
+
searchUrl,
|
|
14456
|
+
start: pageStart,
|
|
14457
|
+
total,
|
|
14458
|
+
});
|
|
14459
|
+
}
|
|
14460
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, pageTasks);
|
|
14461
|
+
}
|
|
14462
|
+
else {
|
|
14463
|
+
const [nextFilterType, ...remainingDimensions] = task.remainingDimensions;
|
|
14464
|
+
const dimension = nextFilterType
|
|
14465
|
+
? dimensions.get(nextFilterType)
|
|
14466
|
+
: null;
|
|
14467
|
+
if (!dimension) {
|
|
14468
|
+
const bayAreaPartitionPlan = buildLocalAccountSearchCollectorBayAreaPartitions(checkpoint.sourceQueryUrl, task.filters);
|
|
14469
|
+
if (!bayAreaPartitionPlan) {
|
|
14470
|
+
const industryPartitionPlan = task.pass.startsWith("coverage-")
|
|
14471
|
+
? buildLocalAccountSearchCollectorIndustryPartitions(searchUrl)
|
|
14472
|
+
: null;
|
|
14473
|
+
if (industryPartitionPlan) {
|
|
14474
|
+
checkpoint.windows.push({
|
|
14475
|
+
id: task.id,
|
|
14476
|
+
pass: task.pass,
|
|
14477
|
+
total,
|
|
14478
|
+
searchUrl,
|
|
14479
|
+
splitOn: industryPartitionPlan.splitOn,
|
|
14480
|
+
});
|
|
14481
|
+
const industryTasks = [];
|
|
14482
|
+
for (const partition of industryPartitionPlan.partitions) {
|
|
14483
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, partition.searchUrl);
|
|
14484
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14485
|
+
checkpoint.duplicateQueries += 1;
|
|
14486
|
+
continue;
|
|
14487
|
+
}
|
|
14488
|
+
seenQueries.add(childQueryKey);
|
|
14489
|
+
industryTasks.push({
|
|
14490
|
+
kind: "node",
|
|
14491
|
+
id: `${task.id}-${industryPartitionPlan.splitOn}-${partition.idSuffix}`,
|
|
14492
|
+
pass: task.pass,
|
|
14493
|
+
filters: task.filters.map((filter) => ({
|
|
14494
|
+
...filter,
|
|
14495
|
+
values: filter.values.map((value) => ({ ...value })),
|
|
14496
|
+
})),
|
|
14497
|
+
remainingDimensions: [],
|
|
14498
|
+
searchUrlOverride: partition.searchUrl,
|
|
14499
|
+
keywordClauses: task.keywordClauses
|
|
14500
|
+
? [...task.keywordClauses]
|
|
14501
|
+
: undefined,
|
|
14502
|
+
keywordDepth: task.keywordDepth,
|
|
14503
|
+
});
|
|
14504
|
+
}
|
|
14505
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, industryTasks);
|
|
14506
|
+
}
|
|
14507
|
+
else {
|
|
14508
|
+
const keywordPartitionPlan = buildLocalAccountSearchCollectorKeywordPartitions(searchUrl, task);
|
|
14509
|
+
if (!keywordPartitionPlan) {
|
|
14510
|
+
const terminalPageLimit = Math.min(total, cap);
|
|
14511
|
+
const pageTasks = [];
|
|
14512
|
+
for (let pageStart = pageSize; pageStart < terminalPageLimit; pageStart += pageSize) {
|
|
14513
|
+
pageTasks.push({
|
|
14514
|
+
kind: "page",
|
|
14515
|
+
id: task.id,
|
|
14516
|
+
pass: task.pass,
|
|
14517
|
+
searchUrl,
|
|
14518
|
+
start: pageStart,
|
|
14519
|
+
total,
|
|
14520
|
+
});
|
|
14521
|
+
}
|
|
14522
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, pageTasks);
|
|
14523
|
+
checkpoint.failures.push({
|
|
14524
|
+
at: new Date().toISOString(),
|
|
14525
|
+
task,
|
|
14526
|
+
total,
|
|
14527
|
+
collectedResultWindow: terminalPageLimit,
|
|
14528
|
+
missingBeyondResultWindow: Math.max(0, total - terminalPageLimit),
|
|
14529
|
+
error: `Account Search slice remains above ${cap} with no split dimension left.`,
|
|
14530
|
+
});
|
|
14531
|
+
}
|
|
14532
|
+
else {
|
|
14533
|
+
checkpoint.windows.push({
|
|
14534
|
+
id: task.id,
|
|
14535
|
+
pass: task.pass,
|
|
14536
|
+
total,
|
|
14537
|
+
searchUrl,
|
|
14538
|
+
splitOn: keywordPartitionPlan.splitOn,
|
|
14539
|
+
});
|
|
14540
|
+
const keywordTasks = [];
|
|
14541
|
+
for (const partition of keywordPartitionPlan.partitions) {
|
|
14542
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, partition.searchUrl);
|
|
14543
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14544
|
+
checkpoint.duplicateQueries += 1;
|
|
14545
|
+
continue;
|
|
14546
|
+
}
|
|
14547
|
+
seenQueries.add(childQueryKey);
|
|
14548
|
+
keywordTasks.push({
|
|
14549
|
+
kind: "node",
|
|
14550
|
+
id: `${task.id}-${keywordPartitionPlan.splitOn}-${partition.idSuffix}`,
|
|
14551
|
+
pass: task.pass,
|
|
14552
|
+
filters: task.filters.map((filter) => ({
|
|
14553
|
+
...filter,
|
|
14554
|
+
values: filter.values.map((value) => ({ ...value })),
|
|
14555
|
+
})),
|
|
14556
|
+
remainingDimensions: [],
|
|
14557
|
+
searchUrlOverride: partition.searchUrl,
|
|
14558
|
+
keywordClauses: partition.keywordClauses,
|
|
14559
|
+
keywordDepth: partition.keywordDepth,
|
|
14560
|
+
});
|
|
14561
|
+
}
|
|
14562
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, keywordTasks);
|
|
14563
|
+
}
|
|
14564
|
+
}
|
|
14565
|
+
}
|
|
14566
|
+
else {
|
|
14567
|
+
checkpoint.windows.push({
|
|
14568
|
+
id: task.id,
|
|
14569
|
+
pass: task.pass,
|
|
14570
|
+
total,
|
|
14571
|
+
searchUrl,
|
|
14572
|
+
splitOn: bayAreaPartitionPlan.splitOn,
|
|
14573
|
+
});
|
|
14574
|
+
const regionTasks = [];
|
|
14575
|
+
bayAreaPartitionPlan.partitions.forEach((filters, index) => {
|
|
14576
|
+
const childSearchUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, filters).slicedQueryUrl;
|
|
14577
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, childSearchUrl);
|
|
14578
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14579
|
+
checkpoint.duplicateQueries += 1;
|
|
14580
|
+
return;
|
|
14581
|
+
}
|
|
14582
|
+
seenQueries.add(childQueryKey);
|
|
14583
|
+
regionTasks.push({
|
|
14584
|
+
kind: "node",
|
|
14585
|
+
id: `${task.id}-${bayAreaPartitionPlan.splitOn}-${index}`,
|
|
14586
|
+
pass: task.pass,
|
|
14587
|
+
filters,
|
|
14588
|
+
remainingDimensions: [],
|
|
14589
|
+
});
|
|
14590
|
+
});
|
|
14591
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, regionTasks);
|
|
14592
|
+
}
|
|
14593
|
+
}
|
|
14594
|
+
else {
|
|
14595
|
+
checkpoint.windows.push({
|
|
14596
|
+
id: task.id,
|
|
14597
|
+
pass: task.pass,
|
|
14598
|
+
total,
|
|
14599
|
+
searchUrl,
|
|
14600
|
+
splitOn: dimension.filterType,
|
|
14601
|
+
});
|
|
14602
|
+
const dimensionTasks = [];
|
|
14603
|
+
buildLocalAccountSearchCollectorDimensionPartitions(dimension).forEach((partition) => {
|
|
14604
|
+
const filters = [
|
|
14605
|
+
...task.filters,
|
|
14606
|
+
partition.filter,
|
|
14607
|
+
];
|
|
14608
|
+
const childSearchUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, filters).slicedQueryUrl;
|
|
14609
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, childSearchUrl);
|
|
14610
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14611
|
+
checkpoint.duplicateQueries += 1;
|
|
14612
|
+
return;
|
|
14613
|
+
}
|
|
14614
|
+
seenQueries.add(childQueryKey);
|
|
14615
|
+
dimensionTasks.push({
|
|
14616
|
+
kind: "node",
|
|
14617
|
+
id: `${task.id}-${dimension.filterType}-${partition.idSuffix}`,
|
|
14618
|
+
pass: task.pass,
|
|
14619
|
+
filters,
|
|
14620
|
+
remainingDimensions,
|
|
14621
|
+
});
|
|
14622
|
+
});
|
|
14623
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, dimensionTasks);
|
|
14624
|
+
}
|
|
14625
|
+
}
|
|
14626
|
+
}
|
|
14627
|
+
checkpoint.taskQueue.shift();
|
|
14628
|
+
checkpoint.seenQueries = [...seenQueries];
|
|
14629
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14630
|
+
if (checkpoint.taskQueue.length > 0 && requestBudgetAvailable()) {
|
|
14631
|
+
const delayMs = randomIntegerBetween(delayMinMs, delayMaxMs);
|
|
14632
|
+
if (delayMs > 0) {
|
|
14633
|
+
delayThisRunMs += delayMs;
|
|
14634
|
+
await delay(delayMs);
|
|
14635
|
+
}
|
|
14636
|
+
}
|
|
14637
|
+
}
|
|
14638
|
+
const payload = {
|
|
14639
|
+
status: stoppedByRateLimit
|
|
14640
|
+
? "rate_limited"
|
|
14641
|
+
: checkpoint.taskQueue.length === 0
|
|
14642
|
+
? "complete"
|
|
14643
|
+
: "checkpointed",
|
|
14644
|
+
mode: "account-local-collector",
|
|
14645
|
+
sourceQueryUrl: checkpoint.sourceQueryUrl,
|
|
14646
|
+
rootTotal: checkpoint.rootTotal,
|
|
14647
|
+
checkpointPath,
|
|
14648
|
+
rawOutputPath,
|
|
14649
|
+
requestsThisRun,
|
|
14650
|
+
delayThisRunMs,
|
|
14651
|
+
requests: checkpoint.requests,
|
|
14652
|
+
pagesWritten: checkpoint.pagesWritten,
|
|
14653
|
+
rowsWritten: checkpoint.rowsWritten,
|
|
14654
|
+
leaves: checkpoint.leaves.length,
|
|
14655
|
+
windows: checkpoint.windows.length,
|
|
14656
|
+
failures: checkpoint.failures.length,
|
|
14657
|
+
remainingTasks: checkpoint.taskQueue.length,
|
|
14658
|
+
distinctAccounts: distinctAccountKeys.size,
|
|
14659
|
+
completedByDistinctCount: checkpoint.completedByDistinctCount === true,
|
|
14660
|
+
rateLimited: checkpoint.rateLimited,
|
|
14661
|
+
browserRelayPort: browserRelay?.port ?? null,
|
|
14662
|
+
};
|
|
14663
|
+
if (options.out) {
|
|
14664
|
+
await writeJsonFile(options.out, payload);
|
|
14665
|
+
}
|
|
14666
|
+
printOutput(payload);
|
|
14667
|
+
}
|
|
14668
|
+
finally {
|
|
14669
|
+
await browserRelay?.close();
|
|
14670
|
+
}
|
|
14671
|
+
});
|
|
12277
14672
|
program
|
|
12278
14673
|
.command("salesnav:accounts:split")
|
|
12279
14674
|
.alias("accounts:split")
|
|
12280
14675
|
.description("Split a Sales Navigator account search into account slices below the result cap.")
|
|
12281
14676
|
.requiredOption("--query-url <url>", "LinkedIn Sales Navigator account search URL")
|
|
12282
14677
|
.option("--curl-file <path>", "Path to a saved account-search curl request used as the local request template")
|
|
12283
|
-
.option("--org-id <id>", "Salesprompter workspace org id for
|
|
14678
|
+
.option("--org-id <id>", "Salesprompter workspace org id for Supabase storage. Defaults to SALESPROMPTER_ORG_ID or saved auth session org.")
|
|
12284
14679
|
.option("--max-results-per-search <number>", "Maximum account results allowed for each final slice", "1000")
|
|
12285
14680
|
.option("--max-windowed-final-results <number>", "Bounded fallback fetch for an oversized final account slice when no split dimensions remain", "1000")
|
|
12286
14681
|
.option("--no-windowed-final-slices", "Do not fetch pageable rows for oversized final account slices after adaptive splitting is exhausted")
|
|
14682
|
+
.option("--include-location-mismatches", "Store account rows even when the visible Sales Navigator account location does not match the source REGION filter", false)
|
|
12287
14683
|
.option("--max-split-depth <number>", "Maximum number of adaptive account split dimensions to use", "3")
|
|
12288
14684
|
.option("--max-slices <number>", "Safety cap for total account slices probed in this invocation", "1000")
|
|
12289
14685
|
.option("--probe-count <number>", "Accounts to request while probing each slice count", "1")
|
|
@@ -12389,6 +14785,7 @@ program
|
|
|
12389
14785
|
.max(300000)
|
|
12390
14786
|
.parse(options.retryMaxDelayMs),
|
|
12391
14787
|
};
|
|
14788
|
+
const filterLocationMismatches = !Boolean(options.includeLocationMismatches);
|
|
12392
14789
|
const curlFile = typeof options.curlFile === "string" && options.curlFile.trim().length > 0
|
|
12393
14790
|
? options.curlFile.trim()
|
|
12394
14791
|
: null;
|
|
@@ -12449,7 +14846,9 @@ program
|
|
|
12449
14846
|
!new URL(templateRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
12450
14847
|
throw new Error("Account split --curl-file must contain a /sales-api/salesApiAccountSearch request.");
|
|
12451
14848
|
}
|
|
12452
|
-
const config = templateRequest
|
|
14849
|
+
const config = templateRequest
|
|
14850
|
+
? null
|
|
14851
|
+
: await readLinkedInAccountSearchConfig(queryUrl);
|
|
12453
14852
|
const buildAccountApiRequest = (slicedQueryUrl) => templateRequest
|
|
12454
14853
|
? buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(slicedQueryUrl, templateRequest, pageSize)
|
|
12455
14854
|
: buildSalesNavigatorAccountApiRequestFromSearchUrl(slicedQueryUrl, config, pageSize);
|
|
@@ -12602,6 +15001,9 @@ program
|
|
|
12602
15001
|
},
|
|
12603
15002
|
exportSlice: async (attempt) => {
|
|
12604
15003
|
const parsedRequest = buildAccountApiRequest(attempt.slicedQueryUrl);
|
|
15004
|
+
const accountLocationFilter = filterLocationMismatches
|
|
15005
|
+
? buildSalesNavigatorAccountLocationFilter(attempt.slicedQueryUrl)
|
|
15006
|
+
: null;
|
|
12605
15007
|
try {
|
|
12606
15008
|
const fetchResult = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
12607
15009
|
requestedAccounts,
|
|
@@ -12610,6 +15012,7 @@ program
|
|
|
12610
15012
|
pageDelayMaxMs,
|
|
12611
15013
|
maxAllowedResults: attempt.maxResultsPerSearch,
|
|
12612
15014
|
retry,
|
|
15015
|
+
accountLocationFilter,
|
|
12613
15016
|
});
|
|
12614
15017
|
const stop = fetchResult.pacing.stop;
|
|
12615
15018
|
const partialError = describeLocalSalesNavigatorAccountPartialError(fetchResult);
|
|
@@ -12630,6 +15033,8 @@ program
|
|
|
12630
15033
|
slicedQueryUrl: attempt.slicedQueryUrl,
|
|
12631
15034
|
pacing: fetchResult.pacing,
|
|
12632
15035
|
stop,
|
|
15036
|
+
accountLocationFilter,
|
|
15037
|
+
filteredAccounts: fetchResult.filteredAccounts,
|
|
12633
15038
|
},
|
|
12634
15039
|
});
|
|
12635
15040
|
const storedAccounts = await upsertSalesNavigatorAccounts({
|
|
@@ -12639,7 +15044,10 @@ program
|
|
|
12639
15044
|
sliceId,
|
|
12640
15045
|
accounts: fetchResult.accounts,
|
|
12641
15046
|
});
|
|
12642
|
-
storedAccountCount
|
|
15047
|
+
storedAccountCount = await countSalesNavigatorAccountsForRun({
|
|
15048
|
+
store,
|
|
15049
|
+
runId,
|
|
15050
|
+
});
|
|
12643
15051
|
if (fetchResult.pacing.rateLimit) {
|
|
12644
15052
|
throw new SalesNavigatorCrawlStopError(fetchResult.pacing.rateLimit.message, {
|
|
12645
15053
|
totalResults: fetchResult.totalResults,
|
|
@@ -12739,6 +15147,9 @@ program
|
|
|
12739
15147
|
if (canFetchWindowedFinalSlice) {
|
|
12740
15148
|
try {
|
|
12741
15149
|
const parsedRequest = buildAccountApiRequest(failure.attempt.slicedQueryUrl);
|
|
15150
|
+
const accountLocationFilter = filterLocationMismatches
|
|
15151
|
+
? buildSalesNavigatorAccountLocationFilter(failure.attempt.slicedQueryUrl)
|
|
15152
|
+
: null;
|
|
12742
15153
|
const fetchResult = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
12743
15154
|
requestedAccounts,
|
|
12744
15155
|
pageSize,
|
|
@@ -12747,6 +15158,7 @@ program
|
|
|
12747
15158
|
maxAllowedResults: maxWindowedFinalResults,
|
|
12748
15159
|
retry,
|
|
12749
15160
|
allowPartialPageErrors: true,
|
|
15161
|
+
accountLocationFilter,
|
|
12750
15162
|
});
|
|
12751
15163
|
const windowedError = describeLocalSalesNavigatorAccountPartialError(fetchResult);
|
|
12752
15164
|
const sliceId = await upsertSalesNavigatorAccountSearchSlice({
|
|
@@ -12768,6 +15180,8 @@ program
|
|
|
12768
15180
|
originalError: failure.error,
|
|
12769
15181
|
pacing: fetchResult.pacing,
|
|
12770
15182
|
stop: fetchResult.pacing.stop,
|
|
15183
|
+
accountLocationFilter,
|
|
15184
|
+
filteredAccounts: fetchResult.filteredAccounts,
|
|
12771
15185
|
},
|
|
12772
15186
|
});
|
|
12773
15187
|
const storedAccounts = await upsertSalesNavigatorAccounts({
|
|
@@ -12777,7 +15191,10 @@ program
|
|
|
12777
15191
|
sliceId,
|
|
12778
15192
|
accounts: fetchResult.accounts,
|
|
12779
15193
|
});
|
|
12780
|
-
storedAccountCount
|
|
15194
|
+
storedAccountCount = await countSalesNavigatorAccountsForRun({
|
|
15195
|
+
store,
|
|
15196
|
+
runId,
|
|
15197
|
+
});
|
|
12781
15198
|
windowedFinalSlices.push({
|
|
12782
15199
|
slicedQueryUrl: failure.attempt.slicedQueryUrl,
|
|
12783
15200
|
appliedFilters: failure.attempt.appliedFilters,
|
|
@@ -13020,10 +15437,12 @@ program
|
|
|
13020
15437
|
program
|
|
13021
15438
|
.command("salesnav:accounts:local-import")
|
|
13022
15439
|
.alias("accounts:local-import")
|
|
13023
|
-
.description("Run a saved Sales Navigator account request locally and store the accounts in
|
|
15440
|
+
.description("Run a saved Sales Navigator account request locally and store the accounts in Supabase.")
|
|
13024
15441
|
.option("--curl-file <path>", "Path to a saved account-search curl request copied from the Sales Navigator network panel")
|
|
13025
15442
|
.option("--query-url <url>", "Sales Navigator account search URL. Uses the locally captured extension session automatically.")
|
|
13026
|
-
.option("--
|
|
15443
|
+
.option("--raw-results-file <path>", "Line-delimited raw Sales Navigator account results collected for --query-url")
|
|
15444
|
+
.option("--reported-total <number>", "Reported total for completeness verification when using --raw-results-file")
|
|
15445
|
+
.option("--org-id <id>", "Salesprompter workspace org id for Supabase storage. Defaults to SALESPROMPTER_ORG_ID or saved auth session org.")
|
|
13027
15446
|
.option("--max-results-per-search <number>", "Maximum expected account results for this local request", "1000")
|
|
13028
15447
|
.option("--number-of-accounts <number>", "Total accounts to import, or 'all' for the full reported result set", "all")
|
|
13029
15448
|
.option("--page-size <number>", "Accounts requested per local Sales Navigator API page", "25")
|
|
@@ -13035,8 +15454,9 @@ program
|
|
|
13035
15454
|
.option("--retry-max-delay-ms <number>", "Maximum delay for full-jitter retry backoff", "30000")
|
|
13036
15455
|
.option("--slice-preset <name>", "Slice preset label stored with the import run", "local-account-search")
|
|
13037
15456
|
.option("--allow-partial-page-errors", "Persist rows fetched before a later non-rate-limit page error", false)
|
|
15457
|
+
.option("--include-location-mismatches", "Store account rows even when the visible Sales Navigator account location does not match the source REGION filter", false)
|
|
13038
15458
|
.option("--out <path>", "Optional local JSON output path")
|
|
13039
|
-
.option("--dry-run", "Run and normalize the local account request without writing to
|
|
15459
|
+
.option("--dry-run", "Run and normalize the local account request without writing to Supabase", false)
|
|
13040
15460
|
.action(async (options) => {
|
|
13041
15461
|
const maxResultsPerSearch = z.coerce
|
|
13042
15462
|
.number()
|
|
@@ -13102,31 +15522,76 @@ program
|
|
|
13102
15522
|
.max(300000)
|
|
13103
15523
|
.parse(options.retryMaxDelayMs),
|
|
13104
15524
|
};
|
|
15525
|
+
const filterLocationMismatches = !Boolean(options.includeLocationMismatches);
|
|
13105
15526
|
const curlFile = typeof options.curlFile === "string" && options.curlFile.trim().length > 0
|
|
13106
15527
|
? options.curlFile.trim()
|
|
13107
15528
|
: null;
|
|
13108
15529
|
const queryUrl = typeof options.queryUrl === "string" && options.queryUrl.trim().length > 0
|
|
13109
15530
|
? z.string().url().parse(options.queryUrl.trim())
|
|
13110
15531
|
: null;
|
|
13111
|
-
|
|
13112
|
-
|
|
15532
|
+
const rawResultsFile = typeof options.rawResultsFile === "string" &&
|
|
15533
|
+
options.rawResultsFile.trim().length > 0
|
|
15534
|
+
? options.rawResultsFile.trim()
|
|
15535
|
+
: null;
|
|
15536
|
+
const reportedTotal = options.reportedTotal == null
|
|
15537
|
+
? null
|
|
15538
|
+
: z.coerce
|
|
15539
|
+
.number()
|
|
15540
|
+
.int()
|
|
15541
|
+
.min(1)
|
|
15542
|
+
.max(1_000_000)
|
|
15543
|
+
.parse(options.reportedTotal);
|
|
15544
|
+
if (rawResultsFile) {
|
|
15545
|
+
if (curlFile || !queryUrl) {
|
|
15546
|
+
throw new Error("--raw-results-file requires --query-url and cannot be combined with --curl-file.");
|
|
15547
|
+
}
|
|
15548
|
+
if (numberOfAccountsOption !== "all") {
|
|
15549
|
+
throw new Error("--raw-results-file requires --number-of-accounts all so completeness can be verified.");
|
|
15550
|
+
}
|
|
13113
15551
|
}
|
|
13114
|
-
|
|
13115
|
-
|
|
13116
|
-
|
|
13117
|
-
if (
|
|
13118
|
-
throw new Error("
|
|
15552
|
+
else if (Boolean(curlFile) === Boolean(queryUrl)) {
|
|
15553
|
+
throw new Error("Provide exactly one of --curl-file or --query-url, or use --raw-results-file with --query-url.");
|
|
15554
|
+
}
|
|
15555
|
+
else if (reportedTotal != null) {
|
|
15556
|
+
throw new Error("--reported-total is only supported with --raw-results-file.");
|
|
15557
|
+
}
|
|
15558
|
+
let accountLocationFilter = filterLocationMismatches
|
|
15559
|
+
? buildSalesNavigatorAccountLocationFilter(queryUrl ?? "")
|
|
15560
|
+
: null;
|
|
15561
|
+
let rawInputStats = null;
|
|
15562
|
+
let localFetch;
|
|
15563
|
+
if (rawResultsFile) {
|
|
15564
|
+
const rawInput = await readLocalSalesNavigatorRawAccountInput({
|
|
15565
|
+
filePath: rawResultsFile,
|
|
15566
|
+
sourceQueryUrl: queryUrl,
|
|
15567
|
+
reportedTotal,
|
|
15568
|
+
accountLocationFilter,
|
|
15569
|
+
});
|
|
15570
|
+
rawInputStats = rawInput.stats;
|
|
15571
|
+
localFetch = rawInput.fetchResult;
|
|
15572
|
+
}
|
|
15573
|
+
else {
|
|
15574
|
+
const parsedRequest = curlFile
|
|
15575
|
+
? parseSalesNavigatorCurlRequest(await readFile(curlFile, "utf8"))
|
|
15576
|
+
: buildSalesNavigatorAccountApiRequestFromSearchUrl(queryUrl, await readLinkedInAccountSearchConfig(queryUrl), pageSize);
|
|
15577
|
+
if (!new URL(parsedRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
15578
|
+
throw new Error("Account local import requires a /sales-api/salesApiAccountSearch request.");
|
|
15579
|
+
}
|
|
15580
|
+
if (filterLocationMismatches && !accountLocationFilter) {
|
|
15581
|
+
accountLocationFilter = buildSalesNavigatorAccountLocationFilter(queryUrl ?? parsedRequest.url);
|
|
15582
|
+
}
|
|
15583
|
+
localFetch = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
15584
|
+
requestedAccounts,
|
|
15585
|
+
startOffset,
|
|
15586
|
+
pageSize,
|
|
15587
|
+
pageDelayMinMs,
|
|
15588
|
+
pageDelayMaxMs,
|
|
15589
|
+
maxAllowedResults: maxResultsPerSearch,
|
|
15590
|
+
retry,
|
|
15591
|
+
allowPartialPageErrors: Boolean(options.allowPartialPageErrors),
|
|
15592
|
+
accountLocationFilter,
|
|
15593
|
+
});
|
|
13119
15594
|
}
|
|
13120
|
-
const localFetch = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
13121
|
-
requestedAccounts,
|
|
13122
|
-
startOffset,
|
|
13123
|
-
pageSize,
|
|
13124
|
-
pageDelayMinMs,
|
|
13125
|
-
pageDelayMaxMs,
|
|
13126
|
-
maxAllowedResults: maxResultsPerSearch,
|
|
13127
|
-
retry,
|
|
13128
|
-
allowPartialPageErrors: Boolean(options.allowPartialPageErrors),
|
|
13129
|
-
});
|
|
13130
15595
|
const accounts = localFetch.accounts;
|
|
13131
15596
|
const totalResults = localFetch.totalResults;
|
|
13132
15597
|
const dryRun = Boolean(options.dryRun);
|
|
@@ -13136,6 +15601,9 @@ program
|
|
|
13136
15601
|
if (accounts.length === 0) {
|
|
13137
15602
|
throw new Error("The local Sales Navigator account response did not contain any importable account rows.");
|
|
13138
15603
|
}
|
|
15604
|
+
if (rawResultsFile && partialError) {
|
|
15605
|
+
throw new Error(`Raw Sales Navigator account input is incomplete and was not stored. ${partialError}`);
|
|
15606
|
+
}
|
|
13139
15607
|
const sourceQueryUrl = queryUrl ?? localFetch.sourceQueryUrl;
|
|
13140
15608
|
const basePayload = {
|
|
13141
15609
|
status: "ok",
|
|
@@ -13144,6 +15612,7 @@ program
|
|
|
13144
15612
|
sourceQueryUrl,
|
|
13145
15613
|
totalResults,
|
|
13146
15614
|
discovered: accounts.length,
|
|
15615
|
+
filteredAccounts: localFetch.filteredAccounts,
|
|
13147
15616
|
fetchedPages: localFetch.fetchedPages,
|
|
13148
15617
|
requestedAccounts: requestedAccounts ?? "all",
|
|
13149
15618
|
maxResultsPerSearch,
|
|
@@ -13153,7 +15622,8 @@ program
|
|
|
13153
15622
|
rateLimit: rateLimit ?? null,
|
|
13154
15623
|
pageError: pageError ?? null,
|
|
13155
15624
|
stop: localFetch.pacing.stop,
|
|
13156
|
-
|
|
15625
|
+
rawInput: rawInputStats,
|
|
15626
|
+
accounts: rawResultsFile && !dryRun ? undefined : accounts,
|
|
13157
15627
|
};
|
|
13158
15628
|
if (dryRun) {
|
|
13159
15629
|
if (options.out) {
|
|
@@ -13188,6 +15658,9 @@ program
|
|
|
13188
15658
|
rateLimit: rateLimit ?? null,
|
|
13189
15659
|
pageError: pageError ?? null,
|
|
13190
15660
|
stop: localFetch.pacing.stop,
|
|
15661
|
+
accountLocationFilter,
|
|
15662
|
+
filteredAccounts: localFetch.filteredAccounts,
|
|
15663
|
+
rawInput: rawInputStats,
|
|
13191
15664
|
},
|
|
13192
15665
|
});
|
|
13193
15666
|
const attempt = {
|
|
@@ -13220,6 +15693,9 @@ program
|
|
|
13220
15693
|
startOffset,
|
|
13221
15694
|
pacing: localFetch.pacing,
|
|
13222
15695
|
stop: localFetch.pacing.stop,
|
|
15696
|
+
accountLocationFilter,
|
|
15697
|
+
filteredAccounts: localFetch.filteredAccounts,
|
|
15698
|
+
rawInput: rawInputStats,
|
|
13223
15699
|
},
|
|
13224
15700
|
});
|
|
13225
15701
|
const storedAccounts = await upsertSalesNavigatorAccounts({
|
|
@@ -13580,6 +16056,7 @@ program
|
|
|
13580
16056
|
.option("--max-retries <number>", "Bounded retries for transient network/5xx failures. Rate limits still stop immediately.", "1")
|
|
13581
16057
|
.option("--retry-base-delay-ms <number>", "Base delay for full-jitter retry backoff", "2000")
|
|
13582
16058
|
.option("--retry-max-delay-ms <number>", "Maximum delay for full-jitter retry backoff", "30000")
|
|
16059
|
+
.option("--include-location-mismatches", "Store account rows even when the visible Sales Navigator account location does not match the source REGION filter", false)
|
|
13583
16060
|
.option("--out <path>", "Optional local JSON output path")
|
|
13584
16061
|
.action(async (options) => {
|
|
13585
16062
|
const runId = z.string().uuid().parse(options.runId);
|
|
@@ -13647,6 +16124,7 @@ program
|
|
|
13647
16124
|
.max(300000)
|
|
13648
16125
|
.parse(options.retryMaxDelayMs),
|
|
13649
16126
|
};
|
|
16127
|
+
const filterLocationMismatches = !Boolean(options.includeLocationMismatches);
|
|
13650
16128
|
const templateRequest = parseSalesNavigatorCurlRequest(await readFile(options.curlFile, "utf8"));
|
|
13651
16129
|
if (!new URL(templateRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
13652
16130
|
throw new Error("Account repair --curl-file must contain a /sales-api/salesApiAccountSearch request.");
|
|
@@ -13701,6 +16179,9 @@ program
|
|
|
13701
16179
|
? requestedAccountsOption
|
|
13702
16180
|
: Math.min(requestedAccountsOption, remainingFromReported);
|
|
13703
16181
|
const parsedRequest = buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(slice.slicedQueryUrl, templateRequest, pageSize);
|
|
16182
|
+
const accountLocationFilter = filterLocationMismatches
|
|
16183
|
+
? buildSalesNavigatorAccountLocationFilter(slice.slicedQueryUrl)
|
|
16184
|
+
: null;
|
|
13704
16185
|
const fetchResult = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
13705
16186
|
requestedAccounts,
|
|
13706
16187
|
startOffset,
|
|
@@ -13710,6 +16191,7 @@ program
|
|
|
13710
16191
|
maxAllowedResults: 100000,
|
|
13711
16192
|
retry,
|
|
13712
16193
|
allowPartialPageErrors: true,
|
|
16194
|
+
accountLocationFilter,
|
|
13713
16195
|
});
|
|
13714
16196
|
const rateLimit = fetchResult.pacing.rateLimit;
|
|
13715
16197
|
const pageError = fetchResult.pacing.pageError;
|
|
@@ -13741,6 +16223,8 @@ program
|
|
|
13741
16223
|
previousStoredAccounts: slice.storedAccounts,
|
|
13742
16224
|
startOffset,
|
|
13743
16225
|
fetchedAccounts: fetchResult.accounts.length,
|
|
16226
|
+
filteredAccounts: fetchResult.filteredAccounts,
|
|
16227
|
+
accountLocationFilter,
|
|
13744
16228
|
},
|
|
13745
16229
|
pacing: fetchResult.pacing,
|
|
13746
16230
|
stop: fetchResult.pacing.stop,
|