salesprompter-cli 0.1.43 → 0.1.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -0
- package/dist/cli.js +2564 -111
- package/dist/linkedin-session.js +103 -17
- package/dist/sales-navigator.js +42 -15
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { spawn } from "node:child_process";
|
|
3
3
|
import { access, appendFile, mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
|
4
|
+
import { createServer } from "node:http";
|
|
4
5
|
import { createRequire } from "node:module";
|
|
5
6
|
import os from "node:os";
|
|
6
7
|
import path from "node:path";
|
|
@@ -26,10 +27,10 @@ import { InstantlySyncProvider } from "./instantly.js";
|
|
|
26
27
|
import { backfillLinkedInCompanies } from "./linkedin-companies.js";
|
|
27
28
|
import { parseLinkedInCompanyPage } from "./linkedin-companies.js";
|
|
28
29
|
import { crawlLinkedInProductCategory } from "./linkedin-products.js";
|
|
29
|
-
import { claimValidatedSalesNavigatorSessionCookieForCli, createLinkedInSessionSupabaseClient, resolveConfiguredEnvValue } from "./linkedin-session.js";
|
|
30
|
+
import { claimLinkedInSessionCookieForCli, claimValidatedSalesNavigatorSessionCookieForCli, createLinkedInSessionSupabaseClient, recordLinkedInSessionCookieAudit, resolveConfiguredEnvValue } from "./linkedin-session.js";
|
|
30
31
|
import { buildLeadlistsFunnelQueries } from "./leadlists-funnel.js";
|
|
31
32
|
import { readJsonFile, splitCsv, writeJsonFile, writeTextFile } from "./io.js";
|
|
32
|
-
import { buildSalesNavigatorCrawlPreview, createSalesNavigatorCrawlSeed, DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS, DEFAULT_SALES_NAVIGATOR_CRAWL_DIMENSIONS, buildSalesNavigatorPeopleSearchUrl, buildSalesNavigatorPeopleSlice, deriveSalesNavigatorTitleQuerySeeds, executeSalesNavigatorAdaptiveCrawl, expandSalesNavigatorCrawlAttempt, SalesNavigatorCrawlStopError, SalesNavigatorSliceTooBroadError } from "./sales-navigator.js";
|
|
33
|
+
import { buildSalesNavigatorCrawlPreview, buildSalesNavigatorAccountQuery, createSalesNavigatorCrawlSeed, DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS, DEFAULT_SALES_NAVIGATOR_CRAWL_DIMENSIONS, buildSalesNavigatorPeopleSearchUrl, buildSalesNavigatorPeopleSlice, deriveSalesNavigatorTitleQuerySeeds, executeSalesNavigatorAdaptiveCrawl, expandSalesNavigatorCrawlAttempt, SalesNavigatorCrawlStopError, SalesNavigatorSliceTooBroadError } from "./sales-navigator.js";
|
|
33
34
|
import { buildSalesNavigatorHistoricalBackfillPlan, ensureSalesNavigatorPeopleCount, resolveSalesNavigatorHistoricalBackfillConfig, resolveSalesNavigatorHistoricalBackfillResumeState, resolveSalesNavigatorHistoricalBackfillOrgId, salesNavigatorHistoricalBackfillDefaults } from "./salesnav-backfill.js";
|
|
34
35
|
const require = createRequire(import.meta.url);
|
|
35
36
|
const { version: packageVersion } = require("../package.json");
|
|
@@ -123,6 +124,84 @@ const LinkedInCompanyBackfillStatusResponseSchema = z.object({
|
|
|
123
124
|
failureCode: z.string().nullable().optional(),
|
|
124
125
|
failureMessage: z.string().nullable().optional()
|
|
125
126
|
});
|
|
127
|
+
const AffiliateCampaignLaunchResponseSchema = z.object({
|
|
128
|
+
status: z.enum(["ok", "partial"]),
|
|
129
|
+
dryRun: z.boolean(),
|
|
130
|
+
product: z.object({
|
|
131
|
+
domain: z.string().min(1),
|
|
132
|
+
affiliateLink: z.string().url()
|
|
133
|
+
}),
|
|
134
|
+
audience: z.object({
|
|
135
|
+
sourceType: z.enum(["sales_navigator_people", "linkedin_content"]),
|
|
136
|
+
linkedInUrl: z.string().url(),
|
|
137
|
+
maxResults: z.number().int().positive(),
|
|
138
|
+
peoplePreview: z.array(z.record(z.string(), z.unknown())).optional(),
|
|
139
|
+
postsPreview: z.array(z.record(z.string(), z.unknown())).optional()
|
|
140
|
+
}),
|
|
141
|
+
extraction: z
|
|
142
|
+
.object({
|
|
143
|
+
provider: z.string().min(1),
|
|
144
|
+
monitorId: z.string().min(1).optional(),
|
|
145
|
+
agentId: z.string().min(1).optional(),
|
|
146
|
+
containerId: z.string().min(1).optional()
|
|
147
|
+
})
|
|
148
|
+
.passthrough()
|
|
149
|
+
.optional(),
|
|
150
|
+
previewUrl: z.string().nullable().optional(),
|
|
151
|
+
next: z.string().optional()
|
|
152
|
+
});
|
|
153
|
+
const AffiliateCampaignLocalResultSchema = z
|
|
154
|
+
.object({
|
|
155
|
+
profileUrl: z.string().min(1),
|
|
156
|
+
})
|
|
157
|
+
.passthrough();
|
|
158
|
+
const DatabaseTimestampSchema = z.string().min(1);
|
|
159
|
+
const AffiliateOutreachRunSchema = z.object({
|
|
160
|
+
id: z.string().uuid(),
|
|
161
|
+
audience_run_id: z.string().uuid(),
|
|
162
|
+
organization_id: z.string().min(1),
|
|
163
|
+
status: z.enum(["preparing", "ready", "active", "failed"]),
|
|
164
|
+
provider: z.literal("instantly"),
|
|
165
|
+
instantly_campaign_id: z.string().min(1).nullable(),
|
|
166
|
+
instantly_campaign_name: z.string().min(1).nullable(),
|
|
167
|
+
sequence: z.array(z.object({
|
|
168
|
+
step: z.number().int().positive(),
|
|
169
|
+
subject: z.string(),
|
|
170
|
+
body: z.string(),
|
|
171
|
+
rationale: z.string(),
|
|
172
|
+
delayDays: z.number().int().nonnegative()
|
|
173
|
+
})),
|
|
174
|
+
enriched_leads: z.array(z.object({
|
|
175
|
+
rowIndex: z.number().int().nonnegative(),
|
|
176
|
+
fullName: z.string().min(1),
|
|
177
|
+
firstName: z.string().min(1),
|
|
178
|
+
lastName: z.string().min(1),
|
|
179
|
+
jobTitle: z.string().nullable(),
|
|
180
|
+
companyName: z.string().min(1),
|
|
181
|
+
companyDomain: z.string().min(1),
|
|
182
|
+
location: z.string().nullable(),
|
|
183
|
+
profileUrl: z.string().nullable(),
|
|
184
|
+
email: z.string().email(),
|
|
185
|
+
emailScore: z.number().min(0).max(100),
|
|
186
|
+
acceptAll: z.boolean(),
|
|
187
|
+
emailSource: z.enum(["hunter", "phantombuster"]).optional()
|
|
188
|
+
})),
|
|
189
|
+
stats: z.record(z.string(), z.unknown()),
|
|
190
|
+
settings: z.record(z.string(), z.unknown()),
|
|
191
|
+
error_message: z.string().nullable(),
|
|
192
|
+
activated_at: DatabaseTimestampSchema.nullable(),
|
|
193
|
+
created_at: DatabaseTimestampSchema,
|
|
194
|
+
updated_at: DatabaseTimestampSchema
|
|
195
|
+
});
|
|
196
|
+
const AffiliateOutreachResponseSchema = z.object({
|
|
197
|
+
status: z.literal("ok"),
|
|
198
|
+
outreach: AffiliateOutreachRunSchema
|
|
199
|
+
});
|
|
200
|
+
const AffiliateOutreachEnrichStartResponseSchema = z.object({
|
|
201
|
+
status: z.literal("accepted"),
|
|
202
|
+
audienceRunId: z.string().uuid(),
|
|
203
|
+
workflowRunId: z.string().min(1)
|
|
204
|
+
});
|
|
126
205
|
const SalesNavigatorLocalImportResponseSchema = z.object({
|
|
127
206
|
status: z.literal("ok"),
|
|
128
207
|
runId: z.string().min(1),
|
|
@@ -398,7 +477,15 @@ const cliPacks = [
|
|
|
398
477
|
slug: "outreach",
|
|
399
478
|
title: "Outreach",
|
|
400
479
|
summary: "Prepare and sync qualified leads into downstream systems.",
|
|
401
|
-
commands: [
|
|
480
|
+
commands: [
|
|
481
|
+
"affiliate:run",
|
|
482
|
+
"affiliate:status",
|
|
483
|
+
"affiliate:enrich",
|
|
484
|
+
"affiliate:activate",
|
|
485
|
+
"affiliate:launch",
|
|
486
|
+
"sync:outreach",
|
|
487
|
+
"sync:crm"
|
|
488
|
+
],
|
|
402
489
|
installStatus: "included"
|
|
403
490
|
}
|
|
404
491
|
];
|
|
@@ -443,6 +530,11 @@ const helpVisibleCommandNames = new Set([
|
|
|
443
530
|
"leads:enrich",
|
|
444
531
|
"leads:score",
|
|
445
532
|
"leads:pipeline",
|
|
533
|
+
"affiliate:run",
|
|
534
|
+
"affiliate:status",
|
|
535
|
+
"affiliate:enrich",
|
|
536
|
+
"affiliate:activate",
|
|
537
|
+
"affiliate:launch",
|
|
446
538
|
"sync:crm",
|
|
447
539
|
"sync:outreach",
|
|
448
540
|
"linkedin-companies:backfill",
|
|
@@ -1300,9 +1392,74 @@ function normalizeLinkedInDirectLookupCookieHeader(cookie) {
|
|
|
1300
1392
|
}
|
|
1301
1393
|
return `li_at=${trimmed}`;
|
|
1302
1394
|
}
|
|
1395
|
+
function mergeLinkedInCookieHeader(cookieHeader, setCookieHeaders) {
|
|
1396
|
+
const cookies = new Map();
|
|
1397
|
+
for (const part of cookieHeader.split(";")) {
|
|
1398
|
+
const separatorIndex = part.indexOf("=");
|
|
1399
|
+
if (separatorIndex <= 0) {
|
|
1400
|
+
continue;
|
|
1401
|
+
}
|
|
1402
|
+
cookies.set(part.slice(0, separatorIndex).trim(), part.slice(separatorIndex + 1).trim());
|
|
1403
|
+
}
|
|
1404
|
+
for (const setCookieHeader of setCookieHeaders) {
|
|
1405
|
+
const firstPart = setCookieHeader.split(";", 1)[0] ?? "";
|
|
1406
|
+
const separatorIndex = firstPart.indexOf("=");
|
|
1407
|
+
if (separatorIndex <= 0) {
|
|
1408
|
+
continue;
|
|
1409
|
+
}
|
|
1410
|
+
cookies.set(firstPart.slice(0, separatorIndex).trim(), firstPart.slice(separatorIndex + 1).trim());
|
|
1411
|
+
}
|
|
1412
|
+
return [...cookies.entries()]
|
|
1413
|
+
.map(([name, value]) => `${name}=${value}`)
|
|
1414
|
+
.join("; ");
|
|
1415
|
+
}
|
|
1416
|
+
async function bootstrapLinkedInAccountSearchConfig(params) {
|
|
1417
|
+
const cookie = normalizeLinkedInDirectLookupCookieHeader(params.sessionCookie);
|
|
1418
|
+
const existingCsrfToken = params.csrfToken || deriveCsrfTokenFromCookie(cookie);
|
|
1419
|
+
if (existingCsrfToken) {
|
|
1420
|
+
return {
|
|
1421
|
+
csrfToken: existingCsrfToken,
|
|
1422
|
+
identity: params.identity,
|
|
1423
|
+
cookie,
|
|
1424
|
+
userAgent: params.userAgent,
|
|
1425
|
+
};
|
|
1426
|
+
}
|
|
1427
|
+
const response = await fetch(params.queryUrl, {
|
|
1428
|
+
redirect: "manual",
|
|
1429
|
+
headers: {
|
|
1430
|
+
accept: "text/html,application/xhtml+xml",
|
|
1431
|
+
cookie,
|
|
1432
|
+
"user-agent": params.userAgent,
|
|
1433
|
+
},
|
|
1434
|
+
});
|
|
1435
|
+
if (response.status === 429 || response.status === 999) {
|
|
1436
|
+
throw new Error("Sales Navigator is rate-limited while bootstrapping the Account Search API session.");
|
|
1437
|
+
}
|
|
1438
|
+
if (response.status >= 300 && response.status < 400) {
|
|
1439
|
+
await response.arrayBuffer();
|
|
1440
|
+
return null;
|
|
1441
|
+
}
|
|
1442
|
+
const headersWithSetCookie = response.headers;
|
|
1443
|
+
const setCookieHeaders = headersWithSetCookie.getSetCookie?.() ??
|
|
1444
|
+
(response.headers.get("set-cookie")
|
|
1445
|
+
? [response.headers.get("set-cookie")]
|
|
1446
|
+
: []);
|
|
1447
|
+
const refreshedCookie = mergeLinkedInCookieHeader(cookie, setCookieHeaders);
|
|
1448
|
+
await response.arrayBuffer();
|
|
1449
|
+
const csrfToken = deriveCsrfTokenFromCookie(refreshedCookie);
|
|
1450
|
+
if (!csrfToken) {
|
|
1451
|
+
return null;
|
|
1452
|
+
}
|
|
1453
|
+
return {
|
|
1454
|
+
csrfToken,
|
|
1455
|
+
identity: params.identity,
|
|
1456
|
+
cookie: refreshedCookie,
|
|
1457
|
+
userAgent: params.userAgent,
|
|
1458
|
+
};
|
|
1459
|
+
}
|
|
1303
1460
|
function parseLocalLinkedInExtensionTokenLog(content) {
|
|
1304
1461
|
const matches = [
|
|
1305
|
-
...content.matchAll(/\{"csrfToken":"([^"]+)","extractedFrom":"sales-api\/
|
|
1462
|
+
...content.matchAll(/\{"csrfToken":"([^"]+)","extractedFrom":"sales-api\/salesApi(?:Lead|Account)Search"[\s\S]*?"linkedInIdentity":"([^"]+)"[\s\S]*?"sessionCookie":"([\s\S]*?)","syncStatus":"(success|captured)"[\s\S]*?"userAgent":"([^"]+)"\}/g)
|
|
1306
1463
|
];
|
|
1307
1464
|
const last = matches.at(-1);
|
|
1308
1465
|
if (!last) {
|
|
@@ -1331,6 +1488,21 @@ async function readLocalLinkedInExtensionTokenLog(filePath) {
|
|
|
1331
1488
|
return null;
|
|
1332
1489
|
}
|
|
1333
1490
|
}
|
|
1491
|
+
async function sortPathsByMostRecentlyModified(paths) {
|
|
1492
|
+
const candidates = await Promise.all(paths.map(async (filePath) => {
|
|
1493
|
+
try {
|
|
1494
|
+
const fileStat = await stat(filePath);
|
|
1495
|
+
return { filePath, modifiedAtMs: fileStat.mtimeMs };
|
|
1496
|
+
}
|
|
1497
|
+
catch {
|
|
1498
|
+
return { filePath, modifiedAtMs: 0 };
|
|
1499
|
+
}
|
|
1500
|
+
}));
|
|
1501
|
+
return candidates
|
|
1502
|
+
.sort((left, right) => right.modifiedAtMs - left.modifiedAtMs ||
|
|
1503
|
+
right.filePath.localeCompare(left.filePath))
|
|
1504
|
+
.map((candidate) => candidate.filePath);
|
|
1505
|
+
}
|
|
1334
1506
|
async function listChromeExtensionTokenLogCandidates() {
|
|
1335
1507
|
const overrideFile = normalizeLookupWhitespace(process.env.SALESPROMPTER_LINKEDIN_EXTENSION_TOKENS_LOG_PATH);
|
|
1336
1508
|
if (overrideFile) {
|
|
@@ -1340,11 +1512,9 @@ async function listChromeExtensionTokenLogCandidates() {
|
|
|
1340
1512
|
if (overrideDir) {
|
|
1341
1513
|
try {
|
|
1342
1514
|
const files = await readdir(overrideDir);
|
|
1343
|
-
return files
|
|
1515
|
+
return sortPathsByMostRecentlyModified(files
|
|
1344
1516
|
.filter((file) => file.endsWith(".log") || file.endsWith(".ldb"))
|
|
1345
|
-
.map((file) => path.join(overrideDir, file))
|
|
1346
|
-
.sort()
|
|
1347
|
-
.reverse();
|
|
1517
|
+
.map((file) => path.join(overrideDir, file)));
|
|
1348
1518
|
}
|
|
1349
1519
|
catch {
|
|
1350
1520
|
return [];
|
|
@@ -1382,7 +1552,7 @@ async function listChromeExtensionTokenLogCandidates() {
|
|
|
1382
1552
|
continue;
|
|
1383
1553
|
}
|
|
1384
1554
|
for (const file of files) {
|
|
1385
|
-
if (!file.endsWith(".log")) {
|
|
1555
|
+
if (!file.endsWith(".log") && !file.endsWith(".ldb")) {
|
|
1386
1556
|
continue;
|
|
1387
1557
|
}
|
|
1388
1558
|
paths.push(path.join(extensionDir, file));
|
|
@@ -1390,7 +1560,7 @@ async function listChromeExtensionTokenLogCandidates() {
|
|
|
1390
1560
|
}
|
|
1391
1561
|
}
|
|
1392
1562
|
}
|
|
1393
|
-
return paths
|
|
1563
|
+
return sortPathsByMostRecentlyModified(paths);
|
|
1394
1564
|
}
|
|
1395
1565
|
async function readLocalLinkedInExtensionDirectLookupConfig() {
|
|
1396
1566
|
const candidates = await listChromeExtensionTokenLogCandidates();
|
|
@@ -1464,11 +1634,12 @@ async function readStoredLinkedInDirectLookupConfig() {
|
|
|
1464
1634
|
return null;
|
|
1465
1635
|
}
|
|
1466
1636
|
const lookupResult = await supabase
|
|
1467
|
-
.from("
|
|
1637
|
+
.from("salesprompter_linkedin_session_cookies")
|
|
1468
1638
|
.select("metadata")
|
|
1469
1639
|
.eq("session_cookie_sha256", claimed.sessionCookieSha256)
|
|
1470
1640
|
.maybeSingle();
|
|
1471
|
-
if (lookupResult.error &&
|
|
1641
|
+
if (lookupResult.error &&
|
|
1642
|
+
!/salesprompter_linkedin_session_cookies/.test(lookupResult.error.message)) {
|
|
1472
1643
|
throw new Error(`Failed to read stored LinkedIn session metadata: ${lookupResult.error.message}`);
|
|
1473
1644
|
}
|
|
1474
1645
|
const metadata = lookupResult.data?.metadata;
|
|
@@ -1488,7 +1659,195 @@ async function readStoredLinkedInDirectLookupConfig() {
|
|
|
1488
1659
|
userAgent
|
|
1489
1660
|
};
|
|
1490
1661
|
}
|
|
1662
|
+
async function readStoredLinkedInAccountSearchConfig(queryUrl) {
|
|
1663
|
+
const supabase = createLinkedInSessionSupabaseClient();
|
|
1664
|
+
if (!supabase) {
|
|
1665
|
+
return null;
|
|
1666
|
+
}
|
|
1667
|
+
let lastFailure = "No Account Search API-capable LinkedIn session was available.";
|
|
1668
|
+
const configuredAttempts = Number(process.env.SALESPROMPTER_ACCOUNT_SEARCH_SESSION_ATTEMPTS);
|
|
1669
|
+
const maxAttempts = Number.isFinite(configuredAttempts) && configuredAttempts > 0
|
|
1670
|
+
? Math.min(64, Math.trunc(configuredAttempts))
|
|
1671
|
+
: 24;
|
|
1672
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
1673
|
+
const claimed = await claimLinkedInSessionCookieForCli({
|
|
1674
|
+
source: "cli_account_search",
|
|
1675
|
+
supabase,
|
|
1676
|
+
});
|
|
1677
|
+
if (!claimed) {
|
|
1678
|
+
return null;
|
|
1679
|
+
}
|
|
1680
|
+
const lookupResult = await supabase
|
|
1681
|
+
.from("salesprompter_linkedin_session_cookies")
|
|
1682
|
+
.select("metadata")
|
|
1683
|
+
.eq("session_cookie_sha256", claimed.sessionCookieSha256)
|
|
1684
|
+
.maybeSingle();
|
|
1685
|
+
if (lookupResult.error) {
|
|
1686
|
+
throw new Error(`Failed to read stored LinkedIn Account Search metadata: ${lookupResult.error.message}`);
|
|
1687
|
+
}
|
|
1688
|
+
const metadata = lookupResult.data?.metadata;
|
|
1689
|
+
const identity = extractStoredLinkedInLookupValue(metadata, "linkedInIdentity");
|
|
1690
|
+
const csrfToken = extractStoredLinkedInLookupValue(metadata, "csrfToken") ||
|
|
1691
|
+
deriveCsrfTokenFromCookie(claimed.sessionCookie);
|
|
1692
|
+
const userAgent = process.env.SALESPROMPTER_LINKEDIN_USER_AGENT?.trim() ||
|
|
1693
|
+
extractStoredLinkedInLookupValue(metadata, "userAgent") ||
|
|
1694
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36";
|
|
1695
|
+
let config = null;
|
|
1696
|
+
try {
|
|
1697
|
+
config = await bootstrapLinkedInAccountSearchConfig({
|
|
1698
|
+
queryUrl,
|
|
1699
|
+
sessionCookie: claimed.sessionCookie,
|
|
1700
|
+
identity,
|
|
1701
|
+
csrfToken,
|
|
1702
|
+
userAgent,
|
|
1703
|
+
});
|
|
1704
|
+
}
|
|
1705
|
+
catch (error) {
|
|
1706
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1707
|
+
if (/rate-limit/i.test(message)) {
|
|
1708
|
+
throw error;
|
|
1709
|
+
}
|
|
1710
|
+
lastFailure = message;
|
|
1711
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1712
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1713
|
+
source: "cli_account_search_api_preflight",
|
|
1714
|
+
feedStatus: null,
|
|
1715
|
+
salesNavigatorStatus: "error",
|
|
1716
|
+
recruiterStatus: null,
|
|
1717
|
+
productClass: "sales_navigator",
|
|
1718
|
+
isActive: true,
|
|
1719
|
+
inactiveReason: null,
|
|
1720
|
+
validationError: message,
|
|
1721
|
+
details: {
|
|
1722
|
+
attemptNumber: attempt,
|
|
1723
|
+
capability: "account_search_api_bootstrap",
|
|
1724
|
+
},
|
|
1725
|
+
});
|
|
1726
|
+
continue;
|
|
1727
|
+
}
|
|
1728
|
+
if (!config) {
|
|
1729
|
+
lastFailure =
|
|
1730
|
+
"Stored LinkedIn session could not bootstrap Account Search credentials.";
|
|
1731
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1732
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1733
|
+
source: "cli_account_search_api_preflight",
|
|
1734
|
+
feedStatus: null,
|
|
1735
|
+
salesNavigatorStatus: "error",
|
|
1736
|
+
recruiterStatus: null,
|
|
1737
|
+
productClass: "sales_navigator",
|
|
1738
|
+
isActive: false,
|
|
1739
|
+
inactiveReason: "salesnav_account_search_bootstrap_failed",
|
|
1740
|
+
validationError: "salesnav_account_search_bootstrap_failed",
|
|
1741
|
+
details: {
|
|
1742
|
+
attemptNumber: attempt,
|
|
1743
|
+
hasStoredIdentity: Boolean(identity),
|
|
1744
|
+
hasStoredCsrfToken: Boolean(csrfToken),
|
|
1745
|
+
},
|
|
1746
|
+
});
|
|
1747
|
+
continue;
|
|
1748
|
+
}
|
|
1749
|
+
const probe = await probeLinkedInAccountSearchConfig(queryUrl, config);
|
|
1750
|
+
if (probe.status === "ok") {
|
|
1751
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1752
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1753
|
+
source: "cli_account_search_api_preflight",
|
|
1754
|
+
feedStatus: null,
|
|
1755
|
+
salesNavigatorStatus: "ok",
|
|
1756
|
+
recruiterStatus: null,
|
|
1757
|
+
productClass: "sales_navigator",
|
|
1758
|
+
isActive: true,
|
|
1759
|
+
inactiveReason: null,
|
|
1760
|
+
validationError: null,
|
|
1761
|
+
details: {
|
|
1762
|
+
attemptNumber: attempt,
|
|
1763
|
+
statusCode: probe.statusCode,
|
|
1764
|
+
capability: "account_search_api",
|
|
1765
|
+
},
|
|
1766
|
+
});
|
|
1767
|
+
return config;
|
|
1768
|
+
}
|
|
1769
|
+
lastFailure =
|
|
1770
|
+
probe.validationError ??
|
|
1771
|
+
`Sales Navigator Account Search preflight returned ${probe.status}.`;
|
|
1772
|
+
if (probe.status === "rate_limited") {
|
|
1773
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1774
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1775
|
+
source: "cli_account_search_api_preflight",
|
|
1776
|
+
feedStatus: null,
|
|
1777
|
+
salesNavigatorStatus: "rate_limited",
|
|
1778
|
+
recruiterStatus: null,
|
|
1779
|
+
productClass: "sales_navigator",
|
|
1780
|
+
isActive: true,
|
|
1781
|
+
inactiveReason: null,
|
|
1782
|
+
validationError: probe.validationError,
|
|
1783
|
+
details: {
|
|
1784
|
+
attemptNumber: attempt,
|
|
1785
|
+
statusCode: probe.statusCode,
|
|
1786
|
+
capability: "account_search_api",
|
|
1787
|
+
},
|
|
1788
|
+
});
|
|
1789
|
+
throw new Error("Sales Navigator Account Search API is rate-limited. The CLI stopped without retrying this session.");
|
|
1790
|
+
}
|
|
1791
|
+
if (probe.status === "sales_seat_required") {
|
|
1792
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1793
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1794
|
+
source: "cli_account_search_api_preflight",
|
|
1795
|
+
feedStatus: null,
|
|
1796
|
+
salesNavigatorStatus: "upsell",
|
|
1797
|
+
recruiterStatus: null,
|
|
1798
|
+
productClass: null,
|
|
1799
|
+
isActive: false,
|
|
1800
|
+
inactiveReason: "salesnav_account_search_seat_required",
|
|
1801
|
+
validationError: probe.validationError,
|
|
1802
|
+
details: {
|
|
1803
|
+
attemptNumber: attempt,
|
|
1804
|
+
statusCode: probe.statusCode,
|
|
1805
|
+
capability: "account_search_api",
|
|
1806
|
+
},
|
|
1807
|
+
});
|
|
1808
|
+
continue;
|
|
1809
|
+
}
|
|
1810
|
+
if (probe.status === "unauthorized" || probe.status === "forbidden") {
|
|
1811
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1812
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1813
|
+
source: "cli_account_search_api_preflight",
|
|
1814
|
+
feedStatus: null,
|
|
1815
|
+
salesNavigatorStatus: "logged_out",
|
|
1816
|
+
recruiterStatus: null,
|
|
1817
|
+
productClass: "sales_navigator",
|
|
1818
|
+
isActive: false,
|
|
1819
|
+
inactiveReason: "salesnav_account_search_unauthorized",
|
|
1820
|
+
validationError: probe.validationError,
|
|
1821
|
+
details: {
|
|
1822
|
+
attemptNumber: attempt,
|
|
1823
|
+
statusCode: probe.statusCode,
|
|
1824
|
+
capability: "account_search_api",
|
|
1825
|
+
},
|
|
1826
|
+
});
|
|
1827
|
+
continue;
|
|
1828
|
+
}
|
|
1829
|
+
await recordLinkedInSessionCookieAudit(supabase, {
|
|
1830
|
+
sessionCookieSha256: claimed.sessionCookieSha256,
|
|
1831
|
+
source: "cli_account_search_api_preflight",
|
|
1832
|
+
feedStatus: null,
|
|
1833
|
+
salesNavigatorStatus: "error",
|
|
1834
|
+
recruiterStatus: null,
|
|
1835
|
+
productClass: "sales_navigator",
|
|
1836
|
+
isActive: true,
|
|
1837
|
+
inactiveReason: null,
|
|
1838
|
+
validationError: probe.validationError,
|
|
1839
|
+
details: {
|
|
1840
|
+
attemptNumber: attempt,
|
|
1841
|
+
statusCode: probe.statusCode,
|
|
1842
|
+
capability: "account_search_api",
|
|
1843
|
+
probeStatus: probe.status,
|
|
1844
|
+
},
|
|
1845
|
+
});
|
|
1846
|
+
}
|
|
1847
|
+
throw new Error(lastFailure);
|
|
1848
|
+
}
|
|
1491
1849
|
let cachedLinkedInDirectLookupConfig = null;
|
|
1850
|
+
let cachedLinkedInAccountSearchConfig = null;
|
|
1492
1851
|
async function readLinkedInDirectLookupConfig() {
|
|
1493
1852
|
if (cachedLinkedInDirectLookupConfig) {
|
|
1494
1853
|
return cachedLinkedInDirectLookupConfig;
|
|
@@ -1519,6 +1878,46 @@ async function readLinkedInDirectLookupConfig() {
|
|
|
1519
1878
|
}
|
|
1520
1879
|
throw new Error("Missing LinkedIn direct lookup session. Set LINKEDIN_CSRF_TOKEN, LINKEDIN_X_LI_IDENTITY, and LINKEDIN_SALES_NAV_COOKIE, or connect the Salesprompter Chrome extension to sync your LinkedIn session to salesprompter.ai.");
|
|
1521
1880
|
}
|
|
1881
|
+
async function readLinkedInAccountSearchConfig(queryUrl) {
|
|
1882
|
+
if (cachedLinkedInAccountSearchConfig) {
|
|
1883
|
+
return cachedLinkedInAccountSearchConfig;
|
|
1884
|
+
}
|
|
1885
|
+
const envConfig = readLinkedInDirectLookupEnvConfig();
|
|
1886
|
+
if (envConfig) {
|
|
1887
|
+
const probe = await probeLinkedInAccountSearchConfig(queryUrl, envConfig);
|
|
1888
|
+
assertLinkedInAccountSearchConfigProbe(probe);
|
|
1889
|
+
cachedLinkedInAccountSearchConfig = envConfig;
|
|
1890
|
+
return envConfig;
|
|
1891
|
+
}
|
|
1892
|
+
let storedError = null;
|
|
1893
|
+
try {
|
|
1894
|
+
const storedConfig = await readStoredLinkedInAccountSearchConfig(queryUrl);
|
|
1895
|
+
if (storedConfig) {
|
|
1896
|
+
cachedLinkedInAccountSearchConfig = storedConfig;
|
|
1897
|
+
return storedConfig;
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
catch (error) {
|
|
1901
|
+
storedError = error;
|
|
1902
|
+
if (isExplicitLinkedInSessionVaultEnabled() ||
|
|
1903
|
+
/rate-limit/i.test(error instanceof Error ? error.message : String(error))) {
|
|
1904
|
+
throw error;
|
|
1905
|
+
}
|
|
1906
|
+
}
|
|
1907
|
+
if (!shouldDisableLinkedInDirectLookupAutodiscovery()) {
|
|
1908
|
+
const localExtensionConfig = await readLocalLinkedInExtensionDirectLookupConfig();
|
|
1909
|
+
if (localExtensionConfig) {
|
|
1910
|
+
const probe = await probeLinkedInAccountSearchConfig(queryUrl, localExtensionConfig);
|
|
1911
|
+
assertLinkedInAccountSearchConfigProbe(probe);
|
|
1912
|
+
cachedLinkedInAccountSearchConfig = localExtensionConfig;
|
|
1913
|
+
return localExtensionConfig;
|
|
1914
|
+
}
|
|
1915
|
+
}
|
|
1916
|
+
if (storedError) {
|
|
1917
|
+
throw storedError;
|
|
1918
|
+
}
|
|
1919
|
+
throw new Error("Missing LinkedIn Account Search API session. Connect the Salesprompter Chrome extension or configure the Salesprompter session vault.");
|
|
1920
|
+
}
|
|
1522
1921
|
function generateLinkedInSessionId() {
|
|
1523
1922
|
return Array.from({ length: 22 }, () => "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789".charAt(Math.floor(Math.random() * 62))).join("");
|
|
1524
1923
|
}
|
|
@@ -2198,6 +2597,203 @@ function normalizeLocalSalesNavigatorAccounts(responseBody, queryUrl) {
|
|
|
2198
2597
|
}
|
|
2199
2598
|
return accounts;
|
|
2200
2599
|
}
|
|
2600
|
+
async function readLocalSalesNavigatorRawAccountInput(params) {
|
|
2601
|
+
const filePath = path.resolve(params.filePath);
|
|
2602
|
+
const content = await readFile(filePath, "utf8");
|
|
2603
|
+
const accounts = [];
|
|
2604
|
+
const seenAccounts = new Set();
|
|
2605
|
+
const requestUrls = new Set();
|
|
2606
|
+
let parsedLines = 0;
|
|
2607
|
+
let parsedElements = 0;
|
|
2608
|
+
let invalidElements = 0;
|
|
2609
|
+
let duplicateAccounts = 0;
|
|
2610
|
+
let filteredAccounts = 0;
|
|
2611
|
+
const addElement = (element, requestUrl) => {
|
|
2612
|
+
parsedElements += 1;
|
|
2613
|
+
if (typeof element !== "object" || element === null || Array.isArray(element)) {
|
|
2614
|
+
invalidElements += 1;
|
|
2615
|
+
return;
|
|
2616
|
+
}
|
|
2617
|
+
const account = normalizeLocalSalesNavigatorAccount(element, requestUrl);
|
|
2618
|
+
if (!account) {
|
|
2619
|
+
invalidElements += 1;
|
|
2620
|
+
return;
|
|
2621
|
+
}
|
|
2622
|
+
if (seenAccounts.has(account.accountKey)) {
|
|
2623
|
+
duplicateAccounts += 1;
|
|
2624
|
+
return;
|
|
2625
|
+
}
|
|
2626
|
+
seenAccounts.add(account.accountKey);
|
|
2627
|
+
if (!salesNavigatorAccountMatchesLocationFilter(account, params.accountLocationFilter)) {
|
|
2628
|
+
filteredAccounts += 1;
|
|
2629
|
+
return;
|
|
2630
|
+
}
|
|
2631
|
+
accounts.push(account);
|
|
2632
|
+
};
|
|
2633
|
+
const lines = content.split(/\r?\n/);
|
|
2634
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
2635
|
+
const line = lines[index]?.trim() ?? "";
|
|
2636
|
+
if (!line) {
|
|
2637
|
+
continue;
|
|
2638
|
+
}
|
|
2639
|
+
parsedLines += 1;
|
|
2640
|
+
let value;
|
|
2641
|
+
try {
|
|
2642
|
+
value = JSON.parse(line);
|
|
2643
|
+
}
|
|
2644
|
+
catch (error) {
|
|
2645
|
+
throw new Error(`Raw Sales Navigator account input has invalid JSON on line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2646
|
+
}
|
|
2647
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
2648
|
+
invalidElements += 1;
|
|
2649
|
+
continue;
|
|
2650
|
+
}
|
|
2651
|
+
const record = value;
|
|
2652
|
+
const requestUrlCandidate = [
|
|
2653
|
+
record.queryUrl,
|
|
2654
|
+
record.requestUrl,
|
|
2655
|
+
record.slicedQueryUrl,
|
|
2656
|
+
].find((candidate) => typeof candidate === "string" && candidate.trim().length > 0);
|
|
2657
|
+
const requestUrl = requestUrlCandidate?.trim() ?? params.sourceQueryUrl;
|
|
2658
|
+
requestUrls.add(requestUrl);
|
|
2659
|
+
if (Array.isArray(record.elements)) {
|
|
2660
|
+
for (const element of record.elements) {
|
|
2661
|
+
addElement(element, requestUrl);
|
|
2662
|
+
}
|
|
2663
|
+
continue;
|
|
2664
|
+
}
|
|
2665
|
+
if ("element" in record) {
|
|
2666
|
+
addElement(record.element, requestUrl);
|
|
2667
|
+
continue;
|
|
2668
|
+
}
|
|
2669
|
+
if ("account" in record) {
|
|
2670
|
+
addElement(record.account, requestUrl);
|
|
2671
|
+
continue;
|
|
2672
|
+
}
|
|
2673
|
+
if (record.type === "metadata") {
|
|
2674
|
+
continue;
|
|
2675
|
+
}
|
|
2676
|
+
addElement(record, requestUrl);
|
|
2677
|
+
}
|
|
2678
|
+
const totalResults = params.reportedTotal ?? accounts.length;
|
|
2679
|
+
const missingFromReported = Math.max(0, totalResults - accounts.length);
|
|
2680
|
+
const complete = missingFromReported === 0;
|
|
2681
|
+
return {
|
|
2682
|
+
fetchResult: {
|
|
2683
|
+
accounts,
|
|
2684
|
+
filteredAccounts,
|
|
2685
|
+
totalResults,
|
|
2686
|
+
fetchedPages: requestUrls.size,
|
|
2687
|
+
sourceQueryUrl: params.sourceQueryUrl,
|
|
2688
|
+
pacing: {
|
|
2689
|
+
startOffset: 0,
|
|
2690
|
+
delayMinMs: 0,
|
|
2691
|
+
delayMaxMs: 0,
|
|
2692
|
+
totalDelayMs: 0,
|
|
2693
|
+
retryCount: 0,
|
|
2694
|
+
stop: {
|
|
2695
|
+
reason: complete
|
|
2696
|
+
? "reported_total_reached"
|
|
2697
|
+
: "raw_results_incomplete",
|
|
2698
|
+
startOffset: 0,
|
|
2699
|
+
pageSize: accounts.length,
|
|
2700
|
+
lastSuccessfulStart: accounts.length > 0 ? 0 : null,
|
|
2701
|
+
lastSuccessfulCount: accounts.length,
|
|
2702
|
+
lastSuccessfulAdded: accounts.length,
|
|
2703
|
+
stoppedAtStart: accounts.length,
|
|
2704
|
+
stoppedAtCount: 0,
|
|
2705
|
+
nextStart: accounts.length,
|
|
2706
|
+
fetchedAccounts: accounts.length,
|
|
2707
|
+
reportedTotal: totalResults,
|
|
2708
|
+
missingFromReported,
|
|
2709
|
+
},
|
|
2710
|
+
},
|
|
2711
|
+
},
|
|
2712
|
+
stats: {
|
|
2713
|
+
filePath,
|
|
2714
|
+
parsedLines,
|
|
2715
|
+
parsedElements,
|
|
2716
|
+
invalidElements,
|
|
2717
|
+
duplicateAccounts,
|
|
2718
|
+
distinctRequestUrls: requestUrls.size,
|
|
2719
|
+
},
|
|
2720
|
+
};
|
|
2721
|
+
}
|
|
2722
|
+
function decodeSalesNavigatorQueryTextValue(value) {
|
|
2723
|
+
let decoded = value.trim();
|
|
2724
|
+
for (let index = 0; index < 3; index += 1) {
|
|
2725
|
+
try {
|
|
2726
|
+
const next = decodeURIComponent(decoded);
|
|
2727
|
+
if (next === decoded) {
|
|
2728
|
+
break;
|
|
2729
|
+
}
|
|
2730
|
+
decoded = next;
|
|
2731
|
+
}
|
|
2732
|
+
catch {
|
|
2733
|
+
break;
|
|
2734
|
+
}
|
|
2735
|
+
}
|
|
2736
|
+
return decoded.replaceAll("+", " ").replace(/\s+/g, " ").trim();
|
|
2737
|
+
}
|
|
2738
|
+
function extractIncludedSalesNavigatorFilterTexts(searchUrl, filterType) {
|
|
2739
|
+
const decodedQuery = decodeSalesNavigatorQueryParam(searchUrl);
|
|
2740
|
+
if (!decodedQuery) {
|
|
2741
|
+
return [];
|
|
2742
|
+
}
|
|
2743
|
+
const values = [];
|
|
2744
|
+
const seen = new Set();
|
|
2745
|
+
const filterPattern = new RegExp(`type:${filterType},values:List\\((.*?)\\)\\)`, "g");
|
|
2746
|
+
for (const filterMatch of decodedQuery.matchAll(filterPattern)) {
|
|
2747
|
+
const block = filterMatch[1] ?? "";
|
|
2748
|
+
for (const valueMatch of block.matchAll(/text:([^,)]+),selectionType:INCLUDED/g)) {
|
|
2749
|
+
const text = decodeSalesNavigatorQueryTextValue(valueMatch[1] ?? "");
|
|
2750
|
+
const key = normalizeLooseMatchText(text);
|
|
2751
|
+
if (text && !seen.has(key)) {
|
|
2752
|
+
seen.add(key);
|
|
2753
|
+
values.push(text);
|
|
2754
|
+
}
|
|
2755
|
+
}
|
|
2756
|
+
}
|
|
2757
|
+
return values;
|
|
2758
|
+
}
|
|
2759
|
+
function buildSalesNavigatorAccountLocationFilter(searchUrl) {
|
|
2760
|
+
const regionTexts = extractIncludedSalesNavigatorFilterTexts(searchUrl, "REGION");
|
|
2761
|
+
const requiredTerms = regionTexts
|
|
2762
|
+
.flatMap((text) => normalizeLooseMatchText(text).split(" "))
|
|
2763
|
+
.filter((term) => term.length >= 3 &&
|
|
2764
|
+
![
|
|
2765
|
+
"area",
|
|
2766
|
+
"metro",
|
|
2767
|
+
"metropolitan",
|
|
2768
|
+
"region",
|
|
2769
|
+
"greater",
|
|
2770
|
+
"united",
|
|
2771
|
+
"states",
|
|
2772
|
+
"kingdom",
|
|
2773
|
+
"germany",
|
|
2774
|
+
"deutschland",
|
|
2775
|
+
"europe",
|
|
2776
|
+
].includes(term));
|
|
2777
|
+
if (regionTexts.length === 0 || requiredTerms.length === 0) {
|
|
2778
|
+
return null;
|
|
2779
|
+
}
|
|
2780
|
+
return {
|
|
2781
|
+
regionTexts,
|
|
2782
|
+
requiredTerms: [...new Set(requiredTerms)],
|
|
2783
|
+
};
|
|
2784
|
+
}
|
|
2785
|
+
function salesNavigatorAccountMatchesLocationFilter(account, filter) {
|
|
2786
|
+
if (!filter || filter.requiredTerms.length === 0) {
|
|
2787
|
+
return true;
|
|
2788
|
+
}
|
|
2789
|
+
const visibleLocation = typeof account.location === "string"
|
|
2790
|
+
? normalizeLooseMatchText(account.location)
|
|
2791
|
+
: "";
|
|
2792
|
+
if (!visibleLocation) {
|
|
2793
|
+
return true;
|
|
2794
|
+
}
|
|
2795
|
+
return filter.requiredTerms.some((term) => visibleLocation.includes(term));
|
|
2796
|
+
}
|
|
2201
2797
|
function buildLinkedInCompanyLookupVariants(params) {
|
|
2202
2798
|
const variants = [];
|
|
2203
2799
|
const seen = new Set();
|
|
@@ -5300,6 +5896,19 @@ function isMissingSupabaseTableError(code, message) {
|
|
|
5300
5896
|
const normalized = message.toLowerCase();
|
|
5301
5897
|
return code === "42P01" || normalized.includes("does not exist");
|
|
5302
5898
|
}
|
|
5899
|
+
function createSalesNavigatorSupabaseFetch(env = process.env) {
|
|
5900
|
+
const configuredTimeoutMs = Number(env.SALESPROMPTER_SALESNAV_SUPABASE_TIMEOUT_MS);
|
|
5901
|
+
const timeoutMs = Number.isFinite(configuredTimeoutMs) && configuredTimeoutMs >= 500
|
|
5902
|
+
? Math.min(30_000, Math.trunc(configuredTimeoutMs))
|
|
5903
|
+
: 2_500;
|
|
5904
|
+
return (input, init) => {
|
|
5905
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
5906
|
+
const signal = init?.signal
|
|
5907
|
+
? AbortSignal.any([init.signal, timeoutSignal])
|
|
5908
|
+
: timeoutSignal;
|
|
5909
|
+
return fetch(input, { ...init, signal });
|
|
5910
|
+
};
|
|
5911
|
+
}
|
|
5303
5912
|
async function createSalesNavigatorCrawlEventStore(options) {
|
|
5304
5913
|
const config = (() => {
|
|
5305
5914
|
try {
|
|
@@ -5317,7 +5926,10 @@ async function createSalesNavigatorCrawlEventStore(options) {
|
|
|
5317
5926
|
auth: {
|
|
5318
5927
|
persistSession: false,
|
|
5319
5928
|
autoRefreshToken: false
|
|
5320
|
-
}
|
|
5929
|
+
},
|
|
5930
|
+
global: {
|
|
5931
|
+
fetch: createSalesNavigatorSupabaseFetch(process.env),
|
|
5932
|
+
},
|
|
5321
5933
|
});
|
|
5322
5934
|
let writable = true;
|
|
5323
5935
|
return {
|
|
@@ -5341,6 +5953,7 @@ async function createSalesNavigatorCrawlEventStore(options) {
|
|
|
5341
5953
|
writable = false;
|
|
5342
5954
|
return;
|
|
5343
5955
|
}
|
|
5956
|
+
writable = false;
|
|
5344
5957
|
throw new Error(`Failed to write crawl event to Supabase: ${error.message}`);
|
|
5345
5958
|
},
|
|
5346
5959
|
listRecentForJob: async (jobId, limit) => {
|
|
@@ -5913,6 +6526,79 @@ async function fetchCliJson(session, request, schema) {
|
|
|
5913
6526
|
return schema.parse(parsed);
|
|
5914
6527
|
});
|
|
5915
6528
|
}
|
|
6529
|
+
async function launchAffiliateCampaignViaApp(session, payload) {
|
|
6530
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-campaigns`, {
|
|
6531
|
+
method: "POST",
|
|
6532
|
+
headers: {
|
|
6533
|
+
"Content-Type": "application/json",
|
|
6534
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6535
|
+
},
|
|
6536
|
+
body: JSON.stringify(payload)
|
|
6537
|
+
}), AffiliateCampaignLaunchResponseSchema);
|
|
6538
|
+
return value;
|
|
6539
|
+
}
|
|
6540
|
+
async function prepareAffiliateOutreachViaApp(session, payload) {
|
|
6541
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach`, {
|
|
6542
|
+
method: "POST",
|
|
6543
|
+
headers: {
|
|
6544
|
+
"Content-Type": "application/json",
|
|
6545
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6546
|
+
},
|
|
6547
|
+
body: JSON.stringify(payload)
|
|
6548
|
+
}), AffiliateOutreachResponseSchema);
|
|
6549
|
+
return value.outreach;
|
|
6550
|
+
}
|
|
6551
|
+
async function getAffiliateOutreachViaApp(session, audienceRunId) {
|
|
6552
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach/${encodeURIComponent(audienceRunId)}`, {
|
|
6553
|
+
headers: {
|
|
6554
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6555
|
+
}
|
|
6556
|
+
}), AffiliateOutreachResponseSchema);
|
|
6557
|
+
return value.outreach;
|
|
6558
|
+
}
|
|
6559
|
+
async function activateAffiliateOutreachViaApp(session, audienceRunId) {
|
|
6560
|
+
const { value } = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach/${encodeURIComponent(audienceRunId)}/activate`, {
|
|
6561
|
+
method: "POST",
|
|
6562
|
+
headers: {
|
|
6563
|
+
"Content-Type": "application/json",
|
|
6564
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6565
|
+
},
|
|
6566
|
+
body: JSON.stringify({})
|
|
6567
|
+
}), AffiliateOutreachResponseSchema);
|
|
6568
|
+
return value.outreach;
|
|
6569
|
+
}
|
|
6570
|
+
async function enrichAffiliateOutreachViaApp(session, audienceRunId) {
|
|
6571
|
+
const before = await getAffiliateOutreachViaApp(session, audienceRunId);
|
|
6572
|
+
const previousAttemptedAt = typeof before.stats.latestTopUp === "object" && before.stats.latestTopUp !== null
|
|
6573
|
+
? String(before.stats.latestTopUp.attemptedAt ?? "")
|
|
6574
|
+
: "";
|
|
6575
|
+
await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/affiliate-outreach/${encodeURIComponent(audienceRunId)}/enrich`, {
|
|
6576
|
+
method: "POST",
|
|
6577
|
+
headers: {
|
|
6578
|
+
"Content-Type": "application/json",
|
|
6579
|
+
Authorization: `Bearer ${currentSession.accessToken}`
|
|
6580
|
+
},
|
|
6581
|
+
body: JSON.stringify({})
|
|
6582
|
+
}), AffiliateOutreachEnrichStartResponseSchema);
|
|
6583
|
+
const deadline = Date.now() + 10 * 60 * 1000;
|
|
6584
|
+
const configuredPollMs = Number(process.env.SALESPROMPTER_AFFILIATE_ENRICH_POLL_MS);
|
|
6585
|
+
const pollMs = Number.isFinite(configuredPollMs) && configuredPollMs > 0
|
|
6586
|
+
? Math.max(10, Math.trunc(configuredPollMs))
|
|
6587
|
+
: 5_000;
|
|
6588
|
+
while (Date.now() < deadline) {
|
|
6589
|
+
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
|
6590
|
+
const current = await getAffiliateOutreachViaApp(session, audienceRunId);
|
|
6591
|
+
const latestTopUp = typeof current.stats.latestTopUp === "object" &&
|
|
6592
|
+
current.stats.latestTopUp !== null
|
|
6593
|
+
? current.stats.latestTopUp
|
|
6594
|
+
: null;
|
|
6595
|
+
const attemptedAt = String(latestTopUp?.attemptedAt ?? "");
|
|
6596
|
+
if (attemptedAt && attemptedAt !== previousAttemptedAt) {
|
|
6597
|
+
return current;
|
|
6598
|
+
}
|
|
6599
|
+
}
|
|
6600
|
+
throw new Error("Email recovery is still running after 10 minutes. Check affiliate:status and retry later.");
|
|
6601
|
+
}
|
|
5916
6602
|
async function uploadLinkedInProductsCatalog(session, payload, batchSize = 100, traceId) {
|
|
5917
6603
|
let imported = 0;
|
|
5918
6604
|
let upserted = 0;
|
|
@@ -6068,12 +6754,15 @@ function normalizeSalesNavigatorApiQueryValue(rawQuery) {
|
|
|
6068
6754
|
: withRecentSearchId.replace(/^\(/, "(spellCorrectionEnabled:true,");
|
|
6069
6755
|
}
|
|
6070
6756
|
function buildSalesNavigatorApiRequestFromSearchUrl(searchUrl, config, count) {
|
|
6757
|
+
return {
|
|
6758
|
+
url: buildSalesNavigatorLeadApiUrlFromSearchUrl(searchUrl, count),
|
|
6759
|
+
headers: buildLinkedInSalesNavigatorLocalImportHeaders(config),
|
|
6760
|
+
};
|
|
6761
|
+
}
|
|
6762
|
+
function buildSalesNavigatorLeadApiUrlFromSearchUrl(searchUrl, count) {
|
|
6071
6763
|
const source = new URL(searchUrl);
|
|
6072
6764
|
if (source.pathname.includes("/sales-api/salesApiLeadSearch")) {
|
|
6073
|
-
return
|
|
6074
|
-
url: source.toString(),
|
|
6075
|
-
headers: buildLinkedInSalesNavigatorLocalImportHeaders(config),
|
|
6076
|
-
};
|
|
6765
|
+
return source.toString();
|
|
6077
6766
|
}
|
|
6078
6767
|
if (!source.pathname.includes("/sales/search/people")) {
|
|
6079
6768
|
throw new Error("Automatic Sales Navigator import requires a /sales/search/people URL.");
|
|
@@ -6092,10 +6781,7 @@ function buildSalesNavigatorApiRequestFromSearchUrl(searchUrl, config, count) {
|
|
|
6092
6781
|
`?q=searchQuery&query=${apiQuery}&start=0&count=${normalizedCount}` +
|
|
6093
6782
|
`&trackingParam=(sessionId:${rawSessionId})` +
|
|
6094
6783
|
"&decorationId=com.linkedin.sales.deco.desktop.searchv2.LeadSearchResult-14";
|
|
6095
|
-
return
|
|
6096
|
-
url: apiUrl,
|
|
6097
|
-
headers: buildLinkedInSalesNavigatorLocalImportHeaders(config),
|
|
6098
|
-
};
|
|
6784
|
+
return apiUrl;
|
|
6099
6785
|
}
|
|
6100
6786
|
function extractSalesNavigatorDecorationId(requestUrl, fallbackDecorationId) {
|
|
6101
6787
|
try {
|
|
@@ -6122,15 +6808,12 @@ function buildSalesNavigatorAccountApiUrlFromSearchUrl(searchUrl, count, options
|
|
|
6122
6808
|
const baseUrl = compactOptionalText(options.baseUrl) ??
|
|
6123
6809
|
compactOptionalText(process.env.SALESPROMPTER_LINKEDIN_SALES_API_BASE_URL) ??
|
|
6124
6810
|
`${source.protocol}//${source.host}`;
|
|
6125
|
-
const
|
|
6126
|
-
encodeURIComponent(generateLinkedInSessionId());
|
|
6127
|
-
const apiQuery = normalizeSalesNavigatorApiQueryValue(rawQuery);
|
|
6811
|
+
const apiQuery = decodeRawSalesNavigatorQueryParam(rawQuery);
|
|
6128
6812
|
const normalizedCount = Math.max(1, Math.min(100, Math.trunc(count)));
|
|
6129
6813
|
const decorationId = compactOptionalText(options.decorationId) ??
|
|
6130
|
-
"com.linkedin.sales.deco.desktop.searchv2.AccountSearchResult-
|
|
6814
|
+
"com.linkedin.sales.deco.desktop.searchv2.AccountSearchResult-4";
|
|
6131
6815
|
return (`${baseUrl.replace(/\/+$/, "")}/sales-api/salesApiAccountSearch` +
|
|
6132
6816
|
`?q=searchQuery&query=${apiQuery}&start=0&count=${normalizedCount}` +
|
|
6133
|
-
`&trackingParam=(sessionId:${rawSessionId})` +
|
|
6134
6817
|
`&decorationId=${decorationId}`);
|
|
6135
6818
|
}
|
|
6136
6819
|
function buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, count) {
|
|
@@ -6146,26 +6829,136 @@ function buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, co
|
|
|
6146
6829
|
headers: buildLinkedInSalesNavigatorAccountSearchHeaders(config),
|
|
6147
6830
|
};
|
|
6148
6831
|
}
|
|
6149
|
-
function
|
|
6150
|
-
const
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6155
|
-
|
|
6156
|
-
|
|
6157
|
-
|
|
6158
|
-
|
|
6159
|
-
|
|
6160
|
-
|
|
6161
|
-
|
|
6162
|
-
|
|
6163
|
-
|
|
6164
|
-
|
|
6165
|
-
|
|
6166
|
-
|
|
6167
|
-
|
|
6168
|
-
|
|
6832
|
+
async function probeLinkedInAccountSearchConfig(searchUrl, config) {
|
|
6833
|
+
const request = buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, 1);
|
|
6834
|
+
const controller = new AbortController();
|
|
6835
|
+
const timeout = setTimeout(() => controller.abort(), 8_000);
|
|
6836
|
+
try {
|
|
6837
|
+
const response = await fetch(request.url, {
|
|
6838
|
+
headers: request.headers,
|
|
6839
|
+
signal: controller.signal,
|
|
6840
|
+
});
|
|
6841
|
+
if (response.status === 401) {
|
|
6842
|
+
return {
|
|
6843
|
+
status: "unauthorized",
|
|
6844
|
+
statusCode: response.status,
|
|
6845
|
+
validationError: "salesnav_account_search_http_401",
|
|
6846
|
+
};
|
|
6847
|
+
}
|
|
6848
|
+
if (response.status === 403) {
|
|
6849
|
+
const body = await response.text();
|
|
6850
|
+
let errorCode = "";
|
|
6851
|
+
try {
|
|
6852
|
+
const parsed = JSON.parse(body);
|
|
6853
|
+
errorCode =
|
|
6854
|
+
typeof parsed.code === "string" ? parsed.code.trim() : "";
|
|
6855
|
+
}
|
|
6856
|
+
catch {
|
|
6857
|
+
errorCode = "";
|
|
6858
|
+
}
|
|
6859
|
+
if (errorCode === "SALES_SEAT_REQUIRED") {
|
|
6860
|
+
return {
|
|
6861
|
+
status: "sales_seat_required",
|
|
6862
|
+
statusCode: response.status,
|
|
6863
|
+
validationError: "salesnav_account_search_sales_seat_required",
|
|
6864
|
+
};
|
|
6865
|
+
}
|
|
6866
|
+
return {
|
|
6867
|
+
status: "forbidden",
|
|
6868
|
+
statusCode: response.status,
|
|
6869
|
+
validationError: "salesnav_account_search_http_403",
|
|
6870
|
+
};
|
|
6871
|
+
}
|
|
6872
|
+
if (response.status === 429 || response.status === 999) {
|
|
6873
|
+
return {
|
|
6874
|
+
status: "rate_limited",
|
|
6875
|
+
statusCode: response.status,
|
|
6876
|
+
validationError: "salesnav_account_search_rate_limited",
|
|
6877
|
+
};
|
|
6878
|
+
}
|
|
6879
|
+
if (!response.ok) {
|
|
6880
|
+
return {
|
|
6881
|
+
status: "error",
|
|
6882
|
+
statusCode: response.status,
|
|
6883
|
+
validationError: `salesnav_account_search_http_${response.status}`,
|
|
6884
|
+
};
|
|
6885
|
+
}
|
|
6886
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
6887
|
+
const body = await response.text();
|
|
6888
|
+
if (!/json/i.test(contentType) && /^\s*</.test(body)) {
|
|
6889
|
+
return {
|
|
6890
|
+
status: "invalid_response",
|
|
6891
|
+
statusCode: response.status,
|
|
6892
|
+
validationError: "salesnav_account_search_returned_html",
|
|
6893
|
+
};
|
|
6894
|
+
}
|
|
6895
|
+
try {
|
|
6896
|
+
const parsed = JSON.parse(body);
|
|
6897
|
+
if (typeof parsed !== "object" || parsed === null) {
|
|
6898
|
+
throw new Error("response is not an object");
|
|
6899
|
+
}
|
|
6900
|
+
}
|
|
6901
|
+
catch {
|
|
6902
|
+
return {
|
|
6903
|
+
status: "invalid_response",
|
|
6904
|
+
statusCode: response.status,
|
|
6905
|
+
validationError: "salesnav_account_search_invalid_json",
|
|
6906
|
+
};
|
|
6907
|
+
}
|
|
6908
|
+
return {
|
|
6909
|
+
status: "ok",
|
|
6910
|
+
statusCode: response.status,
|
|
6911
|
+
validationError: null,
|
|
6912
|
+
};
|
|
6913
|
+
}
|
|
6914
|
+
catch (error) {
|
|
6915
|
+
return {
|
|
6916
|
+
status: "error",
|
|
6917
|
+
statusCode: null,
|
|
6918
|
+
validationError: error instanceof Error && error.name === "AbortError"
|
|
6919
|
+
? "salesnav_account_search_timeout"
|
|
6920
|
+
: `salesnav_account_search_request_failed:${error instanceof Error ? error.message : String(error)}`,
|
|
6921
|
+
};
|
|
6922
|
+
}
|
|
6923
|
+
finally {
|
|
6924
|
+
clearTimeout(timeout);
|
|
6925
|
+
}
|
|
6926
|
+
}
|
|
6927
|
+
function assertLinkedInAccountSearchConfigProbe(probe) {
|
|
6928
|
+
if (probe.status === "ok") {
|
|
6929
|
+
return;
|
|
6930
|
+
}
|
|
6931
|
+
if (probe.status === "rate_limited") {
|
|
6932
|
+
throw new Error("Sales Navigator Account Search API is rate-limited. The CLI stopped without retrying the session.");
|
|
6933
|
+
}
|
|
6934
|
+
if (probe.status === "unauthorized" || probe.status === "forbidden") {
|
|
6935
|
+
throw new Error(`The configured LinkedIn session cannot call the Sales Navigator Account Search API (HTTP ${probe.statusCode}). Refresh the Salesprompter LinkedIn session before retrying.`);
|
|
6936
|
+
}
|
|
6937
|
+
if (probe.status === "sales_seat_required") {
|
|
6938
|
+
throw new Error("The configured LinkedIn session is logged in but does not have a Sales Navigator seat.");
|
|
6939
|
+
}
|
|
6940
|
+
throw new Error(`Sales Navigator Account Search API preflight failed: ${probe.validationError ?? probe.status}`);
|
|
6941
|
+
}
|
|
6942
|
+
function buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(searchUrl, templateRequest, count) {
|
|
6943
|
+
const templateUrl = new URL(templateRequest.url);
|
|
6944
|
+
return {
|
|
6945
|
+
url: buildSalesNavigatorAccountApiUrlFromSearchUrl(searchUrl, count, {
|
|
6946
|
+
baseUrl: `${templateUrl.protocol}//${templateUrl.host}`,
|
|
6947
|
+
decorationId: extractSalesNavigatorDecorationId(templateRequest.url, "com.linkedin.sales.deco.desktop.searchv2.AccountSearchResult-4"),
|
|
6948
|
+
}),
|
|
6949
|
+
headers: { ...templateRequest.headers },
|
|
6950
|
+
};
|
|
6951
|
+
}
|
|
6952
|
+
const SALES_NAVIGATOR_PROFILE_LANGUAGE_VALUES = new Map([
|
|
6953
|
+
["de", { id: "de", text: "German", selectionType: "INCLUDED" }],
|
|
6954
|
+
["german", { id: "de", text: "German", selectionType: "INCLUDED" }],
|
|
6955
|
+
["en", { id: "en", text: "English", selectionType: "INCLUDED" }],
|
|
6956
|
+
["english", { id: "en", text: "English", selectionType: "INCLUDED" }],
|
|
6957
|
+
]);
|
|
6958
|
+
const SALES_NAVIGATOR_SENIORITY_VALUES = new Map([
|
|
6959
|
+
[
|
|
6960
|
+
"owner / partner",
|
|
6961
|
+
{ id: "320", text: "Owner / Partner", selectionType: "INCLUDED" },
|
|
6169
6962
|
],
|
|
6170
6963
|
["owner", { id: "320", text: "Owner / Partner", selectionType: "INCLUDED" }],
|
|
6171
6964
|
[
|
|
@@ -6218,12 +7011,17 @@ const SALES_NAVIGATOR_COMPANY_HEADCOUNT_VALUES = new Map([
|
|
|
6218
7011
|
["11-50", { id: "C", text: "11-50", selectionType: "INCLUDED" }],
|
|
6219
7012
|
["51-200", { id: "D", text: "51-200", selectionType: "INCLUDED" }],
|
|
6220
7013
|
["201-500", { id: "E", text: "201-500", selectionType: "INCLUDED" }],
|
|
6221
|
-
["501-1000", { id: "F", text: "501-
|
|
6222
|
-
["
|
|
6223
|
-
["
|
|
6224
|
-
["
|
|
6225
|
-
["10,000
|
|
6226
|
-
["10000
|
|
7014
|
+
["501-1000", { id: "F", text: "501-1,000", selectionType: "INCLUDED" }],
|
|
7015
|
+
["501-1,000", { id: "F", text: "501-1,000", selectionType: "INCLUDED" }],
|
|
7016
|
+
["1001-5000", { id: "G", text: "1,001-5,000", selectionType: "INCLUDED" }],
|
|
7017
|
+
["1,001-5,000", { id: "G", text: "1,001-5,000", selectionType: "INCLUDED" }],
|
|
7018
|
+
["5001-10,000", { id: "H", text: "5,001-10,000", selectionType: "INCLUDED" }],
|
|
7019
|
+
["5001-10000", { id: "H", text: "5,001-10,000", selectionType: "INCLUDED" }],
|
|
7020
|
+
["5,001-10,000", { id: "H", text: "5,001-10,000", selectionType: "INCLUDED" }],
|
|
7021
|
+
["10,000+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
7022
|
+
["10000+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
7023
|
+
["10,001+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
7024
|
+
["10001+", { id: "I", text: "10,001+", selectionType: "INCLUDED" }],
|
|
6227
7025
|
]);
|
|
6228
7026
|
const SALES_NAVIGATOR_TENURE_VALUES = new Map([
|
|
6229
7027
|
[
|
|
@@ -6447,14 +7245,17 @@ function withSalesNavigatorPaging(url, start, count) {
|
|
|
6447
7245
|
return `${base}?${params.join("&")}${hash}`;
|
|
6448
7246
|
}
|
|
6449
7247
|
function buildLinkedInSalesNavigatorLocalImportHeaders(config) {
|
|
6450
|
-
|
|
7248
|
+
const headers = {
|
|
6451
7249
|
accept: "*/*",
|
|
6452
7250
|
"csrf-token": config.csrfToken,
|
|
6453
7251
|
cookie: config.cookie,
|
|
6454
7252
|
"user-agent": config.userAgent,
|
|
6455
|
-
"x-li-identity": config.identity,
|
|
6456
7253
|
"x-restli-protocol-version": "2.0.0",
|
|
6457
7254
|
};
|
|
7255
|
+
if (config.identity) {
|
|
7256
|
+
headers["x-li-identity"] = config.identity;
|
|
7257
|
+
}
|
|
7258
|
+
return headers;
|
|
6458
7259
|
}
|
|
6459
7260
|
function buildLinkedInSalesNavigatorAccountSearchHeaders(config) {
|
|
6460
7261
|
return {
|
|
@@ -6745,10 +7546,18 @@ function extractLocalSalesNavigatorTotalResults(responseBody) {
|
|
|
6745
7546
|
return null;
|
|
6746
7547
|
}
|
|
6747
7548
|
const record = responseBody;
|
|
7549
|
+
const data = typeof record.data === "object" &&
|
|
7550
|
+
record.data !== null &&
|
|
7551
|
+
!Array.isArray(record.data)
|
|
7552
|
+
? record.data
|
|
7553
|
+
: null;
|
|
6748
7554
|
const candidates = [
|
|
6749
7555
|
record.paging,
|
|
7556
|
+
data?.paging,
|
|
6750
7557
|
record.metadata,
|
|
7558
|
+
data?.metadata,
|
|
6751
7559
|
record.searchResultsMetadata,
|
|
7560
|
+
data,
|
|
6752
7561
|
record,
|
|
6753
7562
|
];
|
|
6754
7563
|
for (const candidate of candidates) {
|
|
@@ -6850,6 +7659,781 @@ async function fetchLocalSalesNavigatorRequest(parsedRequest, retryOptions) {
|
|
|
6850
7659
|
}
|
|
6851
7660
|
}
|
|
6852
7661
|
}
|
|
7662
|
+
async function readLocalAccountSearchRelayBody(request) {
|
|
7663
|
+
const chunks = [];
|
|
7664
|
+
let size = 0;
|
|
7665
|
+
for await (const chunk of request) {
|
|
7666
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
7667
|
+
size += buffer.length;
|
|
7668
|
+
if (size > 50 * 1024 * 1024) {
|
|
7669
|
+
throw new Error("Browser relay response exceeded 50 MB.");
|
|
7670
|
+
}
|
|
7671
|
+
chunks.push(buffer);
|
|
7672
|
+
}
|
|
7673
|
+
const text = Buffer.concat(chunks).toString("utf8");
|
|
7674
|
+
return text ? JSON.parse(text) : null;
|
|
7675
|
+
}
|
|
7676
|
+
function writeLocalAccountSearchRelayJson(response, statusCode, payload) {
|
|
7677
|
+
response.writeHead(statusCode, {
|
|
7678
|
+
"content-type": "application/json; charset=utf-8",
|
|
7679
|
+
"cache-control": "no-store",
|
|
7680
|
+
"access-control-allow-origin": "https://www.linkedin.com",
|
|
7681
|
+
"access-control-allow-methods": "GET, POST, OPTIONS",
|
|
7682
|
+
"access-control-allow-headers": "content-type",
|
|
7683
|
+
"access-control-allow-private-network": "true",
|
|
7684
|
+
});
|
|
7685
|
+
response.end(`${JSON.stringify(payload)}\n`);
|
|
7686
|
+
}
|
|
7687
|
+
async function createLocalAccountSearchBrowserRelay(requestedPort) {
|
|
7688
|
+
let pending = null;
|
|
7689
|
+
let requestSequence = 0;
|
|
7690
|
+
const server = createServer(async (request, response) => {
|
|
7691
|
+
try {
|
|
7692
|
+
const requestUrl = new URL(request.url ?? "/", `http://127.0.0.1:${requestedPort}`);
|
|
7693
|
+
if (request.method === "OPTIONS") {
|
|
7694
|
+
writeLocalAccountSearchRelayJson(response, 204, null);
|
|
7695
|
+
return;
|
|
7696
|
+
}
|
|
7697
|
+
if (request.method === "GET" && requestUrl.pathname === "/health") {
|
|
7698
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7699
|
+
status: "ok",
|
|
7700
|
+
pending: Boolean(pending),
|
|
7701
|
+
});
|
|
7702
|
+
return;
|
|
7703
|
+
}
|
|
7704
|
+
if (request.method === "GET" && requestUrl.pathname === "/next") {
|
|
7705
|
+
writeLocalAccountSearchRelayJson(response, 200, pending
|
|
7706
|
+
? {
|
|
7707
|
+
status: "task",
|
|
7708
|
+
requestId: pending.requestId,
|
|
7709
|
+
url: pending.url,
|
|
7710
|
+
}
|
|
7711
|
+
: { status: "idle" });
|
|
7712
|
+
return;
|
|
7713
|
+
}
|
|
7714
|
+
if (request.method === "POST" && requestUrl.pathname === "/result") {
|
|
7715
|
+
if (!pending) {
|
|
7716
|
+
writeLocalAccountSearchRelayJson(response, 409, {
|
|
7717
|
+
status: "error",
|
|
7718
|
+
error: "No Account Search relay task is pending.",
|
|
7719
|
+
});
|
|
7720
|
+
return;
|
|
7721
|
+
}
|
|
7722
|
+
const value = await readLocalAccountSearchRelayBody(request);
|
|
7723
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
7724
|
+
throw new Error("Browser relay result must be a JSON object.");
|
|
7725
|
+
}
|
|
7726
|
+
const record = value;
|
|
7727
|
+
if (record.requestId !== pending.requestId) {
|
|
7728
|
+
writeLocalAccountSearchRelayJson(response, 409, {
|
|
7729
|
+
status: "error",
|
|
7730
|
+
error: "Browser relay request id does not match the pending task.",
|
|
7731
|
+
});
|
|
7732
|
+
return;
|
|
7733
|
+
}
|
|
7734
|
+
const status = Number(record.status);
|
|
7735
|
+
const retryAfter = typeof record.retryAfter === "string" ? record.retryAfter : null;
|
|
7736
|
+
const body = record.body;
|
|
7737
|
+
const bodyPreview = JSON.stringify(body ?? null).slice(0, 300);
|
|
7738
|
+
const current = pending;
|
|
7739
|
+
pending = null;
|
|
7740
|
+
if (isSalesNavigatorRateLimitStatus(status, bodyPreview)) {
|
|
7741
|
+
current.reject(new LocalSalesNavigatorRateLimitError(status, bodyPreview, parseRetryAfterHeader(retryAfter)));
|
|
7742
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7743
|
+
status: "rate_limited",
|
|
7744
|
+
});
|
|
7745
|
+
return;
|
|
7746
|
+
}
|
|
7747
|
+
if (!Number.isFinite(status) || status < 200 || status >= 300) {
|
|
7748
|
+
current.reject(new Error(`Sales Navigator browser relay failed with HTTP ${status}: ${bodyPreview}`));
|
|
7749
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7750
|
+
status: "failed",
|
|
7751
|
+
});
|
|
7752
|
+
return;
|
|
7753
|
+
}
|
|
7754
|
+
current.resolve({ body, retryCount: 0, retryDelayMs: 0 });
|
|
7755
|
+
writeLocalAccountSearchRelayJson(response, 200, {
|
|
7756
|
+
status: "accepted",
|
|
7757
|
+
});
|
|
7758
|
+
return;
|
|
7759
|
+
}
|
|
7760
|
+
writeLocalAccountSearchRelayJson(response, 404, {
|
|
7761
|
+
status: "error",
|
|
7762
|
+
error: "Not found",
|
|
7763
|
+
});
|
|
7764
|
+
}
|
|
7765
|
+
catch (error) {
|
|
7766
|
+
writeLocalAccountSearchRelayJson(response, 500, {
|
|
7767
|
+
status: "error",
|
|
7768
|
+
error: error instanceof Error ? error.message : String(error),
|
|
7769
|
+
});
|
|
7770
|
+
}
|
|
7771
|
+
});
|
|
7772
|
+
await new Promise((resolve, reject) => {
|
|
7773
|
+
server.once("error", reject);
|
|
7774
|
+
server.listen(requestedPort, "127.0.0.1", () => resolve());
|
|
7775
|
+
});
|
|
7776
|
+
const address = server.address();
|
|
7777
|
+
if (!address || typeof address === "string") {
|
|
7778
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
7779
|
+
throw new Error("Account Search browser relay failed to bind.");
|
|
7780
|
+
}
|
|
7781
|
+
return {
|
|
7782
|
+
port: address.port,
|
|
7783
|
+
request(url) {
|
|
7784
|
+
if (pending) {
|
|
7785
|
+
throw new Error("Account Search browser relay already has a pending task.");
|
|
7786
|
+
}
|
|
7787
|
+
const parsedUrl = new URL(url);
|
|
7788
|
+
if (!/(^|\.)linkedin\.com$/i.test(parsedUrl.hostname) ||
|
|
7789
|
+
!(parsedUrl.pathname.includes("/sales-api/salesApiAccountSearch") ||
|
|
7790
|
+
parsedUrl.pathname.includes("/sales-api/salesApiLeadSearch"))) {
|
|
7791
|
+
throw new Error("Browser relay only accepts LinkedIn Account or Lead Search API URLs.");
|
|
7792
|
+
}
|
|
7793
|
+
requestSequence += 1;
|
|
7794
|
+
return new Promise((resolve, reject) => {
|
|
7795
|
+
pending = {
|
|
7796
|
+
requestId: `sales-nav-search-${requestSequence}`,
|
|
7797
|
+
url,
|
|
7798
|
+
resolve,
|
|
7799
|
+
reject,
|
|
7800
|
+
};
|
|
7801
|
+
});
|
|
7802
|
+
},
|
|
7803
|
+
async close() {
|
|
7804
|
+
if (pending) {
|
|
7805
|
+
pending.reject(new Error("Account Search browser relay closed."));
|
|
7806
|
+
pending = null;
|
|
7807
|
+
}
|
|
7808
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
7809
|
+
},
|
|
7810
|
+
};
|
|
7811
|
+
}
|
|
7812
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_RESULT_WINDOW = 1600;
|
|
7813
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION = 3;
|
|
7814
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS = "coverage-complete";
|
|
7815
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_PASSES = [
|
|
7816
|
+
{
|
|
7817
|
+
name: LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS,
|
|
7818
|
+
filterTypes: [
|
|
7819
|
+
"COMPANY_HEADCOUNT",
|
|
7820
|
+
"NUM_OF_FOLLOWERS",
|
|
7821
|
+
],
|
|
7822
|
+
},
|
|
7823
|
+
{
|
|
7824
|
+
name: "headcount-first",
|
|
7825
|
+
filterTypes: [
|
|
7826
|
+
"COMPANY_HEADCOUNT",
|
|
7827
|
+
"NUM_OF_FOLLOWERS",
|
|
7828
|
+
"ANNUAL_REVENUE",
|
|
7829
|
+
"COMPANY_HEADCOUNT_GROWTH",
|
|
7830
|
+
],
|
|
7831
|
+
},
|
|
7832
|
+
{
|
|
7833
|
+
name: "followers-first",
|
|
7834
|
+
filterTypes: [
|
|
7835
|
+
"NUM_OF_FOLLOWERS",
|
|
7836
|
+
"COMPANY_HEADCOUNT",
|
|
7837
|
+
"ANNUAL_REVENUE",
|
|
7838
|
+
"COMPANY_HEADCOUNT_GROWTH",
|
|
7839
|
+
],
|
|
7840
|
+
},
|
|
7841
|
+
{
|
|
7842
|
+
name: "revenue-first",
|
|
7843
|
+
filterTypes: [
|
|
7844
|
+
"ANNUAL_REVENUE",
|
|
7845
|
+
"COMPANY_HEADCOUNT",
|
|
7846
|
+
"NUM_OF_FOLLOWERS",
|
|
7847
|
+
"COMPANY_HEADCOUNT_GROWTH",
|
|
7848
|
+
],
|
|
7849
|
+
},
|
|
7850
|
+
];
|
|
7851
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_KEYWORD_TOKENS = [
|
|
7852
|
+
"software",
|
|
7853
|
+
"ai",
|
|
7854
|
+
"technology",
|
|
7855
|
+
"data",
|
|
7856
|
+
"platform",
|
|
7857
|
+
"services",
|
|
7858
|
+
"solutions",
|
|
7859
|
+
"digital",
|
|
7860
|
+
"cloud",
|
|
7861
|
+
"business",
|
|
7862
|
+
"internet",
|
|
7863
|
+
"online",
|
|
7864
|
+
"app",
|
|
7865
|
+
"systems",
|
|
7866
|
+
"company",
|
|
7867
|
+
"management",
|
|
7868
|
+
"development",
|
|
7869
|
+
"network",
|
|
7870
|
+
"security",
|
|
7871
|
+
"automation",
|
|
7872
|
+
"marketing",
|
|
7873
|
+
"financial",
|
|
7874
|
+
"media",
|
|
7875
|
+
"consulting",
|
|
7876
|
+
"health",
|
|
7877
|
+
"mobile",
|
|
7878
|
+
"enterprise",
|
|
7879
|
+
"analytics",
|
|
7880
|
+
"engineering",
|
|
7881
|
+
"global",
|
|
7882
|
+
"design",
|
|
7883
|
+
"web",
|
|
7884
|
+
];
|
|
7885
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_GROWTH_DIMENSION = {
|
|
7886
|
+
key: "company-headcount-growth",
|
|
7887
|
+
filterType: "COMPANY_HEADCOUNT_GROWTH",
|
|
7888
|
+
combinableValues: false,
|
|
7889
|
+
values: [
|
|
7890
|
+
{ text: "-1,000,000% to -1%", rangeValue: { min: -1_000_000, max: -1 } },
|
|
7891
|
+
{ text: "0%", rangeValue: { min: 0, max: 0 } },
|
|
7892
|
+
{ text: "1% to 10%", rangeValue: { min: 1, max: 10 } },
|
|
7893
|
+
{ text: "11% to 25%", rangeValue: { min: 11, max: 25 } },
|
|
7894
|
+
{ text: "26% to 50%", rangeValue: { min: 26, max: 50 } },
|
|
7895
|
+
{ text: "51% to 100%", rangeValue: { min: 51, max: 100 } },
|
|
7896
|
+
{ text: "101%+", rangeValue: { min: 101, max: 1_000_000 } },
|
|
7897
|
+
],
|
|
7898
|
+
};
|
|
7899
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_PARENT_REGION = {
|
|
7900
|
+
id: "90000084",
|
|
7901
|
+
text: "San Francisco Bay Area",
|
|
7902
|
+
selectionType: "INCLUDED",
|
|
7903
|
+
};
|
|
7904
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES = [
|
|
7905
|
+
{
|
|
7906
|
+
id: "102839053",
|
|
7907
|
+
text: "Alameda County, California, United States",
|
|
7908
|
+
selectionType: "INCLUDED",
|
|
7909
|
+
},
|
|
7910
|
+
{
|
|
7911
|
+
id: "102476922",
|
|
7912
|
+
text: "Contra Costa County, California, United States",
|
|
7913
|
+
selectionType: "INCLUDED",
|
|
7914
|
+
},
|
|
7915
|
+
{
|
|
7916
|
+
id: "104929029",
|
|
7917
|
+
text: "Marin County, California, United States",
|
|
7918
|
+
selectionType: "INCLUDED",
|
|
7919
|
+
},
|
|
7920
|
+
{
|
|
7921
|
+
id: "105546787",
|
|
7922
|
+
text: "Napa County, California, United States",
|
|
7923
|
+
selectionType: "INCLUDED",
|
|
7924
|
+
},
|
|
7925
|
+
{
|
|
7926
|
+
id: "100901743",
|
|
7927
|
+
text: "San Francisco County, California, United States",
|
|
7928
|
+
selectionType: "INCLUDED",
|
|
7929
|
+
},
|
|
7930
|
+
{
|
|
7931
|
+
id: "104739033",
|
|
7932
|
+
text: "San Mateo County, California, United States",
|
|
7933
|
+
selectionType: "INCLUDED",
|
|
7934
|
+
},
|
|
7935
|
+
{
|
|
7936
|
+
id: "105416315",
|
|
7937
|
+
text: "Santa Clara County, California, United States",
|
|
7938
|
+
selectionType: "INCLUDED",
|
|
7939
|
+
},
|
|
7940
|
+
{
|
|
7941
|
+
id: "103079873",
|
|
7942
|
+
text: "Solano County, California, United States",
|
|
7943
|
+
selectionType: "INCLUDED",
|
|
7944
|
+
},
|
|
7945
|
+
{
|
|
7946
|
+
id: "101079764",
|
|
7947
|
+
text: "Sonoma County, California, United States",
|
|
7948
|
+
selectionType: "INCLUDED",
|
|
7949
|
+
},
|
|
7950
|
+
];
|
|
7951
|
+
const LOCAL_ACCOUNT_SEARCH_COLLECTOR_SAN_FRANCISCO_CITY = {
|
|
7952
|
+
id: "102277331",
|
|
7953
|
+
text: "San Francisco, California, United States",
|
|
7954
|
+
selectionType: "INCLUDED",
|
|
7955
|
+
};
|
|
7956
|
+
function isLocalAccountSearchCollectorNodeTask(value) {
|
|
7957
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
7958
|
+
return false;
|
|
7959
|
+
}
|
|
7960
|
+
const task = value;
|
|
7961
|
+
return (task.kind === "node" &&
|
|
7962
|
+
typeof task.id === "string" &&
|
|
7963
|
+
typeof task.pass === "string" &&
|
|
7964
|
+
Array.isArray(task.filters) &&
|
|
7965
|
+
Array.isArray(task.remainingDimensions));
|
|
7966
|
+
}
|
|
7967
|
+
function localAccountSearchCollectorHasFilterType(filters, filterType) {
|
|
7968
|
+
return filters.some((filter) => filter.type === filterType);
|
|
7969
|
+
}
|
|
7970
|
+
function buildLocalAccountSearchCollectorDimensionPartitions(dimension) {
|
|
7971
|
+
const partitions = dimension.values.map((value, index) => ({
|
|
7972
|
+
idSuffix: String(index),
|
|
7973
|
+
filter: {
|
|
7974
|
+
type: dimension.filterType,
|
|
7975
|
+
values: [{ ...value }],
|
|
7976
|
+
},
|
|
7977
|
+
}));
|
|
7978
|
+
const residualValues = dimension.values
|
|
7979
|
+
.filter((value) => typeof value.id === "string" && value.id.length > 0)
|
|
7980
|
+
.map((value) => ({
|
|
7981
|
+
...value,
|
|
7982
|
+
selectionType: "EXCLUDED",
|
|
7983
|
+
}));
|
|
7984
|
+
if (dimension.supportsResidualExclusion !== false &&
|
|
7985
|
+
residualValues.length === dimension.values.length &&
|
|
7986
|
+
residualValues.length > 0) {
|
|
7987
|
+
partitions.push({
|
|
7988
|
+
idSuffix: "residual",
|
|
7989
|
+
filter: {
|
|
7990
|
+
type: dimension.filterType,
|
|
7991
|
+
values: residualValues,
|
|
7992
|
+
},
|
|
7993
|
+
});
|
|
7994
|
+
}
|
|
7995
|
+
return partitions;
|
|
7996
|
+
}
|
|
7997
|
+
function withLocalAccountSearchCollectorKeywords(searchUrl, keywordExpression) {
|
|
7998
|
+
const url = new URL(searchUrl);
|
|
7999
|
+
const query = url.searchParams.get("query");
|
|
8000
|
+
if (!query) {
|
|
8001
|
+
return null;
|
|
8002
|
+
}
|
|
8003
|
+
let nextQuery;
|
|
8004
|
+
if (/^\(keywords:[\s\S]*?,filters:List\(/.test(query)) {
|
|
8005
|
+
nextQuery = query.replace(/^\(keywords:[\s\S]*?,filters:List\(/, `(keywords:${keywordExpression},filters:List(`);
|
|
8006
|
+
}
|
|
8007
|
+
else if (query.startsWith("(filters:List(")) {
|
|
8008
|
+
nextQuery = `(keywords:${keywordExpression},${query.slice(1)}`;
|
|
8009
|
+
}
|
|
8010
|
+
else {
|
|
8011
|
+
return null;
|
|
8012
|
+
}
|
|
8013
|
+
url.searchParams.set("query", nextQuery);
|
|
8014
|
+
return url.toString();
|
|
8015
|
+
}
|
|
8016
|
+
function buildLocalAccountSearchCollectorKeywordPartitions(searchUrl, task) {
|
|
8017
|
+
const keywordDepth = Math.max(0, Math.trunc(task.keywordDepth ?? 0));
|
|
8018
|
+
const token = LOCAL_ACCOUNT_SEARCH_COLLECTOR_KEYWORD_TOKENS[keywordDepth];
|
|
8019
|
+
if (!token) {
|
|
8020
|
+
return null;
|
|
8021
|
+
}
|
|
8022
|
+
const keywordClauses = Array.isArray(task.keywordClauses)
|
|
8023
|
+
? task.keywordClauses.filter((clause) => clause.trim().length > 0)
|
|
8024
|
+
: [];
|
|
8025
|
+
const branches = [token, `NOT ${token}`].map((clause) => {
|
|
8026
|
+
const nextClauses = [...keywordClauses, clause];
|
|
8027
|
+
return {
|
|
8028
|
+
clause,
|
|
8029
|
+
nextClauses,
|
|
8030
|
+
searchUrl: withLocalAccountSearchCollectorKeywords(searchUrl, nextClauses.join(" AND ")),
|
|
8031
|
+
};
|
|
8032
|
+
});
|
|
8033
|
+
if (branches.some((branch) => !branch.searchUrl)) {
|
|
8034
|
+
return null;
|
|
8035
|
+
}
|
|
8036
|
+
return {
|
|
8037
|
+
splitOn: `KEYWORDS_${token}`,
|
|
8038
|
+
partitions: branches.map((branch, index) => ({
|
|
8039
|
+
idSuffix: index === 0 ? token : `not-${token}`,
|
|
8040
|
+
searchUrl: branch.searchUrl,
|
|
8041
|
+
keywordClauses: branch.nextClauses,
|
|
8042
|
+
keywordDepth: keywordDepth + 1,
|
|
8043
|
+
})),
|
|
8044
|
+
};
|
|
8045
|
+
}
|
|
8046
|
+
function buildLocalAccountSearchCollectorIndustryPartitions(searchUrl) {
|
|
8047
|
+
const url = new URL(searchUrl);
|
|
8048
|
+
const query = url.searchParams.get("query");
|
|
8049
|
+
if (!query) {
|
|
8050
|
+
return null;
|
|
8051
|
+
}
|
|
8052
|
+
const parentIndustryPattern = /\(type:INDUSTRY,values:List\(\(id:6,text:Technology%2C%20Information%20and%20Internet,selectionType:INCLUDED\)\)\)/;
|
|
8053
|
+
if (!parentIndustryPattern.test(query)) {
|
|
8054
|
+
return null;
|
|
8055
|
+
}
|
|
8056
|
+
const industryBlocks = [
|
|
8057
|
+
{
|
|
8058
|
+
idSuffix: "software-development",
|
|
8059
|
+
block: "(type:INDUSTRY,values:List((id:4,text:Software%20Development,selectionType:INCLUDED)))",
|
|
8060
|
+
},
|
|
8061
|
+
{
|
|
8062
|
+
idSuffix: "technology-internet-residual",
|
|
8063
|
+
block: "(type:INDUSTRY,values:List((id:6,text:Technology%2C%20Information%20and%20Internet,selectionType:INCLUDED),(id:4,text:Software%20Development,selectionType:EXCLUDED)))",
|
|
8064
|
+
},
|
|
8065
|
+
];
|
|
8066
|
+
return {
|
|
8067
|
+
splitOn: "INDUSTRY_SOFTWARE_DEVELOPMENT",
|
|
8068
|
+
partitions: industryBlocks.map((partition) => {
|
|
8069
|
+
const childUrl = new URL(url);
|
|
8070
|
+
childUrl.searchParams.set("query", query.replace(parentIndustryPattern, partition.block));
|
|
8071
|
+
return {
|
|
8072
|
+
idSuffix: partition.idSuffix,
|
|
8073
|
+
searchUrl: childUrl.toString(),
|
|
8074
|
+
};
|
|
8075
|
+
}),
|
|
8076
|
+
};
|
|
8077
|
+
}
|
|
8078
|
+
function isSanFranciscoBayAreaAccountSearch(sourceQueryUrl) {
|
|
8079
|
+
const query = new URL(sourceQueryUrl).searchParams.get("query") ?? "";
|
|
8080
|
+
return (query.includes("type:REGION") &&
|
|
8081
|
+
query.includes(`id:${LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_PARENT_REGION.id}`));
|
|
8082
|
+
}
|
|
8083
|
+
function buildLocalAccountSearchCollectorBayAreaPartitions(sourceQueryUrl, filters) {
|
|
8084
|
+
if (!isSanFranciscoBayAreaAccountSearch(sourceQueryUrl)) {
|
|
8085
|
+
return null;
|
|
8086
|
+
}
|
|
8087
|
+
const regionFilter = filters.find((filter) => filter.type === "REGION");
|
|
8088
|
+
if (!regionFilter) {
|
|
8089
|
+
const countyPartitions = LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES.map((county) => [
|
|
8090
|
+
...filters,
|
|
8091
|
+
{ type: "REGION", values: [{ ...county }] },
|
|
8092
|
+
]);
|
|
8093
|
+
const residualPartition = [
|
|
8094
|
+
...filters,
|
|
8095
|
+
{
|
|
8096
|
+
type: "REGION",
|
|
8097
|
+
values: [
|
|
8098
|
+
{ ...LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_PARENT_REGION },
|
|
8099
|
+
...LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES.map((county) => ({
|
|
8100
|
+
...county,
|
|
8101
|
+
selectionType: "EXCLUDED",
|
|
8102
|
+
})),
|
|
8103
|
+
],
|
|
8104
|
+
},
|
|
8105
|
+
];
|
|
8106
|
+
return {
|
|
8107
|
+
splitOn: "REGION_BAY_AREA_COUNTY",
|
|
8108
|
+
partitions: [...countyPartitions, residualPartition],
|
|
8109
|
+
};
|
|
8110
|
+
}
|
|
8111
|
+
const includedRegions = regionFilter.values.filter((value) => (value.selectionType ?? "INCLUDED") === "INCLUDED");
|
|
8112
|
+
const excludedRegions = regionFilter.values.filter((value) => value.selectionType === "EXCLUDED");
|
|
8113
|
+
const sanFranciscoCounty = LOCAL_ACCOUNT_SEARCH_COLLECTOR_BAY_AREA_COUNTIES.find((county) => county.id === "100901743");
|
|
8114
|
+
if (sanFranciscoCounty &&
|
|
8115
|
+
includedRegions.length === 1 &&
|
|
8116
|
+
includedRegions[0]?.id === sanFranciscoCounty.id &&
|
|
8117
|
+
excludedRegions.length === 0) {
|
|
8118
|
+
return {
|
|
8119
|
+
splitOn: "REGION_SAN_FRANCISCO_CITY",
|
|
8120
|
+
partitions: [
|
|
8121
|
+
[
|
|
8122
|
+
...filters.filter((filter) => filter.type !== "REGION"),
|
|
8123
|
+
{
|
|
8124
|
+
type: "REGION",
|
|
8125
|
+
values: [{ ...LOCAL_ACCOUNT_SEARCH_COLLECTOR_SAN_FRANCISCO_CITY }],
|
|
8126
|
+
},
|
|
8127
|
+
],
|
|
8128
|
+
[
|
|
8129
|
+
...filters.filter((filter) => filter.type !== "REGION"),
|
|
8130
|
+
{
|
|
8131
|
+
type: "REGION",
|
|
8132
|
+
values: [
|
|
8133
|
+
{ ...sanFranciscoCounty },
|
|
8134
|
+
{
|
|
8135
|
+
...LOCAL_ACCOUNT_SEARCH_COLLECTOR_SAN_FRANCISCO_CITY,
|
|
8136
|
+
selectionType: "EXCLUDED",
|
|
8137
|
+
},
|
|
8138
|
+
],
|
|
8139
|
+
},
|
|
8140
|
+
],
|
|
8141
|
+
],
|
|
8142
|
+
};
|
|
8143
|
+
}
|
|
8144
|
+
return null;
|
|
8145
|
+
}
|
|
8146
|
+
function migrateLocalAccountSearchCollectorCheckpoint(checkpoint) {
|
|
8147
|
+
let changed = false;
|
|
8148
|
+
const growthFilterType = LOCAL_ACCOUNT_SEARCH_COLLECTOR_GROWTH_DIMENSION.filterType;
|
|
8149
|
+
const needsCoverageMigration = checkpoint.coveragePassVersion !==
|
|
8150
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION;
|
|
8151
|
+
if (needsCoverageMigration) {
|
|
8152
|
+
checkpoint.taskQueue = checkpoint.taskQueue.filter((task) => !task.pass.startsWith("coverage-"));
|
|
8153
|
+
checkpoint.seenQueries = checkpoint.seenQueries.filter((queryKey) => !queryKey.startsWith("coverage-"));
|
|
8154
|
+
checkpoint.coveragePassSeeded = false;
|
|
8155
|
+
checkpoint.coveragePassVersion =
|
|
8156
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION;
|
|
8157
|
+
changed = true;
|
|
8158
|
+
}
|
|
8159
|
+
for (const task of checkpoint.taskQueue) {
|
|
8160
|
+
if (task.kind === "node" && task.pass.startsWith("coverage-")) {
|
|
8161
|
+
const remainingDimensions = task.remainingDimensions.filter((filterType) => filterType !== growthFilterType);
|
|
8162
|
+
if (remainingDimensions.length !== task.remainingDimensions.length) {
|
|
8163
|
+
task.remainingDimensions = remainingDimensions;
|
|
8164
|
+
changed = true;
|
|
8165
|
+
}
|
|
8166
|
+
continue;
|
|
8167
|
+
}
|
|
8168
|
+
if (task.kind === "node" &&
|
|
8169
|
+
!localAccountSearchCollectorHasFilterType(task.filters, growthFilterType) &&
|
|
8170
|
+
!task.remainingDimensions.includes(growthFilterType)) {
|
|
8171
|
+
task.remainingDimensions.push(growthFilterType);
|
|
8172
|
+
changed = true;
|
|
8173
|
+
}
|
|
8174
|
+
}
|
|
8175
|
+
const queuedNodeQueries = new Set(checkpoint.taskQueue
|
|
8176
|
+
.filter(isLocalAccountSearchCollectorNodeTask)
|
|
8177
|
+
.map((task) => localAccountSearchCollectorQueryKey(buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, task.filters).slicedQueryUrl)));
|
|
8178
|
+
const remainingFailures = [];
|
|
8179
|
+
for (const failure of checkpoint.failures) {
|
|
8180
|
+
const error = typeof failure.error === "string" ? failure.error : "";
|
|
8181
|
+
const task = failure.task;
|
|
8182
|
+
const isTerminalSlice = error.includes("with no split dimension left");
|
|
8183
|
+
if (!isTerminalSlice || !isLocalAccountSearchCollectorNodeTask(task)) {
|
|
8184
|
+
remainingFailures.push(failure);
|
|
8185
|
+
continue;
|
|
8186
|
+
}
|
|
8187
|
+
const hasGrowthFilter = localAccountSearchCollectorHasFilterType(task.filters, growthFilterType);
|
|
8188
|
+
const canUseBayAreaFallback = Boolean(buildLocalAccountSearchCollectorBayAreaPartitions(checkpoint.sourceQueryUrl, task.filters));
|
|
8189
|
+
if (hasGrowthFilter && !canUseBayAreaFallback) {
|
|
8190
|
+
remainingFailures.push(failure);
|
|
8191
|
+
continue;
|
|
8192
|
+
}
|
|
8193
|
+
const remainingDimensions = [...task.remainingDimensions];
|
|
8194
|
+
if (!hasGrowthFilter && !remainingDimensions.includes(growthFilterType)) {
|
|
8195
|
+
remainingDimensions.push(growthFilterType);
|
|
8196
|
+
}
|
|
8197
|
+
const queryKey = localAccountSearchCollectorQueryKey(buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, task.filters).slicedQueryUrl);
|
|
8198
|
+
if (!queuedNodeQueries.has(queryKey)) {
|
|
8199
|
+
checkpoint.taskQueue.push({
|
|
8200
|
+
...task,
|
|
8201
|
+
filters: task.filters.map((filter) => ({
|
|
8202
|
+
...filter,
|
|
8203
|
+
values: filter.values.map((value) => ({ ...value })),
|
|
8204
|
+
})),
|
|
8205
|
+
remainingDimensions,
|
|
8206
|
+
});
|
|
8207
|
+
queuedNodeQueries.add(queryKey);
|
|
8208
|
+
}
|
|
8209
|
+
changed = true;
|
|
8210
|
+
}
|
|
8211
|
+
if (remainingFailures.length !== checkpoint.failures.length) {
|
|
8212
|
+
checkpoint.failures = remainingFailures;
|
|
8213
|
+
}
|
|
8214
|
+
const isFreshCheckpoint = checkpoint.taskQueue.length === 0 && checkpoint.seenQueries.length === 0;
|
|
8215
|
+
if (!isFreshCheckpoint &&
|
|
8216
|
+
needsCoverageMigration &&
|
|
8217
|
+
(checkpoint.rootTotal ?? 0) > checkpoint.cap) {
|
|
8218
|
+
seedLocalAccountSearchCollectorPassTasks(checkpoint, LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS, true);
|
|
8219
|
+
checkpoint.coveragePassSeeded = true;
|
|
8220
|
+
changed = true;
|
|
8221
|
+
}
|
|
8222
|
+
const coverageTasks = checkpoint.taskQueue.filter((task) => task.pass.startsWith("coverage-"));
|
|
8223
|
+
if (coverageTasks.length > 0 &&
|
|
8224
|
+
checkpoint.taskQueue.some((task, index) => task.pass.startsWith("coverage-") && index >= coverageTasks.length)) {
|
|
8225
|
+
checkpoint.taskQueue = [
|
|
8226
|
+
...coverageTasks,
|
|
8227
|
+
...checkpoint.taskQueue.filter((task) => !task.pass.startsWith("coverage-")),
|
|
8228
|
+
];
|
|
8229
|
+
changed = true;
|
|
8230
|
+
}
|
|
8231
|
+
return changed;
|
|
8232
|
+
}
|
|
8233
|
+
function parseLocalAccountSearchCollectorCheckpoint(value) {
|
|
8234
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
8235
|
+
throw new Error("Account Search checkpoint must be a JSON object.");
|
|
8236
|
+
}
|
|
8237
|
+
const checkpoint = value;
|
|
8238
|
+
if (checkpoint.version !== 1 ||
|
|
8239
|
+
typeof checkpoint.sourceQueryUrl !== "string" ||
|
|
8240
|
+
!Array.isArray(checkpoint.taskQueue) ||
|
|
8241
|
+
!Array.isArray(checkpoint.leaves) ||
|
|
8242
|
+
!Array.isArray(checkpoint.windows) ||
|
|
8243
|
+
!Array.isArray(checkpoint.failures)) {
|
|
8244
|
+
throw new Error("Unsupported or invalid Account Search checkpoint.");
|
|
8245
|
+
}
|
|
8246
|
+
const seenQueryValues = Array.isArray(checkpoint.seenQueries)
|
|
8247
|
+
? checkpoint.seenQueries
|
|
8248
|
+
: checkpoint.seenQueries && typeof checkpoint.seenQueries === "object"
|
|
8249
|
+
? Object.keys(checkpoint.seenQueries)
|
|
8250
|
+
: [];
|
|
8251
|
+
return {
|
|
8252
|
+
...checkpoint,
|
|
8253
|
+
seenQueries: seenQueryValues.map(localAccountSearchCollectorQueryKey),
|
|
8254
|
+
};
|
|
8255
|
+
}
|
|
8256
|
+
function localAccountSearchCollectorQueryKey(searchUrlOrQuery) {
|
|
8257
|
+
try {
|
|
8258
|
+
const query = new URL(searchUrlOrQuery).searchParams.get("query");
|
|
8259
|
+
return query ?? searchUrlOrQuery;
|
|
8260
|
+
}
|
|
8261
|
+
catch {
|
|
8262
|
+
return searchUrlOrQuery;
|
|
8263
|
+
}
|
|
8264
|
+
}
|
|
8265
|
+
function localAccountSearchCollectorSeenKey(pass, searchUrlOrQuery) {
|
|
8266
|
+
const queryKey = localAccountSearchCollectorQueryKey(searchUrlOrQuery);
|
|
8267
|
+
return pass.startsWith("coverage-") ? `${pass}:${queryKey}` : queryKey;
|
|
8268
|
+
}
|
|
8269
|
+
function createLocalAccountSearchCollectorCheckpoint(params) {
|
|
8270
|
+
const now = new Date().toISOString();
|
|
8271
|
+
return {
|
|
8272
|
+
version: 1,
|
|
8273
|
+
sourceQueryUrl: params.sourceQueryUrl,
|
|
8274
|
+
rootTotal: params.rootTotal,
|
|
8275
|
+
cap: params.cap,
|
|
8276
|
+
pageSize: params.pageSize,
|
|
8277
|
+
delayMinMs: params.delayMinMs,
|
|
8278
|
+
delayMaxMs: params.delayMaxMs,
|
|
8279
|
+
requests: 0,
|
|
8280
|
+
pagesWritten: 0,
|
|
8281
|
+
rowsWritten: 0,
|
|
8282
|
+
startedAt: now,
|
|
8283
|
+
updatedAt: now,
|
|
8284
|
+
taskQueue: [],
|
|
8285
|
+
seenQueries: [],
|
|
8286
|
+
duplicateQueries: 0,
|
|
8287
|
+
leaves: [],
|
|
8288
|
+
windows: [],
|
|
8289
|
+
failures: [],
|
|
8290
|
+
rateLimited: null,
|
|
8291
|
+
coveragePassSeeded: false,
|
|
8292
|
+
coveragePassVersion: LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION,
|
|
8293
|
+
distinctAccounts: 0,
|
|
8294
|
+
completedByDistinctCount: false,
|
|
8295
|
+
};
|
|
8296
|
+
}
|
|
8297
|
+
function seedLocalAccountSearchCollectorTasks(checkpoint) {
|
|
8298
|
+
for (const pass of LOCAL_ACCOUNT_SEARCH_COLLECTOR_PASSES) {
|
|
8299
|
+
seedLocalAccountSearchCollectorPassTasks(checkpoint, pass.name, false);
|
|
8300
|
+
}
|
|
8301
|
+
checkpoint.coveragePassSeeded = true;
|
|
8302
|
+
checkpoint.coveragePassVersion =
|
|
8303
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION;
|
|
8304
|
+
}
|
|
8305
|
+
function seedLocalAccountSearchCollectorPassTasks(checkpoint, passName, prepend) {
|
|
8306
|
+
const dimensions = new Map(DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS.map((dimension) => [
|
|
8307
|
+
dimension.filterType,
|
|
8308
|
+
dimension,
|
|
8309
|
+
]));
|
|
8310
|
+
const seenQueries = new Set(checkpoint.seenQueries);
|
|
8311
|
+
const pass = LOCAL_ACCOUNT_SEARCH_COLLECTOR_PASSES.find((candidate) => candidate.name === passName);
|
|
8312
|
+
if (!pass) {
|
|
8313
|
+
return false;
|
|
8314
|
+
}
|
|
8315
|
+
const [firstFilterType, ...remainingDimensions] = pass.filterTypes;
|
|
8316
|
+
const dimension = dimensions.get(firstFilterType);
|
|
8317
|
+
if (!dimension) {
|
|
8318
|
+
throw new Error(`Missing Account Search split dimension ${firstFilterType}.`);
|
|
8319
|
+
}
|
|
8320
|
+
const tasks = [];
|
|
8321
|
+
for (const partition of buildLocalAccountSearchCollectorDimensionPartitions(dimension)) {
|
|
8322
|
+
const filters = [partition.filter];
|
|
8323
|
+
const searchUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, filters).slicedQueryUrl;
|
|
8324
|
+
const queryKey = localAccountSearchCollectorSeenKey(pass.name, searchUrl);
|
|
8325
|
+
if (seenQueries.has(queryKey)) {
|
|
8326
|
+
continue;
|
|
8327
|
+
}
|
|
8328
|
+
seenQueries.add(queryKey);
|
|
8329
|
+
tasks.push({
|
|
8330
|
+
kind: "node",
|
|
8331
|
+
id: `${pass.name}-${partition.idSuffix}`,
|
|
8332
|
+
pass: pass.name,
|
|
8333
|
+
filters,
|
|
8334
|
+
remainingDimensions: [...remainingDimensions],
|
|
8335
|
+
});
|
|
8336
|
+
}
|
|
8337
|
+
if (prepend) {
|
|
8338
|
+
checkpoint.taskQueue.unshift(...tasks);
|
|
8339
|
+
}
|
|
8340
|
+
else {
|
|
8341
|
+
checkpoint.taskQueue.push(...tasks);
|
|
8342
|
+
}
|
|
8343
|
+
checkpoint.seenQueries = [...seenQueries];
|
|
8344
|
+
return tasks.length > 0;
|
|
8345
|
+
}
|
|
8346
|
+
function enqueueLocalAccountSearchCollectorTasks(checkpoint, parentTask, tasks) {
|
|
8347
|
+
if (tasks.length === 0) {
|
|
8348
|
+
return;
|
|
8349
|
+
}
|
|
8350
|
+
if (parentTask.pass.startsWith("coverage-")) {
|
|
8351
|
+
checkpoint.taskQueue.splice(1, 0, ...tasks);
|
|
8352
|
+
return;
|
|
8353
|
+
}
|
|
8354
|
+
checkpoint.taskQueue.push(...tasks);
|
|
8355
|
+
}
|
|
8356
|
+
function extractLocalSalesNavigatorElements(body) {
|
|
8357
|
+
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
8358
|
+
return [];
|
|
8359
|
+
}
|
|
8360
|
+
const record = body;
|
|
8361
|
+
const elements = record.elements;
|
|
8362
|
+
if (Array.isArray(elements)) {
|
|
8363
|
+
return elements;
|
|
8364
|
+
}
|
|
8365
|
+
const data = typeof record.data === "object" &&
|
|
8366
|
+
record.data !== null &&
|
|
8367
|
+
!Array.isArray(record.data)
|
|
8368
|
+
? record.data
|
|
8369
|
+
: null;
|
|
8370
|
+
const nestedElements = data?.elements;
|
|
8371
|
+
if (Array.isArray(nestedElements)) {
|
|
8372
|
+
return nestedElements;
|
|
8373
|
+
}
|
|
8374
|
+
const references = data?.["*elements"];
|
|
8375
|
+
const included = record.included;
|
|
8376
|
+
if (!Array.isArray(references) || !Array.isArray(included)) {
|
|
8377
|
+
return [];
|
|
8378
|
+
}
|
|
8379
|
+
const referencedUrns = new Set(references.filter((reference) => typeof reference === "string" && reference.length > 0));
|
|
8380
|
+
return included.filter((value) => {
|
|
8381
|
+
if (typeof value !== "object" ||
|
|
8382
|
+
value === null ||
|
|
8383
|
+
Array.isArray(value)) {
|
|
8384
|
+
return false;
|
|
8385
|
+
}
|
|
8386
|
+
const entityUrn = value.entityUrn;
|
|
8387
|
+
return typeof entityUrn === "string" && referencedUrns.has(entityUrn);
|
|
8388
|
+
});
|
|
8389
|
+
}
|
|
8390
|
+
async function appendLocalAccountSearchRawElements(params) {
|
|
8391
|
+
if (params.elements.length === 0) {
|
|
8392
|
+
return;
|
|
8393
|
+
}
|
|
8394
|
+
await mkdir(path.dirname(params.rawOutputPath), { recursive: true });
|
|
8395
|
+
await appendFile(params.rawOutputPath, `${params.elements
|
|
8396
|
+
.map((element) => JSON.stringify({ queryUrl: params.searchUrl, element }))
|
|
8397
|
+
.join("\n")}\n`, "utf8");
|
|
8398
|
+
}
|
|
8399
|
+
function addLocalAccountSearchElementKey(accountKeys, element, queryUrl) {
|
|
8400
|
+
if (typeof element !== "object" || element === null || Array.isArray(element)) {
|
|
8401
|
+
return;
|
|
8402
|
+
}
|
|
8403
|
+
const companyId = extractLinkedInSalesCompanyIdFromUrn(element.entityUrn);
|
|
8404
|
+
if (companyId) {
|
|
8405
|
+
accountKeys.add(`company-id:${companyId}`);
|
|
8406
|
+
return;
|
|
8407
|
+
}
|
|
8408
|
+
const account = normalizeLocalSalesNavigatorAccount(element, queryUrl);
|
|
8409
|
+
if (account) {
|
|
8410
|
+
accountKeys.add(account.accountKey);
|
|
8411
|
+
}
|
|
8412
|
+
}
|
|
8413
|
+
async function loadLocalAccountSearchRawAccountKeys(rawOutputPath) {
|
|
8414
|
+
const accountKeys = new Set();
|
|
8415
|
+
if (!(await fileExists(rawOutputPath))) {
|
|
8416
|
+
return accountKeys;
|
|
8417
|
+
}
|
|
8418
|
+
const raw = await readFile(rawOutputPath, "utf8");
|
|
8419
|
+
for (const line of raw.split(/\r?\n/)) {
|
|
8420
|
+
if (!line.trim()) {
|
|
8421
|
+
continue;
|
|
8422
|
+
}
|
|
8423
|
+
try {
|
|
8424
|
+
const row = JSON.parse(line);
|
|
8425
|
+
addLocalAccountSearchElementKey(accountKeys, row.element, typeof row.queryUrl === "string" ? row.queryUrl : "");
|
|
8426
|
+
}
|
|
8427
|
+
catch {
|
|
8428
|
+
// The import command reports malformed raw rows with line-level context.
|
|
8429
|
+
}
|
|
8430
|
+
}
|
|
8431
|
+
return accountKeys;
|
|
8432
|
+
}
|
|
8433
|
+
async function saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint) {
|
|
8434
|
+
checkpoint.updatedAt = new Date().toISOString();
|
|
8435
|
+
await writeJsonFile(checkpointPath, checkpoint);
|
|
8436
|
+
}
|
|
6853
8437
|
async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
|
|
6854
8438
|
const people = [];
|
|
6855
8439
|
const seen = new Set();
|
|
@@ -6873,7 +8457,9 @@ async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
|
|
|
6873
8457
|
...parsedRequest,
|
|
6874
8458
|
url: withSalesNavigatorPaging(parsedRequest.url, start, currentCount),
|
|
6875
8459
|
};
|
|
6876
|
-
const response =
|
|
8460
|
+
const response = options.executeRequest
|
|
8461
|
+
? await options.executeRequest(pageRequest)
|
|
8462
|
+
: await fetchLocalSalesNavigatorRequest(pageRequest, options.retry);
|
|
6877
8463
|
retryCount += response.retryCount;
|
|
6878
8464
|
totalDelayMs += response.retryDelayMs;
|
|
6879
8465
|
const responseBody = response.body;
|
|
@@ -6919,6 +8505,7 @@ async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
|
|
|
6919
8505
|
async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
6920
8506
|
const accounts = [];
|
|
6921
8507
|
const seen = new Set();
|
|
8508
|
+
let filteredAccounts = 0;
|
|
6922
8509
|
let totalResults = null;
|
|
6923
8510
|
let fetchedPages = 0;
|
|
6924
8511
|
let totalDelayMs = 0;
|
|
@@ -6993,13 +8580,20 @@ async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
|
6993
8580
|
}
|
|
6994
8581
|
const pageAccounts = normalizeLocalSalesNavigatorAccounts(responseBody, pageRequest.url);
|
|
6995
8582
|
let added = 0;
|
|
8583
|
+
let addedOrFiltered = 0;
|
|
6996
8584
|
for (const account of pageAccounts) {
|
|
6997
8585
|
if (seen.has(account.accountKey)) {
|
|
6998
8586
|
continue;
|
|
6999
8587
|
}
|
|
7000
8588
|
seen.add(account.accountKey);
|
|
8589
|
+
if (!salesNavigatorAccountMatchesLocationFilter(account, options.accountLocationFilter)) {
|
|
8590
|
+
filteredAccounts += 1;
|
|
8591
|
+
addedOrFiltered += 1;
|
|
8592
|
+
continue;
|
|
8593
|
+
}
|
|
7001
8594
|
accounts.push(account);
|
|
7002
8595
|
added += 1;
|
|
8596
|
+
addedOrFiltered += 1;
|
|
7003
8597
|
if (requestedAccounts != null && accounts.length >= requestedAccounts) {
|
|
7004
8598
|
break;
|
|
7005
8599
|
}
|
|
@@ -7024,7 +8618,7 @@ async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
|
7024
8618
|
nextStart = start;
|
|
7025
8619
|
break;
|
|
7026
8620
|
}
|
|
7027
|
-
if (
|
|
8621
|
+
if (addedOrFiltered === 0) {
|
|
7028
8622
|
stopReason = "duplicate_page";
|
|
7029
8623
|
stoppedAtStart = start;
|
|
7030
8624
|
stoppedAtCount = currentCount;
|
|
@@ -7048,6 +8642,7 @@ async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
|
|
|
7048
8642
|
const missingFromReported = totalResults == null ? null : Math.max(0, totalResults - fetchedThrough);
|
|
7049
8643
|
return {
|
|
7050
8644
|
accounts,
|
|
8645
|
+
filteredAccounts,
|
|
7051
8646
|
totalResults,
|
|
7052
8647
|
fetchedPages,
|
|
7053
8648
|
sourceQueryUrl: parsedRequest.url,
|
|
@@ -7129,6 +8724,13 @@ async function importLocalSalesNavigatorPeopleViaApp(session, payload) {
|
|
|
7129
8724
|
}, SalesNavigatorLocalImportResponseSchema);
|
|
7130
8725
|
return value;
|
|
7131
8726
|
}
|
|
8727
|
+
const SALESPROMPTER_SUPABASE_TABLES = {
|
|
8728
|
+
accountSearchRuns: "salesprompter_linkedin_sales_nav_account_search_runs",
|
|
8729
|
+
accountSearchSlices: "salesprompter_linkedin_sales_nav_account_search_slices",
|
|
8730
|
+
accountSearchAccounts: "salesprompter_linkedin_sales_nav_accounts",
|
|
8731
|
+
accountRunMemberships: "salesprompter_linkedin_sales_nav_account_run_memberships",
|
|
8732
|
+
linkedinCompaniesProcessed: "salesprompter_linkedin_companies_processed",
|
|
8733
|
+
};
|
|
7132
8734
|
async function createSalesNavigatorAccountSearchStore() {
|
|
7133
8735
|
const databaseUrl = process.env.SALESPROMPTER_DATABASE_URL?.trim() ||
|
|
7134
8736
|
process.env.DATABASE_URL?.trim() ||
|
|
@@ -7159,7 +8761,7 @@ async function closeSalesNavigatorAccountSearchStore(store) {
|
|
|
7159
8761
|
}
|
|
7160
8762
|
async function createSalesNavigatorAccountSearchRun(params) {
|
|
7161
8763
|
if (params.store.kind === "postgres") {
|
|
7162
|
-
const result = await params.store.client.query(`INSERT INTO public.
|
|
8764
|
+
const result = await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_account_search_runs (
|
|
7163
8765
|
org_id, source_query_url, slice_preset, max_results_per_search, page_size, status, raw_payload
|
|
7164
8766
|
)
|
|
7165
8767
|
VALUES ($1, $2, $3, $4, $5, 'running', $6::jsonb)
|
|
@@ -7173,12 +8775,12 @@ async function createSalesNavigatorAccountSearchRun(params) {
|
|
|
7173
8775
|
]);
|
|
7174
8776
|
const id = result.rows[0]?.id ?? "";
|
|
7175
8777
|
if (!id) {
|
|
7176
|
-
throw new Error("Failed to create account search run in
|
|
8778
|
+
throw new Error("Failed to create account search run in Supabase: missing run id.");
|
|
7177
8779
|
}
|
|
7178
8780
|
return id;
|
|
7179
8781
|
}
|
|
7180
8782
|
const { data, error } = await params.store.client
|
|
7181
|
-
.from(
|
|
8783
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchRuns)
|
|
7182
8784
|
.insert({
|
|
7183
8785
|
org_id: params.orgId,
|
|
7184
8786
|
source_query_url: params.sourceQueryUrl,
|
|
@@ -7191,17 +8793,17 @@ async function createSalesNavigatorAccountSearchRun(params) {
|
|
|
7191
8793
|
.select("id")
|
|
7192
8794
|
.single();
|
|
7193
8795
|
if (error) {
|
|
7194
|
-
throw new Error(`Failed to create account search run in
|
|
8796
|
+
throw new Error(`Failed to create account search run in Supabase: ${error.message}`);
|
|
7195
8797
|
}
|
|
7196
8798
|
const id = typeof data?.id === "string" ? data.id : "";
|
|
7197
8799
|
if (!id) {
|
|
7198
|
-
throw new Error("Failed to create account search run in
|
|
8800
|
+
throw new Error("Failed to create account search run in Supabase: missing run id.");
|
|
7199
8801
|
}
|
|
7200
8802
|
return id;
|
|
7201
8803
|
}
|
|
7202
8804
|
async function updateSalesNavigatorAccountSearchRun(params) {
|
|
7203
8805
|
if (params.store.kind === "postgres") {
|
|
7204
|
-
await params.store.client.query(`UPDATE public.
|
|
8806
|
+
await params.store.client.query(`UPDATE public.salesprompter_linkedin_sales_nav_account_search_runs
|
|
7205
8807
|
SET status = $2,
|
|
7206
8808
|
total_slices = $3,
|
|
7207
8809
|
stored_accounts = $4,
|
|
@@ -7234,12 +8836,12 @@ async function updateSalesNavigatorAccountSearchRun(params) {
|
|
|
7234
8836
|
let mergedRawPayload = null;
|
|
7235
8837
|
if (params.rawPayload) {
|
|
7236
8838
|
const { data: existing, error: selectError } = await params.store.client
|
|
7237
|
-
.from(
|
|
8839
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchRuns)
|
|
7238
8840
|
.select("raw_payload")
|
|
7239
8841
|
.eq("id", params.runId)
|
|
7240
8842
|
.maybeSingle();
|
|
7241
8843
|
if (selectError) {
|
|
7242
|
-
throw new Error(`Failed to read account search run payload in
|
|
8844
|
+
throw new Error(`Failed to read account search run payload in Supabase: ${selectError.message}`);
|
|
7243
8845
|
}
|
|
7244
8846
|
const existingPayload = typeof existing?.raw_payload === "object" &&
|
|
7245
8847
|
existing.raw_payload !== null &&
|
|
@@ -7252,7 +8854,7 @@ async function updateSalesNavigatorAccountSearchRun(params) {
|
|
|
7252
8854
|
};
|
|
7253
8855
|
}
|
|
7254
8856
|
const { error } = await params.store.client
|
|
7255
|
-
.from(
|
|
8857
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchRuns)
|
|
7256
8858
|
.update({
|
|
7257
8859
|
status: params.status,
|
|
7258
8860
|
total_slices: params.totalSlices,
|
|
@@ -7272,12 +8874,12 @@ async function updateSalesNavigatorAccountSearchRun(params) {
|
|
|
7272
8874
|
})
|
|
7273
8875
|
.eq("id", params.runId);
|
|
7274
8876
|
if (error) {
|
|
7275
|
-
throw new Error(`Failed to update account search run in
|
|
8877
|
+
throw new Error(`Failed to update account search run in Supabase: ${error.message}`);
|
|
7276
8878
|
}
|
|
7277
8879
|
}
|
|
7278
8880
|
async function upsertSalesNavigatorAccountSearchSlice(params) {
|
|
7279
8881
|
if (params.store.kind === "postgres") {
|
|
7280
|
-
const result = await params.store.client.query(`INSERT INTO public.
|
|
8882
|
+
const result = await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_account_search_slices (
|
|
7281
8883
|
run_id, org_id, sliced_query_url, applied_filters, split_trail, depth, status,
|
|
7282
8884
|
total_results, fetched_pages, stored_accounts, retry_count, retry_delay_ms, last_error, raw_payload
|
|
7283
8885
|
)
|
|
@@ -7313,12 +8915,12 @@ async function upsertSalesNavigatorAccountSearchSlice(params) {
|
|
|
7313
8915
|
]);
|
|
7314
8916
|
const id = result.rows[0]?.id ?? "";
|
|
7315
8917
|
if (!id) {
|
|
7316
|
-
throw new Error("Failed to upsert account search slice in
|
|
8918
|
+
throw new Error("Failed to upsert account search slice in Supabase: missing slice id.");
|
|
7317
8919
|
}
|
|
7318
8920
|
return id;
|
|
7319
8921
|
}
|
|
7320
8922
|
const { data, error } = await params.store.client
|
|
7321
|
-
.from(
|
|
8923
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchSlices)
|
|
7322
8924
|
.upsert({
|
|
7323
8925
|
run_id: params.runId,
|
|
7324
8926
|
org_id: params.orgId,
|
|
@@ -7338,11 +8940,11 @@ async function upsertSalesNavigatorAccountSearchSlice(params) {
|
|
|
7338
8940
|
.select("id")
|
|
7339
8941
|
.single();
|
|
7340
8942
|
if (error) {
|
|
7341
|
-
throw new Error(`Failed to upsert account search slice in
|
|
8943
|
+
throw new Error(`Failed to upsert account search slice in Supabase: ${error.message}`);
|
|
7342
8944
|
}
|
|
7343
8945
|
const id = typeof data?.id === "string" ? data.id : "";
|
|
7344
8946
|
if (!id) {
|
|
7345
|
-
throw new Error("Failed to upsert account search slice in
|
|
8947
|
+
throw new Error("Failed to upsert account search slice in Supabase: missing slice id.");
|
|
7346
8948
|
}
|
|
7347
8949
|
return id;
|
|
7348
8950
|
}
|
|
@@ -7821,7 +9423,7 @@ async function upsertLocalLinkedInCompanyScrape(row) {
|
|
|
7821
9423
|
let committed = false;
|
|
7822
9424
|
try {
|
|
7823
9425
|
await client.query("BEGIN");
|
|
7824
|
-
await client.query(`INSERT INTO public.
|
|
9426
|
+
await client.query(`INSERT INTO public.salesprompter_linkedin_companies_processed (
|
|
7825
9427
|
error, query, timestamp, name, description, follower_count, website, domain, industry,
|
|
7826
9428
|
company_size, founded, company_address, main_company_id, linkedin_id, sales_navigator_link,
|
|
7827
9429
|
employees_on_linked_in, company_name, total_employee_count, company_url, headquarters
|
|
@@ -8027,7 +9629,7 @@ async function upsertSalesNavigatorAccounts(params) {
|
|
|
8027
9629
|
}));
|
|
8028
9630
|
if (params.store.kind === "postgres") {
|
|
8029
9631
|
for (const row of rows) {
|
|
8030
|
-
await params.store.client.query(`INSERT INTO public.
|
|
9632
|
+
await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_accounts (
|
|
8031
9633
|
org_id, last_run_id, last_slice_id, account_key, sales_nav_company_url,
|
|
8032
9634
|
linkedin_company_url, company_id, company_name, industry, location, headquarters,
|
|
8033
9635
|
employee_count, follower_count, company_type, description, website, search_query_url,
|
|
@@ -8076,17 +9678,62 @@ async function upsertSalesNavigatorAccounts(params) {
|
|
|
8076
9678
|
row.scraped_at,
|
|
8077
9679
|
JSON.stringify(row.raw_payload),
|
|
8078
9680
|
]);
|
|
9681
|
+
await params.store.client.query(`INSERT INTO public.salesprompter_linkedin_sales_nav_account_run_memberships (
|
|
9682
|
+
run_id, org_id, account_key, last_slice_id
|
|
9683
|
+
)
|
|
9684
|
+
VALUES ($1, $2, $3, $4)
|
|
9685
|
+
ON CONFLICT (run_id, account_key)
|
|
9686
|
+
DO UPDATE SET
|
|
9687
|
+
last_slice_id = excluded.last_slice_id,
|
|
9688
|
+
updated_at = now()`, [
|
|
9689
|
+
row.last_run_id,
|
|
9690
|
+
row.org_id,
|
|
9691
|
+
row.account_key,
|
|
9692
|
+
row.last_slice_id,
|
|
9693
|
+
]);
|
|
8079
9694
|
}
|
|
8080
9695
|
return rows.length;
|
|
8081
9696
|
}
|
|
8082
|
-
const
|
|
8083
|
-
|
|
8084
|
-
.
|
|
8085
|
-
|
|
8086
|
-
|
|
9697
|
+
const supabaseBatchSize = 250;
|
|
9698
|
+
for (let index = 0; index < rows.length; index += supabaseBatchSize) {
|
|
9699
|
+
const batch = rows.slice(index, index + supabaseBatchSize);
|
|
9700
|
+
const { error } = await params.store.client
|
|
9701
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchAccounts)
|
|
9702
|
+
.upsert(batch, { onConflict: "org_id,account_key" });
|
|
9703
|
+
if (error) {
|
|
9704
|
+
throw new Error(`Failed to upsert Sales Navigator accounts in Supabase: ${error.message}`);
|
|
9705
|
+
}
|
|
9706
|
+
const { error: membershipError } = await params.store.client
|
|
9707
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountRunMemberships)
|
|
9708
|
+
.upsert(batch.map((row) => ({
|
|
9709
|
+
run_id: row.last_run_id,
|
|
9710
|
+
org_id: row.org_id,
|
|
9711
|
+
account_key: row.account_key,
|
|
9712
|
+
last_slice_id: row.last_slice_id,
|
|
9713
|
+
updated_at: new Date().toISOString(),
|
|
9714
|
+
})), { onConflict: "run_id,account_key" });
|
|
9715
|
+
if (membershipError) {
|
|
9716
|
+
throw new Error(`Failed to record Sales Navigator account run membership in Supabase: ${membershipError.message}`);
|
|
9717
|
+
}
|
|
8087
9718
|
}
|
|
8088
9719
|
return rows.length;
|
|
8089
9720
|
}
|
|
9721
|
+
async function countSalesNavigatorAccountsForRun(params) {
|
|
9722
|
+
if (params.store.kind === "postgres") {
|
|
9723
|
+
const result = await params.store.client.query(`SELECT count(*)::text AS count
|
|
9724
|
+
FROM public.salesprompter_linkedin_sales_nav_account_run_memberships
|
|
9725
|
+
WHERE run_id = $1`, [params.runId]);
|
|
9726
|
+
return Number(result.rows[0]?.count ?? 0);
|
|
9727
|
+
}
|
|
9728
|
+
const { count, error } = await params.store.client
|
|
9729
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountRunMemberships)
|
|
9730
|
+
.select("account_key", { count: "exact", head: true })
|
|
9731
|
+
.eq("run_id", params.runId);
|
|
9732
|
+
if (error) {
|
|
9733
|
+
throw new Error(`Failed to count Sales Navigator account run membership in Supabase: ${error.message}`);
|
|
9734
|
+
}
|
|
9735
|
+
return count ?? 0;
|
|
9736
|
+
}
|
|
8090
9737
|
function asJsonObject(value) {
|
|
8091
9738
|
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
8092
9739
|
? value
|
|
@@ -8145,8 +9792,8 @@ async function listSalesNavigatorAccountSearchRepairSlices(params) {
|
|
|
8145
9792
|
runs.slice_preset,
|
|
8146
9793
|
runs.max_results_per_search,
|
|
8147
9794
|
runs.page_size
|
|
8148
|
-
FROM public.
|
|
8149
|
-
INNER JOIN public.
|
|
9795
|
+
FROM public.salesprompter_linkedin_sales_nav_account_search_slices AS slices
|
|
9796
|
+
INNER JOIN public.salesprompter_linkedin_sales_nav_account_search_runs AS runs
|
|
8150
9797
|
ON runs.id = slices.run_id
|
|
8151
9798
|
WHERE slices.run_id = $1
|
|
8152
9799
|
AND slices.status IN ('failed', 'unresolved')
|
|
@@ -8178,32 +9825,52 @@ async function listSalesNavigatorAccountSearchRepairSlices(params) {
|
|
|
8178
9825
|
}));
|
|
8179
9826
|
}
|
|
8180
9827
|
async function summarizeSalesNavigatorAccountSearchRunFromSlices(params) {
|
|
8181
|
-
if (params.store.kind
|
|
8182
|
-
|
|
9828
|
+
if (params.store.kind === "supabase") {
|
|
9829
|
+
const { data, error } = await params.store.client
|
|
9830
|
+
.from(SALESPROMPTER_SUPABASE_TABLES.accountSearchSlices)
|
|
9831
|
+
.select("status,last_error,updated_at")
|
|
9832
|
+
.eq("run_id", params.runId);
|
|
9833
|
+
if (error) {
|
|
9834
|
+
throw new Error(`Failed to summarize Sales Navigator account slices in Supabase: ${error.message}`);
|
|
9835
|
+
}
|
|
9836
|
+
const rows = (data ?? []);
|
|
9837
|
+
const finalRows = rows.filter((row) => row.status !== "split");
|
|
9838
|
+
const failedRows = rows.filter((row) => row.status === "failed");
|
|
9839
|
+
const unresolvedRows = rows.filter((row) => row.status === "unresolved");
|
|
9840
|
+
const latestProblem = [...failedRows, ...unresolvedRows].sort((a, b) => Date.parse(b.updated_at) - Date.parse(a.updated_at))[0];
|
|
9841
|
+
return {
|
|
9842
|
+
totalSlices: finalRows.length,
|
|
9843
|
+
storedAccounts: await countSalesNavigatorAccountsForRun(params),
|
|
9844
|
+
attemptedSlices: finalRows.length,
|
|
9845
|
+
splitEvents: rows.filter((row) => row.status === "split").length,
|
|
9846
|
+
failedSlices: failedRows.length,
|
|
9847
|
+
unresolvedSlices: unresolvedRows.length,
|
|
9848
|
+
truncated: failedRows.length > 0 || unresolvedRows.length > 0,
|
|
9849
|
+
lastError: latestProblem?.last_error ?? null,
|
|
9850
|
+
};
|
|
8183
9851
|
}
|
|
8184
9852
|
const result = await params.store.client.query(`SELECT
|
|
8185
9853
|
count(*) FILTER (WHERE status <> 'split')::int AS total_slices,
|
|
8186
|
-
COALESCE(sum(stored_accounts) FILTER (WHERE status <> 'split'), 0)::int AS stored_accounts,
|
|
8187
9854
|
count(*) FILTER (WHERE status <> 'split')::int AS attempted_slices,
|
|
8188
9855
|
count(*) FILTER (WHERE status = 'split')::int AS split_events,
|
|
8189
9856
|
count(*) FILTER (WHERE status = 'failed')::int AS failed_slices,
|
|
8190
9857
|
count(*) FILTER (WHERE status = 'unresolved')::int AS unresolved_slices,
|
|
8191
9858
|
(
|
|
8192
9859
|
SELECT last_error
|
|
8193
|
-
FROM public.
|
|
9860
|
+
FROM public.salesprompter_linkedin_sales_nav_account_search_slices
|
|
8194
9861
|
WHERE run_id = $1
|
|
8195
9862
|
AND status IN ('failed', 'unresolved')
|
|
8196
9863
|
ORDER BY updated_at DESC
|
|
8197
9864
|
LIMIT 1
|
|
8198
9865
|
) AS last_error
|
|
8199
|
-
FROM public.
|
|
9866
|
+
FROM public.salesprompter_linkedin_sales_nav_account_search_slices
|
|
8200
9867
|
WHERE run_id = $1`, [params.runId]);
|
|
8201
9868
|
const row = result.rows[0] ?? {};
|
|
8202
9869
|
const failedSlices = Number(row.failed_slices ?? 0);
|
|
8203
9870
|
const unresolvedSlices = Number(row.unresolved_slices ?? 0);
|
|
8204
9871
|
return {
|
|
8205
9872
|
totalSlices: Number(row.total_slices ?? 0),
|
|
8206
|
-
storedAccounts:
|
|
9873
|
+
storedAccounts: await countSalesNavigatorAccountsForRun(params),
|
|
8207
9874
|
attemptedSlices: Number(row.attempted_slices ?? 0),
|
|
8208
9875
|
splitEvents: Number(row.split_events ?? 0),
|
|
8209
9876
|
failedSlices,
|
|
@@ -11499,6 +13166,187 @@ program
|
|
|
11499
13166
|
}
|
|
11500
13167
|
});
|
|
11501
13168
|
});
|
|
13169
|
+
async function runAffiliateLaunchCommand(options) {
|
|
13170
|
+
const affiliateLink = z.string().url().parse(options.affiliateLink);
|
|
13171
|
+
const linkedInUrl = z.string().url().parse(options.linkedinUrl);
|
|
13172
|
+
const maxResults = z.coerce.number().int().min(1).max(1000).parse(options.maxResults);
|
|
13173
|
+
const session = await requireAuthSession();
|
|
13174
|
+
const isSalesNavigatorPeopleSearch = new URL(linkedInUrl).pathname.includes("/sales/search/people");
|
|
13175
|
+
let localResults;
|
|
13176
|
+
let localCollection;
|
|
13177
|
+
if (!options.dryRun && isSalesNavigatorPeopleSearch) {
|
|
13178
|
+
const pageSize = z.coerce.number().int().min(1).max(100).parse(options.pageSize);
|
|
13179
|
+
const pageDelayMinMs = z.coerce.number().int().min(0).parse(options.pageDelayMinMs);
|
|
13180
|
+
const pageDelayMaxMs = z.coerce.number().int().min(pageDelayMinMs).parse(options.pageDelayMaxMs);
|
|
13181
|
+
const browserRelayPort = options.browserRelayPort == null
|
|
13182
|
+
? null
|
|
13183
|
+
: z.coerce.number().int().min(0).max(65535).parse(options.browserRelayPort);
|
|
13184
|
+
if (options.curlFile && browserRelayPort != null) {
|
|
13185
|
+
throw new Error("Use either --curl-file or --browser-relay-port, not both.");
|
|
13186
|
+
}
|
|
13187
|
+
const parsedRequest = options.curlFile
|
|
13188
|
+
? parseSalesNavigatorCurlRequest(await readFile(path.resolve(String(options.curlFile)), "utf8"))
|
|
13189
|
+
: browserRelayPort != null
|
|
13190
|
+
? {
|
|
13191
|
+
url: buildSalesNavigatorLeadApiUrlFromSearchUrl(linkedInUrl, pageSize),
|
|
13192
|
+
headers: {},
|
|
13193
|
+
}
|
|
13194
|
+
: buildSalesNavigatorApiRequestFromSearchUrl(linkedInUrl, await readLinkedInDirectLookupConfig(), pageSize);
|
|
13195
|
+
if (!new URL(parsedRequest.url).pathname.includes("/sales-api/salesApiLeadSearch")) {
|
|
13196
|
+
throw new Error("Affiliate --curl-file must contain a /sales-api/salesApiLeadSearch request.");
|
|
13197
|
+
}
|
|
13198
|
+
const browserRelay = browserRelayPort == null
|
|
13199
|
+
? null
|
|
13200
|
+
: await createLocalAccountSearchBrowserRelay(browserRelayPort);
|
|
13201
|
+
let collected;
|
|
13202
|
+
try {
|
|
13203
|
+
collected = await fetchAllLocalSalesNavigatorPeople(parsedRequest, {
|
|
13204
|
+
requestedProfiles: maxResults,
|
|
13205
|
+
pageSize,
|
|
13206
|
+
pageDelayMinMs,
|
|
13207
|
+
pageDelayMaxMs,
|
|
13208
|
+
retry: {
|
|
13209
|
+
maxRetries: 2,
|
|
13210
|
+
retryBaseDelayMs: 2_000,
|
|
13211
|
+
retryMaxDelayMs: 10_000,
|
|
13212
|
+
},
|
|
13213
|
+
executeRequest: browserRelay
|
|
13214
|
+
? (request) => browserRelay.request(request.url)
|
|
13215
|
+
: undefined,
|
|
13216
|
+
});
|
|
13217
|
+
}
|
|
13218
|
+
finally {
|
|
13219
|
+
await browserRelay?.close();
|
|
13220
|
+
}
|
|
13221
|
+
localResults = collected.people;
|
|
13222
|
+
localCollection = {
|
|
13223
|
+
totalResults: collected.totalResults,
|
|
13224
|
+
fetchedPages: collected.fetchedPages,
|
|
13225
|
+
pacing: collected.pacing,
|
|
13226
|
+
};
|
|
13227
|
+
}
|
|
13228
|
+
const payload = await launchAffiliateCampaignViaApp(session, {
|
|
13229
|
+
affiliateLink,
|
|
13230
|
+
linkedInUrl,
|
|
13231
|
+
maxResults,
|
|
13232
|
+
dryRun: Boolean(options.dryRun),
|
|
13233
|
+
localResults,
|
|
13234
|
+
localCollection,
|
|
13235
|
+
});
|
|
13236
|
+
if (options.out) {
|
|
13237
|
+
await writeJsonFile(options.out, payload);
|
|
13238
|
+
}
|
|
13239
|
+
return { session, payload };
|
|
13240
|
+
}
|
|
13241
|
+
function addAffiliateAudienceOptions(command) {
|
|
13242
|
+
return command
|
|
13243
|
+
.requiredOption("--affiliate-link <url>", "Affiliate link to promote")
|
|
13244
|
+
.requiredOption("--linkedin-url <url>", "Sales Navigator people or LinkedIn content search URL")
|
|
13245
|
+
.option("--max-results <number>", "Maximum audience results", "1000")
|
|
13246
|
+
.option("--curl-file <path>", "Optional copied Sales Navigator Lead Search curl request")
|
|
13247
|
+
.option("--browser-relay-port <number>", "Use the signed-in browser through a loopback relay")
|
|
13248
|
+
.option("--page-size <number>", "Direct Sales Navigator page size", "100")
|
|
13249
|
+
.option("--page-delay-min-ms <number>", "Minimum delay between direct pages", "5000")
|
|
13250
|
+
.option("--page-delay-max-ms <number>", "Maximum delay between direct pages", "8000")
|
|
13251
|
+
.option("--dry-run", "Validate and preview without creating remote resources", false)
|
|
13252
|
+
.option("--out <path>", "Optional local JSON output path");
|
|
13253
|
+
}
|
|
13254
|
+
addAffiliateAudienceOptions(program
|
|
13255
|
+
.command("affiliate:launch")
|
|
13256
|
+
.description("Build an affiliate audience from an affiliate link and LinkedIn search URL."))
|
|
13257
|
+
.action(async (options) => {
|
|
13258
|
+
const { payload } = await runAffiliateLaunchCommand(options);
|
|
13259
|
+
printOutput(payload);
|
|
13260
|
+
});
|
|
13261
|
+
addAffiliateAudienceOptions(program
|
|
13262
|
+
.command("affiliate:run")
|
|
13263
|
+
.description("Build an affiliate audience and prepare a reviewed Instantly campaign end to end."))
|
|
13264
|
+
.option("--campaign-name <name>", "Optional Instantly campaign name")
|
|
13265
|
+
.option("--language <language>", "Sequence language", "English")
|
|
13266
|
+
.option("--steps <number>", "Number of sequence emails", "3")
|
|
13267
|
+
.option("--min-email-score <number>", "Minimum Hunter confidence score", "80")
|
|
13268
|
+
.option("--allow-accept-all", "Include catch-all domains that meet the score threshold", false)
|
|
13269
|
+
.option("--daily-limit <number>", "Maximum new leads contacted per day", "25")
|
|
13270
|
+
.option("--timezone <timezone>", "Instantly sending timezone", "America/Detroit")
|
|
13271
|
+
.option("--send-from <time>", "Weekday sending window start (HH:MM)", "09:00")
|
|
13272
|
+
.option("--send-to <time>", "Weekday sending window end (HH:MM)", "16:00")
|
|
13273
|
+
.option("--activate", "Activate the prepared Instantly campaign after review data is returned", false)
|
|
13274
|
+
.action(async (options) => {
|
|
13275
|
+
const launched = await runAffiliateLaunchCommand(options);
|
|
13276
|
+
if (options.dryRun) {
|
|
13277
|
+
printOutput({
|
|
13278
|
+
...launched.payload,
|
|
13279
|
+
outreach: {
|
|
13280
|
+
status: "dry-run",
|
|
13281
|
+
provider: "instantly",
|
|
13282
|
+
activation: "not_requested"
|
|
13283
|
+
}
|
|
13284
|
+
});
|
|
13285
|
+
return;
|
|
13286
|
+
}
|
|
13287
|
+
const runId = launched.payload.extraction &&
|
|
13288
|
+
typeof launched.payload.extraction.runId === "string"
|
|
13289
|
+
? launched.payload.extraction.runId
|
|
13290
|
+
: null;
|
|
13291
|
+
if (!runId) {
|
|
13292
|
+
throw new Error("The audience provider did not return a durable run id. End-to-end outreach requires a Sales Navigator people search.");
|
|
13293
|
+
}
|
|
13294
|
+
const outreach = await prepareAffiliateOutreachViaApp(launched.session, {
|
|
13295
|
+
audienceRunId: z.string().uuid().parse(runId),
|
|
13296
|
+
campaignName: options.campaignName?.trim() || undefined,
|
|
13297
|
+
language: z.string().trim().min(1).max(40).parse(options.language),
|
|
13298
|
+
numberOfSteps: z.coerce.number().int().min(1).max(6).parse(options.steps),
|
|
13299
|
+
minEmailScore: z.coerce.number().int().min(0).max(100).parse(options.minEmailScore),
|
|
13300
|
+
allowAcceptAll: Boolean(options.allowAcceptAll),
|
|
13301
|
+
dailyLimit: z.coerce.number().int().min(1).max(1000).parse(options.dailyLimit),
|
|
13302
|
+
timezone: z.string().trim().min(1).max(100).parse(options.timezone),
|
|
13303
|
+
sendFrom: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/).parse(options.sendFrom),
|
|
13304
|
+
sendTo: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/).parse(options.sendTo)
|
|
13305
|
+
});
|
|
13306
|
+
const finalOutreach = options.activate
|
|
13307
|
+
? await activateAffiliateOutreachViaApp(launched.session, runId)
|
|
13308
|
+
: outreach;
|
|
13309
|
+
printOutput({
|
|
13310
|
+
status: "ok",
|
|
13311
|
+
audience: launched.payload,
|
|
13312
|
+
outreach: finalOutreach,
|
|
13313
|
+
next: finalOutreach.status === "ready"
|
|
13314
|
+
? `Review the sequence, then run: salesprompter affiliate:activate ${runId}`
|
|
13315
|
+
: "The Instantly campaign is active."
|
|
13316
|
+
});
|
|
13317
|
+
});
|
|
13318
|
+
program
|
|
13319
|
+
.command("affiliate:status <run-id>")
|
|
13320
|
+
.description("Show enrichment, sequence, Instantly, and activation status for an affiliate run.")
|
|
13321
|
+
.action(async (runId) => {
|
|
13322
|
+
const session = await requireAuthSession();
|
|
13323
|
+
const outreach = await getAffiliateOutreachViaApp(session, z.string().uuid().parse(runId));
|
|
13324
|
+
printOutput({ status: "ok", outreach });
|
|
13325
|
+
});
|
|
13326
|
+
program
|
|
13327
|
+
.command("affiliate:enrich <run-id>")
|
|
13328
|
+
.description("Find additional verified emails and add only new leads to the campaign.")
|
|
13329
|
+
.action(async (runId) => {
|
|
13330
|
+
const session = await requireAuthSession();
|
|
13331
|
+
const outreach = await enrichAffiliateOutreachViaApp(session, z.string().uuid().parse(runId));
|
|
13332
|
+
printOutput({
|
|
13333
|
+
status: "ok",
|
|
13334
|
+
outreach,
|
|
13335
|
+
message: "Email recovery finished and new verified leads were added."
|
|
13336
|
+
});
|
|
13337
|
+
});
|
|
13338
|
+
program
|
|
13339
|
+
.command("affiliate:activate <run-id>")
|
|
13340
|
+
.description("Activate a reviewed Instantly affiliate campaign.")
|
|
13341
|
+
.action(async (runId) => {
|
|
13342
|
+
const session = await requireAuthSession();
|
|
13343
|
+
const outreach = await activateAffiliateOutreachViaApp(session, z.string().uuid().parse(runId));
|
|
13344
|
+
printOutput({
|
|
13345
|
+
status: "ok",
|
|
13346
|
+
outreach,
|
|
13347
|
+
message: "The Instantly campaign is active."
|
|
13348
|
+
});
|
|
13349
|
+
});
|
|
11502
13350
|
program
|
|
11503
13351
|
.command("sync:crm")
|
|
11504
13352
|
.description("Dry-run sync scored leads into a CRM target.")
|
|
@@ -12274,16 +14122,533 @@ program
|
|
|
12274
14122
|
await writeJsonFile(samplesPath, samples);
|
|
12275
14123
|
printOutput(payload);
|
|
12276
14124
|
});
|
|
14125
|
+
program
|
|
14126
|
+
.command("salesnav:accounts:collect")
|
|
14127
|
+
.alias("accounts:collect")
|
|
14128
|
+
.description("Collect an exhaustive Sales Navigator account search locally with a durable checkpoint.")
|
|
14129
|
+
.requiredOption("--query-url <url>", "LinkedIn Sales Navigator account search URL")
|
|
14130
|
+
.requiredOption("--checkpoint <path>", "Checkpoint file used to resume after rate limits or interruptions")
|
|
14131
|
+
.requiredOption("--raw-out <path>", "Line-delimited raw account results used by accounts:local-import")
|
|
14132
|
+
.option("--curl-file <path>", "Path to a saved account-search curl request used as the local request template")
|
|
14133
|
+
.option("--browser-relay-port <number>", "Use the signed-in local browser as Account Search transport through this loopback port")
|
|
14134
|
+
.option("--reported-total <number>", "Known root result total. Skips the initial one-request root count probe.")
|
|
14135
|
+
.option("--max-results-per-search <number>", "Maximum pageable results per final filter slice", String(LOCAL_ACCOUNT_SEARCH_COLLECTOR_RESULT_WINDOW))
|
|
14136
|
+
.option("--page-size <number>", "Accounts requested per Account Search API request", "100")
|
|
14137
|
+
.option("--delay-min-ms <number>", "Minimum randomized delay between Account Search API requests", "5000")
|
|
14138
|
+
.option("--delay-max-ms <number>", "Maximum randomized delay between Account Search API requests", "8000")
|
|
14139
|
+
.option("--max-requests <number>", "Maximum API requests in this invocation, or 'all'", "all")
|
|
14140
|
+
.option("--max-retries <number>", "Retries for transient network/5xx failures. Rate limits stop immediately.", "1")
|
|
14141
|
+
.option("--retry-base-delay-ms <number>", "Base delay for full-jitter retry backoff", "2000")
|
|
14142
|
+
.option("--retry-max-delay-ms <number>", "Maximum delay for full-jitter retry backoff", "30000")
|
|
14143
|
+
.option("--out <path>", "Optional JSON summary output path")
|
|
14144
|
+
.action(async (options) => {
|
|
14145
|
+
const queryUrl = z.string().url().parse(options.queryUrl);
|
|
14146
|
+
const checkpointPath = path.resolve(z.string().min(1).parse(options.checkpoint));
|
|
14147
|
+
const rawOutputPath = path.resolve(z.string().min(1).parse(options.rawOut));
|
|
14148
|
+
const cap = z.coerce
|
|
14149
|
+
.number()
|
|
14150
|
+
.int()
|
|
14151
|
+
.min(1)
|
|
14152
|
+
.max(LOCAL_ACCOUNT_SEARCH_COLLECTOR_RESULT_WINDOW)
|
|
14153
|
+
.parse(options.maxResultsPerSearch);
|
|
14154
|
+
const pageSize = z.coerce
|
|
14155
|
+
.number()
|
|
14156
|
+
.int()
|
|
14157
|
+
.min(1)
|
|
14158
|
+
.max(100)
|
|
14159
|
+
.parse(options.pageSize);
|
|
14160
|
+
const delayMinMs = z.coerce
|
|
14161
|
+
.number()
|
|
14162
|
+
.int()
|
|
14163
|
+
.min(0)
|
|
14164
|
+
.max(300000)
|
|
14165
|
+
.parse(options.delayMinMs);
|
|
14166
|
+
const delayMaxMs = z.coerce
|
|
14167
|
+
.number()
|
|
14168
|
+
.int()
|
|
14169
|
+
.min(0)
|
|
14170
|
+
.max(300000)
|
|
14171
|
+
.parse(options.delayMaxMs);
|
|
14172
|
+
if (delayMaxMs < delayMinMs) {
|
|
14173
|
+
throw new Error("--delay-max-ms must be greater than or equal to --delay-min-ms.");
|
|
14174
|
+
}
|
|
14175
|
+
const maxRequestsOption = String(options.maxRequests ?? "all")
|
|
14176
|
+
.trim()
|
|
14177
|
+
.toLowerCase();
|
|
14178
|
+
const maxRequests = maxRequestsOption === "all"
|
|
14179
|
+
? null
|
|
14180
|
+
: z.coerce.number().int().min(1).max(100000).parse(options.maxRequests);
|
|
14181
|
+
const reportedTotal = options.reportedTotal == null
|
|
14182
|
+
? null
|
|
14183
|
+
: z.coerce
|
|
14184
|
+
.number()
|
|
14185
|
+
.int()
|
|
14186
|
+
.min(1)
|
|
14187
|
+
.max(1_000_000)
|
|
14188
|
+
.parse(options.reportedTotal);
|
|
14189
|
+
const retry = {
|
|
14190
|
+
maxRetries: z.coerce
|
|
14191
|
+
.number()
|
|
14192
|
+
.int()
|
|
14193
|
+
.min(0)
|
|
14194
|
+
.max(5)
|
|
14195
|
+
.parse(options.maxRetries),
|
|
14196
|
+
retryBaseDelayMs: z.coerce
|
|
14197
|
+
.number()
|
|
14198
|
+
.int()
|
|
14199
|
+
.min(0)
|
|
14200
|
+
.max(300000)
|
|
14201
|
+
.parse(options.retryBaseDelayMs),
|
|
14202
|
+
retryMaxDelayMs: z.coerce
|
|
14203
|
+
.number()
|
|
14204
|
+
.int()
|
|
14205
|
+
.min(0)
|
|
14206
|
+
.max(300000)
|
|
14207
|
+
.parse(options.retryMaxDelayMs),
|
|
14208
|
+
};
|
|
14209
|
+
const normalizedSourceQueryUrl = buildSalesNavigatorAccountQuery(queryUrl, []).sourceQueryUrl;
|
|
14210
|
+
let checkpoint;
|
|
14211
|
+
if (await fileExists(checkpointPath)) {
|
|
14212
|
+
checkpoint = parseLocalAccountSearchCollectorCheckpoint(JSON.parse(await readFile(checkpointPath, "utf8")));
|
|
14213
|
+
const normalizedCheckpointUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, []).sourceQueryUrl;
|
|
14214
|
+
if (normalizedCheckpointUrl !== normalizedSourceQueryUrl) {
|
|
14215
|
+
throw new Error("The checkpoint belongs to a different Sales Navigator account search.");
|
|
14216
|
+
}
|
|
14217
|
+
if (checkpoint.pageSize !== pageSize || checkpoint.cap > cap) {
|
|
14218
|
+
throw new Error(`Checkpoint paging does not match this invocation. Use --max-results-per-search ${checkpoint.cap} --page-size ${checkpoint.pageSize}.`);
|
|
14219
|
+
}
|
|
14220
|
+
if (checkpoint.cap < cap) {
|
|
14221
|
+
checkpoint.cap = cap;
|
|
14222
|
+
}
|
|
14223
|
+
checkpoint.delayMinMs = delayMinMs;
|
|
14224
|
+
checkpoint.delayMaxMs = delayMaxMs;
|
|
14225
|
+
if (reportedTotal != null && checkpoint.rootTotal == null) {
|
|
14226
|
+
checkpoint.rootTotal = reportedTotal;
|
|
14227
|
+
}
|
|
14228
|
+
}
|
|
14229
|
+
else {
|
|
14230
|
+
checkpoint = createLocalAccountSearchCollectorCheckpoint({
|
|
14231
|
+
sourceQueryUrl: queryUrl,
|
|
14232
|
+
rootTotal: reportedTotal,
|
|
14233
|
+
cap,
|
|
14234
|
+
pageSize,
|
|
14235
|
+
delayMinMs,
|
|
14236
|
+
delayMaxMs,
|
|
14237
|
+
});
|
|
14238
|
+
}
|
|
14239
|
+
if (migrateLocalAccountSearchCollectorCheckpoint(checkpoint)) {
|
|
14240
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14241
|
+
}
|
|
14242
|
+
const curlFile = typeof options.curlFile === "string" && options.curlFile.trim().length > 0
|
|
14243
|
+
? options.curlFile.trim()
|
|
14244
|
+
: null;
|
|
14245
|
+
const browserRelayPort = options.browserRelayPort == null
|
|
14246
|
+
? null
|
|
14247
|
+
: z.coerce
|
|
14248
|
+
.number()
|
|
14249
|
+
.int()
|
|
14250
|
+
.min(1)
|
|
14251
|
+
.max(65535)
|
|
14252
|
+
.parse(options.browserRelayPort);
|
|
14253
|
+
if (curlFile && browserRelayPort != null) {
|
|
14254
|
+
throw new Error("Use either --curl-file or --browser-relay-port, not both.");
|
|
14255
|
+
}
|
|
14256
|
+
const templateRequest = curlFile
|
|
14257
|
+
? parseSalesNavigatorCurlRequest(await readFile(curlFile, "utf8"))
|
|
14258
|
+
: null;
|
|
14259
|
+
if (templateRequest &&
|
|
14260
|
+
!new URL(templateRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
14261
|
+
throw new Error("Account collection --curl-file must contain a /sales-api/salesApiAccountSearch request.");
|
|
14262
|
+
}
|
|
14263
|
+
const config = templateRequest || browserRelayPort != null
|
|
14264
|
+
? null
|
|
14265
|
+
: await readLinkedInAccountSearchConfig(queryUrl);
|
|
14266
|
+
const browserRelay = browserRelayPort == null
|
|
14267
|
+
? null
|
|
14268
|
+
: await createLocalAccountSearchBrowserRelay(browserRelayPort);
|
|
14269
|
+
const buildRequest = (searchUrl, start, count) => {
|
|
14270
|
+
const request = browserRelay
|
|
14271
|
+
? {
|
|
14272
|
+
url: buildSalesNavigatorAccountApiUrlFromSearchUrl(searchUrl, pageSize),
|
|
14273
|
+
headers: {},
|
|
14274
|
+
}
|
|
14275
|
+
: templateRequest
|
|
14276
|
+
? buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(searchUrl, templateRequest, pageSize)
|
|
14277
|
+
: buildSalesNavigatorAccountApiRequestFromSearchUrl(searchUrl, config, pageSize);
|
|
14278
|
+
return {
|
|
14279
|
+
...request,
|
|
14280
|
+
url: withSalesNavigatorPaging(request.url, start, count),
|
|
14281
|
+
};
|
|
14282
|
+
};
|
|
14283
|
+
const executeRequest = (request) => browserRelay
|
|
14284
|
+
? browserRelay.request(request.url)
|
|
14285
|
+
: fetchLocalSalesNavigatorRequest(request, retry);
|
|
14286
|
+
try {
|
|
14287
|
+
let requestsThisRun = 0;
|
|
14288
|
+
let delayThisRunMs = 0;
|
|
14289
|
+
const distinctAccountKeys = await loadLocalAccountSearchRawAccountKeys(rawOutputPath);
|
|
14290
|
+
checkpoint.distinctAccounts = distinctAccountKeys.size;
|
|
14291
|
+
checkpoint.completedByDistinctCount =
|
|
14292
|
+
checkpoint.rootTotal != null &&
|
|
14293
|
+
distinctAccountKeys.size >= checkpoint.rootTotal;
|
|
14294
|
+
if (checkpoint.completedByDistinctCount) {
|
|
14295
|
+
checkpoint.taskQueue = [];
|
|
14296
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14297
|
+
}
|
|
14298
|
+
const requestBudgetAvailable = () => maxRequests == null || requestsThisRun < maxRequests;
|
|
14299
|
+
const stopForRateLimit = async (error, task) => {
|
|
14300
|
+
checkpoint.rateLimited = {
|
|
14301
|
+
at: new Date().toISOString(),
|
|
14302
|
+
retryAfter: error.retryAfterMs,
|
|
14303
|
+
status: error.status,
|
|
14304
|
+
task,
|
|
14305
|
+
};
|
|
14306
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14307
|
+
};
|
|
14308
|
+
if (checkpoint.rootTotal == null && requestBudgetAvailable()) {
|
|
14309
|
+
try {
|
|
14310
|
+
const rootResponse = await executeRequest(buildRequest(queryUrl, 0, 1));
|
|
14311
|
+
requestsThisRun += 1;
|
|
14312
|
+
checkpoint.requests += 1;
|
|
14313
|
+
checkpoint.rootTotal = extractLocalSalesNavigatorTotalResults(rootResponse.body);
|
|
14314
|
+
checkpoint.rateLimited = null;
|
|
14315
|
+
if (checkpoint.rootTotal == null) {
|
|
14316
|
+
throw new Error("Sales Navigator Account Search root response did not include a result total.");
|
|
14317
|
+
}
|
|
14318
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14319
|
+
}
|
|
14320
|
+
catch (error) {
|
|
14321
|
+
if (error instanceof LocalSalesNavigatorRateLimitError) {
|
|
14322
|
+
await stopForRateLimit(error, { kind: "root" });
|
|
14323
|
+
const payload = {
|
|
14324
|
+
status: "rate_limited",
|
|
14325
|
+
mode: "account-local-collector",
|
|
14326
|
+
checkpointPath,
|
|
14327
|
+
rawOutputPath,
|
|
14328
|
+
requestsThisRun,
|
|
14329
|
+
requests: checkpoint.requests,
|
|
14330
|
+
remainingTasks: checkpoint.taskQueue.length,
|
|
14331
|
+
rateLimited: checkpoint.rateLimited,
|
|
14332
|
+
};
|
|
14333
|
+
if (options.out) {
|
|
14334
|
+
await writeJsonFile(options.out, payload);
|
|
14335
|
+
}
|
|
14336
|
+
printOutput(payload);
|
|
14337
|
+
return;
|
|
14338
|
+
}
|
|
14339
|
+
throw error;
|
|
14340
|
+
}
|
|
14341
|
+
}
|
|
14342
|
+
if (checkpoint.rootTotal != null &&
|
|
14343
|
+
checkpoint.taskQueue.length === 0 &&
|
|
14344
|
+
checkpoint.seenQueries.length === 0) {
|
|
14345
|
+
seedLocalAccountSearchCollectorTasks(checkpoint);
|
|
14346
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14347
|
+
}
|
|
14348
|
+
const dimensions = new Map([
|
|
14349
|
+
...DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS,
|
|
14350
|
+
LOCAL_ACCOUNT_SEARCH_COLLECTOR_GROWTH_DIMENSION,
|
|
14351
|
+
].map((dimension) => [dimension.filterType, dimension]));
|
|
14352
|
+
const seenQueries = new Set(checkpoint.seenQueries);
|
|
14353
|
+
let stoppedByRateLimit = false;
|
|
14354
|
+
while (checkpoint.taskQueue.length > 0 && requestBudgetAvailable()) {
|
|
14355
|
+
const task = checkpoint.taskQueue[0];
|
|
14356
|
+
const searchUrl = task.kind === "node"
|
|
14357
|
+
? task.searchUrlOverride ??
|
|
14358
|
+
buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, task.filters).slicedQueryUrl
|
|
14359
|
+
: task.searchUrl;
|
|
14360
|
+
const start = task.kind === "page" ? task.start : 0;
|
|
14361
|
+
let response;
|
|
14362
|
+
try {
|
|
14363
|
+
response = await executeRequest(buildRequest(searchUrl, start, pageSize));
|
|
14364
|
+
}
|
|
14365
|
+
catch (error) {
|
|
14366
|
+
if (error instanceof LocalSalesNavigatorRateLimitError) {
|
|
14367
|
+
stoppedByRateLimit = true;
|
|
14368
|
+
await stopForRateLimit(error, task);
|
|
14369
|
+
break;
|
|
14370
|
+
}
|
|
14371
|
+
checkpoint.failures.push({
|
|
14372
|
+
at: new Date().toISOString(),
|
|
14373
|
+
task,
|
|
14374
|
+
error: error instanceof Error ? error.message : String(error),
|
|
14375
|
+
});
|
|
14376
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14377
|
+
throw error;
|
|
14378
|
+
}
|
|
14379
|
+
requestsThisRun += 1;
|
|
14380
|
+
checkpoint.requests += 1;
|
|
14381
|
+
checkpoint.rateLimited = null;
|
|
14382
|
+
const elements = extractLocalSalesNavigatorElements(response.body);
|
|
14383
|
+
await appendLocalAccountSearchRawElements({
|
|
14384
|
+
rawOutputPath,
|
|
14385
|
+
searchUrl,
|
|
14386
|
+
elements,
|
|
14387
|
+
});
|
|
14388
|
+
for (const element of elements) {
|
|
14389
|
+
addLocalAccountSearchElementKey(distinctAccountKeys, element, searchUrl);
|
|
14390
|
+
}
|
|
14391
|
+
checkpoint.distinctAccounts = distinctAccountKeys.size;
|
|
14392
|
+
checkpoint.pagesWritten += 1;
|
|
14393
|
+
checkpoint.rowsWritten += elements.length;
|
|
14394
|
+
if (checkpoint.rootTotal != null &&
|
|
14395
|
+
distinctAccountKeys.size >= checkpoint.rootTotal) {
|
|
14396
|
+
checkpoint.taskQueue = [];
|
|
14397
|
+
checkpoint.completedByDistinctCount = true;
|
|
14398
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14399
|
+
break;
|
|
14400
|
+
}
|
|
14401
|
+
if (task.kind === "node") {
|
|
14402
|
+
const total = extractLocalSalesNavigatorTotalResults(response.body);
|
|
14403
|
+
if (total == null) {
|
|
14404
|
+
checkpoint.failures.push({
|
|
14405
|
+
at: new Date().toISOString(),
|
|
14406
|
+
task,
|
|
14407
|
+
error: "Account Search response did not include a result total.",
|
|
14408
|
+
});
|
|
14409
|
+
}
|
|
14410
|
+
else if (total <= cap) {
|
|
14411
|
+
checkpoint.leaves.push({
|
|
14412
|
+
id: task.id,
|
|
14413
|
+
pass: task.pass,
|
|
14414
|
+
total,
|
|
14415
|
+
searchUrl,
|
|
14416
|
+
filters: task.filters,
|
|
14417
|
+
});
|
|
14418
|
+
const pageTasks = [];
|
|
14419
|
+
for (let pageStart = pageSize; pageStart < total; pageStart += pageSize) {
|
|
14420
|
+
pageTasks.push({
|
|
14421
|
+
kind: "page",
|
|
14422
|
+
id: task.id,
|
|
14423
|
+
pass: task.pass,
|
|
14424
|
+
searchUrl,
|
|
14425
|
+
start: pageStart,
|
|
14426
|
+
total,
|
|
14427
|
+
});
|
|
14428
|
+
}
|
|
14429
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, pageTasks);
|
|
14430
|
+
}
|
|
14431
|
+
else {
|
|
14432
|
+
const [nextFilterType, ...remainingDimensions] = task.remainingDimensions;
|
|
14433
|
+
const dimension = nextFilterType
|
|
14434
|
+
? dimensions.get(nextFilterType)
|
|
14435
|
+
: null;
|
|
14436
|
+
if (!dimension) {
|
|
14437
|
+
const bayAreaPartitionPlan = buildLocalAccountSearchCollectorBayAreaPartitions(checkpoint.sourceQueryUrl, task.filters);
|
|
14438
|
+
if (!bayAreaPartitionPlan) {
|
|
14439
|
+
const industryPartitionPlan = task.pass.startsWith("coverage-")
|
|
14440
|
+
? buildLocalAccountSearchCollectorIndustryPartitions(searchUrl)
|
|
14441
|
+
: null;
|
|
14442
|
+
if (industryPartitionPlan) {
|
|
14443
|
+
checkpoint.windows.push({
|
|
14444
|
+
id: task.id,
|
|
14445
|
+
pass: task.pass,
|
|
14446
|
+
total,
|
|
14447
|
+
searchUrl,
|
|
14448
|
+
splitOn: industryPartitionPlan.splitOn,
|
|
14449
|
+
});
|
|
14450
|
+
const industryTasks = [];
|
|
14451
|
+
for (const partition of industryPartitionPlan.partitions) {
|
|
14452
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, partition.searchUrl);
|
|
14453
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14454
|
+
checkpoint.duplicateQueries += 1;
|
|
14455
|
+
continue;
|
|
14456
|
+
}
|
|
14457
|
+
seenQueries.add(childQueryKey);
|
|
14458
|
+
industryTasks.push({
|
|
14459
|
+
kind: "node",
|
|
14460
|
+
id: `${task.id}-${industryPartitionPlan.splitOn}-${partition.idSuffix}`,
|
|
14461
|
+
pass: task.pass,
|
|
14462
|
+
filters: task.filters.map((filter) => ({
|
|
14463
|
+
...filter,
|
|
14464
|
+
values: filter.values.map((value) => ({ ...value })),
|
|
14465
|
+
})),
|
|
14466
|
+
remainingDimensions: [],
|
|
14467
|
+
searchUrlOverride: partition.searchUrl,
|
|
14468
|
+
keywordClauses: task.keywordClauses
|
|
14469
|
+
? [...task.keywordClauses]
|
|
14470
|
+
: undefined,
|
|
14471
|
+
keywordDepth: task.keywordDepth,
|
|
14472
|
+
});
|
|
14473
|
+
}
|
|
14474
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, industryTasks);
|
|
14475
|
+
}
|
|
14476
|
+
else {
|
|
14477
|
+
const keywordPartitionPlan = buildLocalAccountSearchCollectorKeywordPartitions(searchUrl, task);
|
|
14478
|
+
if (!keywordPartitionPlan) {
|
|
14479
|
+
const terminalPageLimit = Math.min(total, cap);
|
|
14480
|
+
const pageTasks = [];
|
|
14481
|
+
for (let pageStart = pageSize; pageStart < terminalPageLimit; pageStart += pageSize) {
|
|
14482
|
+
pageTasks.push({
|
|
14483
|
+
kind: "page",
|
|
14484
|
+
id: task.id,
|
|
14485
|
+
pass: task.pass,
|
|
14486
|
+
searchUrl,
|
|
14487
|
+
start: pageStart,
|
|
14488
|
+
total,
|
|
14489
|
+
});
|
|
14490
|
+
}
|
|
14491
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, pageTasks);
|
|
14492
|
+
checkpoint.failures.push({
|
|
14493
|
+
at: new Date().toISOString(),
|
|
14494
|
+
task,
|
|
14495
|
+
total,
|
|
14496
|
+
collectedResultWindow: terminalPageLimit,
|
|
14497
|
+
missingBeyondResultWindow: Math.max(0, total - terminalPageLimit),
|
|
14498
|
+
error: `Account Search slice remains above ${cap} with no split dimension left.`,
|
|
14499
|
+
});
|
|
14500
|
+
}
|
|
14501
|
+
else {
|
|
14502
|
+
checkpoint.windows.push({
|
|
14503
|
+
id: task.id,
|
|
14504
|
+
pass: task.pass,
|
|
14505
|
+
total,
|
|
14506
|
+
searchUrl,
|
|
14507
|
+
splitOn: keywordPartitionPlan.splitOn,
|
|
14508
|
+
});
|
|
14509
|
+
const keywordTasks = [];
|
|
14510
|
+
for (const partition of keywordPartitionPlan.partitions) {
|
|
14511
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, partition.searchUrl);
|
|
14512
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14513
|
+
checkpoint.duplicateQueries += 1;
|
|
14514
|
+
continue;
|
|
14515
|
+
}
|
|
14516
|
+
seenQueries.add(childQueryKey);
|
|
14517
|
+
keywordTasks.push({
|
|
14518
|
+
kind: "node",
|
|
14519
|
+
id: `${task.id}-${keywordPartitionPlan.splitOn}-${partition.idSuffix}`,
|
|
14520
|
+
pass: task.pass,
|
|
14521
|
+
filters: task.filters.map((filter) => ({
|
|
14522
|
+
...filter,
|
|
14523
|
+
values: filter.values.map((value) => ({ ...value })),
|
|
14524
|
+
})),
|
|
14525
|
+
remainingDimensions: [],
|
|
14526
|
+
searchUrlOverride: partition.searchUrl,
|
|
14527
|
+
keywordClauses: partition.keywordClauses,
|
|
14528
|
+
keywordDepth: partition.keywordDepth,
|
|
14529
|
+
});
|
|
14530
|
+
}
|
|
14531
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, keywordTasks);
|
|
14532
|
+
}
|
|
14533
|
+
}
|
|
14534
|
+
}
|
|
14535
|
+
else {
|
|
14536
|
+
checkpoint.windows.push({
|
|
14537
|
+
id: task.id,
|
|
14538
|
+
pass: task.pass,
|
|
14539
|
+
total,
|
|
14540
|
+
searchUrl,
|
|
14541
|
+
splitOn: bayAreaPartitionPlan.splitOn,
|
|
14542
|
+
});
|
|
14543
|
+
const regionTasks = [];
|
|
14544
|
+
bayAreaPartitionPlan.partitions.forEach((filters, index) => {
|
|
14545
|
+
const childSearchUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, filters).slicedQueryUrl;
|
|
14546
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, childSearchUrl);
|
|
14547
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14548
|
+
checkpoint.duplicateQueries += 1;
|
|
14549
|
+
return;
|
|
14550
|
+
}
|
|
14551
|
+
seenQueries.add(childQueryKey);
|
|
14552
|
+
regionTasks.push({
|
|
14553
|
+
kind: "node",
|
|
14554
|
+
id: `${task.id}-${bayAreaPartitionPlan.splitOn}-${index}`,
|
|
14555
|
+
pass: task.pass,
|
|
14556
|
+
filters,
|
|
14557
|
+
remainingDimensions: [],
|
|
14558
|
+
});
|
|
14559
|
+
});
|
|
14560
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, regionTasks);
|
|
14561
|
+
}
|
|
14562
|
+
}
|
|
14563
|
+
else {
|
|
14564
|
+
checkpoint.windows.push({
|
|
14565
|
+
id: task.id,
|
|
14566
|
+
pass: task.pass,
|
|
14567
|
+
total,
|
|
14568
|
+
searchUrl,
|
|
14569
|
+
splitOn: dimension.filterType,
|
|
14570
|
+
});
|
|
14571
|
+
const dimensionTasks = [];
|
|
14572
|
+
buildLocalAccountSearchCollectorDimensionPartitions(dimension).forEach((partition) => {
|
|
14573
|
+
const filters = [
|
|
14574
|
+
...task.filters,
|
|
14575
|
+
partition.filter,
|
|
14576
|
+
];
|
|
14577
|
+
const childSearchUrl = buildSalesNavigatorAccountQuery(checkpoint.sourceQueryUrl, filters).slicedQueryUrl;
|
|
14578
|
+
const childQueryKey = localAccountSearchCollectorSeenKey(task.pass, childSearchUrl);
|
|
14579
|
+
if (seenQueries.has(childQueryKey)) {
|
|
14580
|
+
checkpoint.duplicateQueries += 1;
|
|
14581
|
+
return;
|
|
14582
|
+
}
|
|
14583
|
+
seenQueries.add(childQueryKey);
|
|
14584
|
+
dimensionTasks.push({
|
|
14585
|
+
kind: "node",
|
|
14586
|
+
id: `${task.id}-${dimension.filterType}-${partition.idSuffix}`,
|
|
14587
|
+
pass: task.pass,
|
|
14588
|
+
filters,
|
|
14589
|
+
remainingDimensions,
|
|
14590
|
+
});
|
|
14591
|
+
});
|
|
14592
|
+
enqueueLocalAccountSearchCollectorTasks(checkpoint, task, dimensionTasks);
|
|
14593
|
+
}
|
|
14594
|
+
}
|
|
14595
|
+
}
|
|
14596
|
+
checkpoint.taskQueue.shift();
|
|
14597
|
+
checkpoint.seenQueries = [...seenQueries];
|
|
14598
|
+
await saveLocalAccountSearchCollectorCheckpoint(checkpointPath, checkpoint);
|
|
14599
|
+
if (checkpoint.taskQueue.length > 0 && requestBudgetAvailable()) {
|
|
14600
|
+
const delayMs = randomIntegerBetween(delayMinMs, delayMaxMs);
|
|
14601
|
+
if (delayMs > 0) {
|
|
14602
|
+
delayThisRunMs += delayMs;
|
|
14603
|
+
await delay(delayMs);
|
|
14604
|
+
}
|
|
14605
|
+
}
|
|
14606
|
+
}
|
|
14607
|
+
const payload = {
|
|
14608
|
+
status: stoppedByRateLimit
|
|
14609
|
+
? "rate_limited"
|
|
14610
|
+
: checkpoint.taskQueue.length === 0
|
|
14611
|
+
? "complete"
|
|
14612
|
+
: "checkpointed",
|
|
14613
|
+
mode: "account-local-collector",
|
|
14614
|
+
sourceQueryUrl: checkpoint.sourceQueryUrl,
|
|
14615
|
+
rootTotal: checkpoint.rootTotal,
|
|
14616
|
+
checkpointPath,
|
|
14617
|
+
rawOutputPath,
|
|
14618
|
+
requestsThisRun,
|
|
14619
|
+
delayThisRunMs,
|
|
14620
|
+
requests: checkpoint.requests,
|
|
14621
|
+
pagesWritten: checkpoint.pagesWritten,
|
|
14622
|
+
rowsWritten: checkpoint.rowsWritten,
|
|
14623
|
+
leaves: checkpoint.leaves.length,
|
|
14624
|
+
windows: checkpoint.windows.length,
|
|
14625
|
+
failures: checkpoint.failures.length,
|
|
14626
|
+
remainingTasks: checkpoint.taskQueue.length,
|
|
14627
|
+
distinctAccounts: distinctAccountKeys.size,
|
|
14628
|
+
completedByDistinctCount: checkpoint.completedByDistinctCount === true,
|
|
14629
|
+
rateLimited: checkpoint.rateLimited,
|
|
14630
|
+
browserRelayPort: browserRelay?.port ?? null,
|
|
14631
|
+
};
|
|
14632
|
+
if (options.out) {
|
|
14633
|
+
await writeJsonFile(options.out, payload);
|
|
14634
|
+
}
|
|
14635
|
+
printOutput(payload);
|
|
14636
|
+
}
|
|
14637
|
+
finally {
|
|
14638
|
+
await browserRelay?.close();
|
|
14639
|
+
}
|
|
14640
|
+
});
|
|
12277
14641
|
program
|
|
12278
14642
|
.command("salesnav:accounts:split")
|
|
12279
14643
|
.alias("accounts:split")
|
|
12280
14644
|
.description("Split a Sales Navigator account search into account slices below the result cap.")
|
|
12281
14645
|
.requiredOption("--query-url <url>", "LinkedIn Sales Navigator account search URL")
|
|
12282
14646
|
.option("--curl-file <path>", "Path to a saved account-search curl request used as the local request template")
|
|
12283
|
-
.option("--org-id <id>", "Salesprompter workspace org id for
|
|
14647
|
+
.option("--org-id <id>", "Salesprompter workspace org id for Supabase storage. Defaults to SALESPROMPTER_ORG_ID or saved auth session org.")
|
|
12284
14648
|
.option("--max-results-per-search <number>", "Maximum account results allowed for each final slice", "1000")
|
|
12285
14649
|
.option("--max-windowed-final-results <number>", "Bounded fallback fetch for an oversized final account slice when no split dimensions remain", "1000")
|
|
12286
14650
|
.option("--no-windowed-final-slices", "Do not fetch pageable rows for oversized final account slices after adaptive splitting is exhausted")
|
|
14651
|
+
.option("--include-location-mismatches", "Store account rows even when the visible Sales Navigator account location does not match the source REGION filter", false)
|
|
12287
14652
|
.option("--max-split-depth <number>", "Maximum number of adaptive account split dimensions to use", "3")
|
|
12288
14653
|
.option("--max-slices <number>", "Safety cap for total account slices probed in this invocation", "1000")
|
|
12289
14654
|
.option("--probe-count <number>", "Accounts to request while probing each slice count", "1")
|
|
@@ -12389,6 +14754,7 @@ program
|
|
|
12389
14754
|
.max(300000)
|
|
12390
14755
|
.parse(options.retryMaxDelayMs),
|
|
12391
14756
|
};
|
|
14757
|
+
const filterLocationMismatches = !Boolean(options.includeLocationMismatches);
|
|
12392
14758
|
const curlFile = typeof options.curlFile === "string" && options.curlFile.trim().length > 0
|
|
12393
14759
|
? options.curlFile.trim()
|
|
12394
14760
|
: null;
|
|
@@ -12449,7 +14815,9 @@ program
|
|
|
12449
14815
|
!new URL(templateRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
12450
14816
|
throw new Error("Account split --curl-file must contain a /sales-api/salesApiAccountSearch request.");
|
|
12451
14817
|
}
|
|
12452
|
-
const config = templateRequest
|
|
14818
|
+
const config = templateRequest
|
|
14819
|
+
? null
|
|
14820
|
+
: await readLinkedInAccountSearchConfig(queryUrl);
|
|
12453
14821
|
const buildAccountApiRequest = (slicedQueryUrl) => templateRequest
|
|
12454
14822
|
? buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(slicedQueryUrl, templateRequest, pageSize)
|
|
12455
14823
|
: buildSalesNavigatorAccountApiRequestFromSearchUrl(slicedQueryUrl, config, pageSize);
|
|
@@ -12602,6 +14970,9 @@ program
|
|
|
12602
14970
|
},
|
|
12603
14971
|
exportSlice: async (attempt) => {
|
|
12604
14972
|
const parsedRequest = buildAccountApiRequest(attempt.slicedQueryUrl);
|
|
14973
|
+
const accountLocationFilter = filterLocationMismatches
|
|
14974
|
+
? buildSalesNavigatorAccountLocationFilter(attempt.slicedQueryUrl)
|
|
14975
|
+
: null;
|
|
12605
14976
|
try {
|
|
12606
14977
|
const fetchResult = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
12607
14978
|
requestedAccounts,
|
|
@@ -12610,6 +14981,7 @@ program
|
|
|
12610
14981
|
pageDelayMaxMs,
|
|
12611
14982
|
maxAllowedResults: attempt.maxResultsPerSearch,
|
|
12612
14983
|
retry,
|
|
14984
|
+
accountLocationFilter,
|
|
12613
14985
|
});
|
|
12614
14986
|
const stop = fetchResult.pacing.stop;
|
|
12615
14987
|
const partialError = describeLocalSalesNavigatorAccountPartialError(fetchResult);
|
|
@@ -12630,6 +15002,8 @@ program
|
|
|
12630
15002
|
slicedQueryUrl: attempt.slicedQueryUrl,
|
|
12631
15003
|
pacing: fetchResult.pacing,
|
|
12632
15004
|
stop,
|
|
15005
|
+
accountLocationFilter,
|
|
15006
|
+
filteredAccounts: fetchResult.filteredAccounts,
|
|
12633
15007
|
},
|
|
12634
15008
|
});
|
|
12635
15009
|
const storedAccounts = await upsertSalesNavigatorAccounts({
|
|
@@ -12639,7 +15013,10 @@ program
|
|
|
12639
15013
|
sliceId,
|
|
12640
15014
|
accounts: fetchResult.accounts,
|
|
12641
15015
|
});
|
|
12642
|
-
storedAccountCount
|
|
15016
|
+
storedAccountCount = await countSalesNavigatorAccountsForRun({
|
|
15017
|
+
store,
|
|
15018
|
+
runId,
|
|
15019
|
+
});
|
|
12643
15020
|
if (fetchResult.pacing.rateLimit) {
|
|
12644
15021
|
throw new SalesNavigatorCrawlStopError(fetchResult.pacing.rateLimit.message, {
|
|
12645
15022
|
totalResults: fetchResult.totalResults,
|
|
@@ -12739,6 +15116,9 @@ program
|
|
|
12739
15116
|
if (canFetchWindowedFinalSlice) {
|
|
12740
15117
|
try {
|
|
12741
15118
|
const parsedRequest = buildAccountApiRequest(failure.attempt.slicedQueryUrl);
|
|
15119
|
+
const accountLocationFilter = filterLocationMismatches
|
|
15120
|
+
? buildSalesNavigatorAccountLocationFilter(failure.attempt.slicedQueryUrl)
|
|
15121
|
+
: null;
|
|
12742
15122
|
const fetchResult = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
12743
15123
|
requestedAccounts,
|
|
12744
15124
|
pageSize,
|
|
@@ -12747,6 +15127,7 @@ program
|
|
|
12747
15127
|
maxAllowedResults: maxWindowedFinalResults,
|
|
12748
15128
|
retry,
|
|
12749
15129
|
allowPartialPageErrors: true,
|
|
15130
|
+
accountLocationFilter,
|
|
12750
15131
|
});
|
|
12751
15132
|
const windowedError = describeLocalSalesNavigatorAccountPartialError(fetchResult);
|
|
12752
15133
|
const sliceId = await upsertSalesNavigatorAccountSearchSlice({
|
|
@@ -12768,6 +15149,8 @@ program
|
|
|
12768
15149
|
originalError: failure.error,
|
|
12769
15150
|
pacing: fetchResult.pacing,
|
|
12770
15151
|
stop: fetchResult.pacing.stop,
|
|
15152
|
+
accountLocationFilter,
|
|
15153
|
+
filteredAccounts: fetchResult.filteredAccounts,
|
|
12771
15154
|
},
|
|
12772
15155
|
});
|
|
12773
15156
|
const storedAccounts = await upsertSalesNavigatorAccounts({
|
|
@@ -12777,7 +15160,10 @@ program
|
|
|
12777
15160
|
sliceId,
|
|
12778
15161
|
accounts: fetchResult.accounts,
|
|
12779
15162
|
});
|
|
12780
|
-
storedAccountCount
|
|
15163
|
+
storedAccountCount = await countSalesNavigatorAccountsForRun({
|
|
15164
|
+
store,
|
|
15165
|
+
runId,
|
|
15166
|
+
});
|
|
12781
15167
|
windowedFinalSlices.push({
|
|
12782
15168
|
slicedQueryUrl: failure.attempt.slicedQueryUrl,
|
|
12783
15169
|
appliedFilters: failure.attempt.appliedFilters,
|
|
@@ -13020,10 +15406,12 @@ program
|
|
|
13020
15406
|
program
|
|
13021
15407
|
.command("salesnav:accounts:local-import")
|
|
13022
15408
|
.alias("accounts:local-import")
|
|
13023
|
-
.description("Run a saved Sales Navigator account request locally and store the accounts in
|
|
15409
|
+
.description("Run a saved Sales Navigator account request locally and store the accounts in Supabase.")
|
|
13024
15410
|
.option("--curl-file <path>", "Path to a saved account-search curl request copied from the Sales Navigator network panel")
|
|
13025
15411
|
.option("--query-url <url>", "Sales Navigator account search URL. Uses the locally captured extension session automatically.")
|
|
13026
|
-
.option("--
|
|
15412
|
+
.option("--raw-results-file <path>", "Line-delimited raw Sales Navigator account results collected for --query-url")
|
|
15413
|
+
.option("--reported-total <number>", "Reported total for completeness verification when using --raw-results-file")
|
|
15414
|
+
.option("--org-id <id>", "Salesprompter workspace org id for Supabase storage. Defaults to SALESPROMPTER_ORG_ID or saved auth session org.")
|
|
13027
15415
|
.option("--max-results-per-search <number>", "Maximum expected account results for this local request", "1000")
|
|
13028
15416
|
.option("--number-of-accounts <number>", "Total accounts to import, or 'all' for the full reported result set", "all")
|
|
13029
15417
|
.option("--page-size <number>", "Accounts requested per local Sales Navigator API page", "25")
|
|
@@ -13035,8 +15423,9 @@ program
|
|
|
13035
15423
|
.option("--retry-max-delay-ms <number>", "Maximum delay for full-jitter retry backoff", "30000")
|
|
13036
15424
|
.option("--slice-preset <name>", "Slice preset label stored with the import run", "local-account-search")
|
|
13037
15425
|
.option("--allow-partial-page-errors", "Persist rows fetched before a later non-rate-limit page error", false)
|
|
15426
|
+
.option("--include-location-mismatches", "Store account rows even when the visible Sales Navigator account location does not match the source REGION filter", false)
|
|
13038
15427
|
.option("--out <path>", "Optional local JSON output path")
|
|
13039
|
-
.option("--dry-run", "Run and normalize the local account request without writing to
|
|
15428
|
+
.option("--dry-run", "Run and normalize the local account request without writing to Supabase", false)
|
|
13040
15429
|
.action(async (options) => {
|
|
13041
15430
|
const maxResultsPerSearch = z.coerce
|
|
13042
15431
|
.number()
|
|
@@ -13102,31 +15491,76 @@ program
|
|
|
13102
15491
|
.max(300000)
|
|
13103
15492
|
.parse(options.retryMaxDelayMs),
|
|
13104
15493
|
};
|
|
15494
|
+
const filterLocationMismatches = !Boolean(options.includeLocationMismatches);
|
|
13105
15495
|
const curlFile = typeof options.curlFile === "string" && options.curlFile.trim().length > 0
|
|
13106
15496
|
? options.curlFile.trim()
|
|
13107
15497
|
: null;
|
|
13108
15498
|
const queryUrl = typeof options.queryUrl === "string" && options.queryUrl.trim().length > 0
|
|
13109
15499
|
? z.string().url().parse(options.queryUrl.trim())
|
|
13110
15500
|
: null;
|
|
13111
|
-
|
|
13112
|
-
|
|
15501
|
+
const rawResultsFile = typeof options.rawResultsFile === "string" &&
|
|
15502
|
+
options.rawResultsFile.trim().length > 0
|
|
15503
|
+
? options.rawResultsFile.trim()
|
|
15504
|
+
: null;
|
|
15505
|
+
const reportedTotal = options.reportedTotal == null
|
|
15506
|
+
? null
|
|
15507
|
+
: z.coerce
|
|
15508
|
+
.number()
|
|
15509
|
+
.int()
|
|
15510
|
+
.min(1)
|
|
15511
|
+
.max(1_000_000)
|
|
15512
|
+
.parse(options.reportedTotal);
|
|
15513
|
+
if (rawResultsFile) {
|
|
15514
|
+
if (curlFile || !queryUrl) {
|
|
15515
|
+
throw new Error("--raw-results-file requires --query-url and cannot be combined with --curl-file.");
|
|
15516
|
+
}
|
|
15517
|
+
if (numberOfAccountsOption !== "all") {
|
|
15518
|
+
throw new Error("--raw-results-file requires --number-of-accounts all so completeness can be verified.");
|
|
15519
|
+
}
|
|
13113
15520
|
}
|
|
13114
|
-
|
|
13115
|
-
|
|
13116
|
-
|
|
13117
|
-
if (
|
|
13118
|
-
throw new Error("
|
|
15521
|
+
else if (Boolean(curlFile) === Boolean(queryUrl)) {
|
|
15522
|
+
throw new Error("Provide exactly one of --curl-file or --query-url, or use --raw-results-file with --query-url.");
|
|
15523
|
+
}
|
|
15524
|
+
else if (reportedTotal != null) {
|
|
15525
|
+
throw new Error("--reported-total is only supported with --raw-results-file.");
|
|
15526
|
+
}
|
|
15527
|
+
let accountLocationFilter = filterLocationMismatches
|
|
15528
|
+
? buildSalesNavigatorAccountLocationFilter(queryUrl ?? "")
|
|
15529
|
+
: null;
|
|
15530
|
+
let rawInputStats = null;
|
|
15531
|
+
let localFetch;
|
|
15532
|
+
if (rawResultsFile) {
|
|
15533
|
+
const rawInput = await readLocalSalesNavigatorRawAccountInput({
|
|
15534
|
+
filePath: rawResultsFile,
|
|
15535
|
+
sourceQueryUrl: queryUrl,
|
|
15536
|
+
reportedTotal,
|
|
15537
|
+
accountLocationFilter,
|
|
15538
|
+
});
|
|
15539
|
+
rawInputStats = rawInput.stats;
|
|
15540
|
+
localFetch = rawInput.fetchResult;
|
|
15541
|
+
}
|
|
15542
|
+
else {
|
|
15543
|
+
const parsedRequest = curlFile
|
|
15544
|
+
? parseSalesNavigatorCurlRequest(await readFile(curlFile, "utf8"))
|
|
15545
|
+
: buildSalesNavigatorAccountApiRequestFromSearchUrl(queryUrl, await readLinkedInAccountSearchConfig(queryUrl), pageSize);
|
|
15546
|
+
if (!new URL(parsedRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
15547
|
+
throw new Error("Account local import requires a /sales-api/salesApiAccountSearch request.");
|
|
15548
|
+
}
|
|
15549
|
+
if (filterLocationMismatches && !accountLocationFilter) {
|
|
15550
|
+
accountLocationFilter = buildSalesNavigatorAccountLocationFilter(queryUrl ?? parsedRequest.url);
|
|
15551
|
+
}
|
|
15552
|
+
localFetch = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
15553
|
+
requestedAccounts,
|
|
15554
|
+
startOffset,
|
|
15555
|
+
pageSize,
|
|
15556
|
+
pageDelayMinMs,
|
|
15557
|
+
pageDelayMaxMs,
|
|
15558
|
+
maxAllowedResults: maxResultsPerSearch,
|
|
15559
|
+
retry,
|
|
15560
|
+
allowPartialPageErrors: Boolean(options.allowPartialPageErrors),
|
|
15561
|
+
accountLocationFilter,
|
|
15562
|
+
});
|
|
13119
15563
|
}
|
|
13120
|
-
const localFetch = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
13121
|
-
requestedAccounts,
|
|
13122
|
-
startOffset,
|
|
13123
|
-
pageSize,
|
|
13124
|
-
pageDelayMinMs,
|
|
13125
|
-
pageDelayMaxMs,
|
|
13126
|
-
maxAllowedResults: maxResultsPerSearch,
|
|
13127
|
-
retry,
|
|
13128
|
-
allowPartialPageErrors: Boolean(options.allowPartialPageErrors),
|
|
13129
|
-
});
|
|
13130
15564
|
const accounts = localFetch.accounts;
|
|
13131
15565
|
const totalResults = localFetch.totalResults;
|
|
13132
15566
|
const dryRun = Boolean(options.dryRun);
|
|
@@ -13136,6 +15570,9 @@ program
|
|
|
13136
15570
|
if (accounts.length === 0) {
|
|
13137
15571
|
throw new Error("The local Sales Navigator account response did not contain any importable account rows.");
|
|
13138
15572
|
}
|
|
15573
|
+
if (rawResultsFile && partialError) {
|
|
15574
|
+
throw new Error(`Raw Sales Navigator account input is incomplete and was not stored. ${partialError}`);
|
|
15575
|
+
}
|
|
13139
15576
|
const sourceQueryUrl = queryUrl ?? localFetch.sourceQueryUrl;
|
|
13140
15577
|
const basePayload = {
|
|
13141
15578
|
status: "ok",
|
|
@@ -13144,6 +15581,7 @@ program
|
|
|
13144
15581
|
sourceQueryUrl,
|
|
13145
15582
|
totalResults,
|
|
13146
15583
|
discovered: accounts.length,
|
|
15584
|
+
filteredAccounts: localFetch.filteredAccounts,
|
|
13147
15585
|
fetchedPages: localFetch.fetchedPages,
|
|
13148
15586
|
requestedAccounts: requestedAccounts ?? "all",
|
|
13149
15587
|
maxResultsPerSearch,
|
|
@@ -13153,7 +15591,8 @@ program
|
|
|
13153
15591
|
rateLimit: rateLimit ?? null,
|
|
13154
15592
|
pageError: pageError ?? null,
|
|
13155
15593
|
stop: localFetch.pacing.stop,
|
|
13156
|
-
|
|
15594
|
+
rawInput: rawInputStats,
|
|
15595
|
+
accounts: rawResultsFile && !dryRun ? undefined : accounts,
|
|
13157
15596
|
};
|
|
13158
15597
|
if (dryRun) {
|
|
13159
15598
|
if (options.out) {
|
|
@@ -13188,6 +15627,9 @@ program
|
|
|
13188
15627
|
rateLimit: rateLimit ?? null,
|
|
13189
15628
|
pageError: pageError ?? null,
|
|
13190
15629
|
stop: localFetch.pacing.stop,
|
|
15630
|
+
accountLocationFilter,
|
|
15631
|
+
filteredAccounts: localFetch.filteredAccounts,
|
|
15632
|
+
rawInput: rawInputStats,
|
|
13191
15633
|
},
|
|
13192
15634
|
});
|
|
13193
15635
|
const attempt = {
|
|
@@ -13220,6 +15662,9 @@ program
|
|
|
13220
15662
|
startOffset,
|
|
13221
15663
|
pacing: localFetch.pacing,
|
|
13222
15664
|
stop: localFetch.pacing.stop,
|
|
15665
|
+
accountLocationFilter,
|
|
15666
|
+
filteredAccounts: localFetch.filteredAccounts,
|
|
15667
|
+
rawInput: rawInputStats,
|
|
13223
15668
|
},
|
|
13224
15669
|
});
|
|
13225
15670
|
const storedAccounts = await upsertSalesNavigatorAccounts({
|
|
@@ -13580,6 +16025,7 @@ program
|
|
|
13580
16025
|
.option("--max-retries <number>", "Bounded retries for transient network/5xx failures. Rate limits still stop immediately.", "1")
|
|
13581
16026
|
.option("--retry-base-delay-ms <number>", "Base delay for full-jitter retry backoff", "2000")
|
|
13582
16027
|
.option("--retry-max-delay-ms <number>", "Maximum delay for full-jitter retry backoff", "30000")
|
|
16028
|
+
.option("--include-location-mismatches", "Store account rows even when the visible Sales Navigator account location does not match the source REGION filter", false)
|
|
13583
16029
|
.option("--out <path>", "Optional local JSON output path")
|
|
13584
16030
|
.action(async (options) => {
|
|
13585
16031
|
const runId = z.string().uuid().parse(options.runId);
|
|
@@ -13647,6 +16093,7 @@ program
|
|
|
13647
16093
|
.max(300000)
|
|
13648
16094
|
.parse(options.retryMaxDelayMs),
|
|
13649
16095
|
};
|
|
16096
|
+
const filterLocationMismatches = !Boolean(options.includeLocationMismatches);
|
|
13650
16097
|
const templateRequest = parseSalesNavigatorCurlRequest(await readFile(options.curlFile, "utf8"));
|
|
13651
16098
|
if (!new URL(templateRequest.url).pathname.includes("/sales-api/salesApiAccountSearch")) {
|
|
13652
16099
|
throw new Error("Account repair --curl-file must contain a /sales-api/salesApiAccountSearch request.");
|
|
@@ -13701,6 +16148,9 @@ program
|
|
|
13701
16148
|
? requestedAccountsOption
|
|
13702
16149
|
: Math.min(requestedAccountsOption, remainingFromReported);
|
|
13703
16150
|
const parsedRequest = buildSalesNavigatorAccountApiRequestFromSearchUrlWithTemplate(slice.slicedQueryUrl, templateRequest, pageSize);
|
|
16151
|
+
const accountLocationFilter = filterLocationMismatches
|
|
16152
|
+
? buildSalesNavigatorAccountLocationFilter(slice.slicedQueryUrl)
|
|
16153
|
+
: null;
|
|
13704
16154
|
const fetchResult = await fetchAllLocalSalesNavigatorAccounts(parsedRequest, {
|
|
13705
16155
|
requestedAccounts,
|
|
13706
16156
|
startOffset,
|
|
@@ -13710,6 +16160,7 @@ program
|
|
|
13710
16160
|
maxAllowedResults: 100000,
|
|
13711
16161
|
retry,
|
|
13712
16162
|
allowPartialPageErrors: true,
|
|
16163
|
+
accountLocationFilter,
|
|
13713
16164
|
});
|
|
13714
16165
|
const rateLimit = fetchResult.pacing.rateLimit;
|
|
13715
16166
|
const pageError = fetchResult.pacing.pageError;
|
|
@@ -13741,6 +16192,8 @@ program
|
|
|
13741
16192
|
previousStoredAccounts: slice.storedAccounts,
|
|
13742
16193
|
startOffset,
|
|
13743
16194
|
fetchedAccounts: fetchResult.accounts.length,
|
|
16195
|
+
filteredAccounts: fetchResult.filteredAccounts,
|
|
16196
|
+
accountLocationFilter,
|
|
13744
16197
|
},
|
|
13745
16198
|
pacing: fetchResult.pacing,
|
|
13746
16199
|
stop: fetchResult.pacing.stop,
|