salesprompter-cli 0.1.56 → 0.1.58

package/README.md

@@ -66,9 +66,9 @@ salesprompter companies:resolve-linkedin-urls --in ./companies.txt --out ./compa
 salesprompter contacts:resolve-emails --in ./contacts.tsv --out-dir ./email-run --dry-run
 
 # Collect a Sales Navigator people search into the active workspace
+# Oversized or pagination-short searches are split by company headcount and deduplicated.
 salesprompter leads:collect \
-  --linkedin-url "$SALES_NAV_PEOPLE_URL" \
-  --max-results 2000
+  --linkedin-url "$SALES_NAV_PEOPLE_URL"
 
 # Enrich one or more stored imports: Sales Navigator company -> official domain -> Hunter email
 # This updates workspace data only. It does not create or start outreach.

package/dist/cli.js

@@ -31,7 +31,7 @@ import { crawlLinkedInProductCategory } from "./linkedin-products.js";
 import { claimLinkedInSessionCookieForCli, claimValidatedSalesNavigatorSessionCookieForCli, createLinkedInSessionSupabaseClient, recordLinkedInSessionCookieAudit, resolveConfiguredEnvValue } from "./linkedin-session.js";
 import { buildLeadlistsFunnelQueries } from "./leadlists-funnel.js";
 import { readJsonFile, splitCsv, writeJsonFile, writeTextFile } from "./io.js";
-import { buildSalesNavigatorCrawlPreview, buildSalesNavigatorAccountQuery, createSalesNavigatorCrawlSeed, DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS, DEFAULT_SALES_NAVIGATOR_CRAWL_DIMENSIONS, buildSalesNavigatorPeopleSearchUrl, buildSalesNavigatorPeopleSlice, deriveSalesNavigatorTitleQuerySeeds, executeSalesNavigatorAdaptiveCrawl, expandSalesNavigatorCrawlAttempt, SalesNavigatorCrawlStopError, SalesNavigatorSliceTooBroadError } from "./sales-navigator.js";
+import { buildSalesNavigatorCrawlPreview, buildSalesNavigatorAccountQuery, buildSalesNavigatorResidualCrawlAttempt, createSalesNavigatorCrawlSeed, DEFAULT_SALES_NAVIGATOR_ACCOUNT_CRAWL_DIMENSIONS, DEFAULT_SALES_NAVIGATOR_CRAWL_DIMENSIONS, buildSalesNavigatorPeopleSearchUrl, buildSalesNavigatorPeopleSlice, deriveSalesNavigatorTitleQuerySeeds, executeSalesNavigatorAdaptiveCrawl, expandSalesNavigatorCrawlAttempt, getSalesNavigatorPeopleAccessibleResultLimit, salesNavigatorPeopleSearchHasFilter, SalesNavigatorCrawlStopError, SalesNavigatorSliceTooBroadError } from "./sales-navigator.js";
 import { buildSalesNavigatorHistoricalBackfillPlan, ensureSalesNavigatorPeopleCount, resolveSalesNavigatorHistoricalBackfillConfig, resolveSalesNavigatorHistoricalBackfillResumeState, resolveSalesNavigatorHistoricalBackfillOrgId, salesNavigatorHistoricalBackfillDefaults } from "./salesnav-backfill.js";
 const require = createRequire(import.meta.url);
 const { version: packageVersion } = require("../package.json");
@@ -8676,6 +8676,18 @@ async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
         fetchedPages += 1;
         totalResults =
             totalResults ?? extractLocalSalesNavigatorTotalResults(responseBody);
+        if (totalResults != null &&
+            options.maxAllowedResults != null &&
+            totalResults > Math.max(1, Math.trunc(options.maxAllowedResults))) {
+            throw new SalesNavigatorSliceTooBroadError(`People slice has ${totalResults} results, above ${options.maxAllowedResults}.`, {
+                totalResults,
+                details: {
+                    fetchedPages,
+                    retryCount,
+                    totalDelayMs,
+                },
+            });
+        }
         const pagePeople = normalizeLocalSalesNavigatorPeople(responseBody, pageRequest.url);
         let added = 0;
         for (const person of pagePeople) {
@@ -8712,6 +8724,151 @@ async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
         },
     };
 }
+function canonicalLocalSalesNavigatorLeadKey(person) {
+    try {
+        const url = new URL(person.profileUrl);
+        url.hash = "";
+        url.search = "";
+        url.hostname = url.hostname.toLowerCase();
+        url.pathname = url.pathname.replace(/\/+$/, "");
+        return url.toString();
+    }
+    catch {
+        return person.profileUrl.trim().toLowerCase();
+    }
+}
+function localPeopleSplitDimensions(sourceQueryUrl) {
+    return DEFAULT_SALES_NAVIGATOR_CRAWL_DIMENSIONS.filter((dimension) => !salesNavigatorPeopleSearchHasFilter(sourceQueryUrl, dimension.filterType));
+}
+async function fetchCompleteLocalSalesNavigatorPeople(sourceQueryUrl, requestHeaders, options) {
+    const root = createSalesNavigatorCrawlSeed({
+        sourceQueryUrl,
+        baselineFilters: [],
+        maxResultsPerSearch: options.maxResultsPerSearch,
+        numberOfProfiles: options.maxResultsPerSearch,
+        slicePreset: "local-salesnav-people-adaptive",
+        searchType: "people",
+    });
+    const dimensions = localPeopleSplitDimensions(root.sourceQueryUrl);
+    const queue = [root];
+    const queuedUrls = new Set([root.slicedQueryUrl]);
+    const peopleByKey = new Map();
+    const slices = [];
+    let rootTotalResults = null;
+    let fetchedPages = 0;
+    let totalDelayMs = 0;
+    let retryCount = 0;
+    while (queue.length > 0) {
+        const attempt = queue.shift();
+        const parsedRequest = {
+            url: buildSalesNavigatorLeadApiUrlFromSearchUrl(attempt.slicedQueryUrl, options.pageSize),
+            headers: requestHeaders,
+        };
+        let collected = null;
+        let tooBroadTotal = null;
+        let probeFetchedPages = 0;
+        try {
+            collected = await fetchAllLocalSalesNavigatorPeople(parsedRequest, {
+                requestedProfiles: options.maxResultsPerSearch,
+                maxAllowedResults: options.maxResultsPerSearch,
+                pageSize: options.pageSize,
+                pageDelayMinMs: options.pageDelayMinMs,
+                pageDelayMaxMs: options.pageDelayMaxMs,
+                retry: options.retry,
+                executeRequest: options.executeRequest,
+            });
+            fetchedPages += collected.fetchedPages;
+            totalDelayMs += collected.pacing.totalDelayMs;
+            retryCount += collected.pacing.retryCount;
+            if (attempt.depth === 0) {
+                rootTotalResults = collected.totalResults;
+            }
+        }
+        catch (error) {
+            if (!(error instanceof SalesNavigatorSliceTooBroadError)) {
+                throw error;
+            }
+            tooBroadTotal = error.totalResults;
+            const details = error.details && typeof error.details === "object"
+                ? error.details
+                : {};
+            probeFetchedPages = Number(details.fetchedPages ?? 1);
+            fetchedPages += probeFetchedPages;
+            retryCount += Number(details.retryCount ?? 0);
+            totalDelayMs += Number(details.totalDelayMs ?? 0);
+            if (attempt.depth === 0) {
+                rootTotalResults = tooBroadTotal;
+            }
+        }
+        const needsSplit = tooBroadTotal != null ||
+            (collected?.totalResults == null &&
+                collected != null &&
+                collected.people.length >= options.maxResultsPerSearch) ||
+            (collected?.totalResults != null &&
+                collected.people.length < collected.totalResults);
+        if (needsSplit) {
+            const usedDimensionKeys = new Set(attempt.splitTrail.map((entry) => entry.key.replace(/-residual$/, "")));
+            const nextDimension = dimensions.find((dimension) => !usedDimensionKeys.has(dimension.key));
+            if (!nextDimension) {
+                throw new Error(`Sales Navigator search still exceeds the ${options.maxResultsPerSearch}-person window after all safe split dimensions were used. Narrow the source search further.`);
+            }
+            const children = expandSalesNavigatorCrawlAttempt(attempt, nextDimension, "people");
+            if (nextDimension.supportsResidualExclusion !== false) {
+                children.push(buildSalesNavigatorResidualCrawlAttempt(attempt, nextDimension, "people"));
+            }
+            for (const child of children) {
+                if (queuedUrls.has(child.slicedQueryUrl))
+                    continue;
+                queuedUrls.add(child.slicedQueryUrl);
+                queue.push(child);
+            }
+            slices.push({
+                depth: attempt.depth,
+                splitTrail: attempt.splitTrail,
+                totalResults: tooBroadTotal ?? collected?.totalResults ?? null,
+                collected: 0,
+                fetchedPages: probeFetchedPages || collected?.fetchedPages || 0,
+                status: "split",
+                splitDimension: nextDimension.key,
+            });
+            continue;
+        }
+        if (!collected) {
+            throw new Error("Sales Navigator slice did not return a collection result.");
+        }
+        for (const person of collected.people) {
+            peopleByKey.set(canonicalLocalSalesNavigatorLeadKey(person), person);
+        }
+        slices.push({
+            depth: attempt.depth,
+            splitTrail: attempt.splitTrail,
+            totalResults: collected.totalResults,
+            collected: collected.people.length,
+            fetchedPages: collected.fetchedPages,
+            status: "collected",
+            splitDimension: null,
+        });
+    }
+    const people = [...peopleByKey.values()];
+    if (rootTotalResults != null && people.length < rootTotalResults) {
+        throw new Error(`Adaptive collection found ${people.length} unique people but the root search reported ${rootTotalResults}. No import was written because coverage is incomplete.`);
+    }
+    return {
+        people,
+        totalResults: rootTotalResults,
+        rootTotalResults,
+        fetchedPages,
+        sourceQueryUrl,
+        collectionMode: "adaptive",
+        slices,
+        pacing: {
+            delayMinMs: options.pageDelayMinMs,
+            delayMaxMs: options.pageDelayMaxMs,
+            totalDelayMs,
+            retryCount,
+        },
+    };
+}
 async function fetchAllLocalSalesNavigatorAccounts(parsedRequest, options) {
     const accounts = [];
     const seen = new Set();
@@ -8913,6 +9070,7 @@ async function importLocalSalesNavigatorPeopleViaApp(session, payload) {
             fetchedPages: payload.fetchedPages,
             people: payload.people,
             rawPayload: {
+                ...payload.rawPayload,
                 clientId: workspaceClientId ?? undefined,
                 client_id: workspaceClientId ?? undefined,
                 leadList: workspaceClientId
@@ -13383,15 +13541,8 @@ program
         }
     });
 });
-const SALES_NAVIGATOR_PEOPLE_MAX_RESULTS = 2_000;
 async function runSalesNavigatorPeopleCollectCommand(options) {
     const linkedInUrl = z.string().url().parse(options.linkedinUrl);
-    const maxResults = z.coerce
-        .number()
-        .int()
-        .min(1)
-        .max(SALES_NAVIGATOR_PEOPLE_MAX_RESULTS)
-        .parse(options.maxResults);
     const pageSize = z.coerce.number().int().min(1).max(100).parse(options.pageSize);
     const normalizedSourceUrl = (() => {
         const source = new URL(linkedInUrl);
@@ -13401,12 +13552,24 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
         }
         return source.toString();
     })();
+    const accessibleResultLimit = getSalesNavigatorPeopleAccessibleResultLimit(normalizedSourceUrl);
+    const maxResults = options.maxResults == null
+        ? accessibleResultLimit
+        : z.coerce
+            .number()
+            .int()
+            .min(1)
+            .max(accessibleResultLimit)
+            .parse(options.maxResults);
+    const complete = Boolean(options.complete || options.maxResults == null);
     if (options.dryRun) {
         const payload = {
             status: "ok",
             dryRun: true,
             linkedInUrl: normalizedSourceUrl,
             maxResults,
+            accessibleResultLimit,
+            complete,
             destination: "/leads/cli-imports"
         };
         if (options.out)
@@ -13438,24 +13601,35 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
         : await createLocalAccountSearchBrowserRelay(browserRelayPort);
     let collected;
     try {
-        collected = await fetchAllLocalSalesNavigatorPeople(parsedRequest, {
-            requestedProfiles: maxResults,
+        const collectorOptions = {
             pageSize,
             pageDelayMinMs,
             pageDelayMaxMs,
             retry: {
                 maxRetries: 2,
                 retryBaseDelayMs: 2_000,
-                retryMaxDelayMs: 10_000
+                retryMaxDelayMs: 10_000,
             },
             executeRequest: browserRelay
                 ? (request) => browserRelay.request(request)
-                : undefined
-        });
+                : undefined,
+        };
+        collected = complete
+            ? await fetchCompleteLocalSalesNavigatorPeople(normalizedSourceUrl, parsedRequest.headers, {
+                maxResultsPerSearch: maxResults,
+                ...collectorOptions,
+            })
+            : await fetchAllLocalSalesNavigatorPeople(parsedRequest, {
+                requestedProfiles: maxResults,
+                ...collectorOptions,
+            });
     }
     finally {
         await browserRelay?.close();
     }
+    const adaptiveCollection = "collectionMode" in collected && collected.collectionMode === "adaptive"
+        ? collected
+        : null;
     const imported = await importLocalSalesNavigatorPeopleViaApp(session, {
         sourceQueryUrl: normalizedSourceUrl,
         people: collected.people,
@@ -13463,8 +13637,21 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
         fetchedPages: collected.fetchedPages,
         pacing: collected.pacing,
         maxResultsPerSearch: maxResults,
-        numberOfProfiles: maxResults,
-        slicePreset: "local-salesnav-people"
+        numberOfProfiles: adaptiveCollection ? collected.people.length : maxResults,
+        slicePreset: adaptiveCollection
+            ? "local-salesnav-people-adaptive"
+            : "local-salesnav-people",
+        rawPayload: adaptiveCollection
+            ? {
+                collectionMode: adaptiveCollection.collectionMode,
+                accessibleResultLimit,
+                sliceCount: adaptiveCollection.slices.filter((slice) => slice.status === "collected").length,
+                slices: adaptiveCollection.slices,
+            }
+            : {
+                collectionMode: "bounded",
+                accessibleResultLimit,
+            },
     });
     const payload = {
         status: "ok",
@@ -13480,6 +13667,10 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
             ? null
             : Math.max(0, collected.totalResults - collected.people.length),
         fetchedPages: collected.fetchedPages,
+        accessibleResultLimit,
+        collectionMode: adaptiveCollection ? "adaptive" : "bounded",
+        sliceCount: adaptiveCollection?.slices.filter((slice) => slice.status === "collected")
+            .length ?? 1,
         runId: imported.runId,
         imported: imported.imported,
         upserted: imported.upserted,
@@ -14247,12 +14438,15 @@ async function runSalesNavigatorImportEnrichmentCommand(options) {
 async function runAffiliateLaunchCommand(options) {
     const affiliateLink = z.string().url().parse(options.affiliateLink);
     const linkedInUrl = z.string().url().parse(options.linkedinUrl);
-    const maxResults = z.coerce
-        .number()
-        .int()
-        .min(1)
-        .max(SALES_NAVIGATOR_PEOPLE_MAX_RESULTS)
-        .parse(options.maxResults);
+    const accessibleResultLimit = getSalesNavigatorPeopleAccessibleResultLimit(linkedInUrl);
+    const maxResults = options.maxResults == null
+        ? accessibleResultLimit
+        : z.coerce
+            .number()
+            .int()
+            .min(1)
+            .max(accessibleResultLimit)
+            .parse(options.maxResults);
     const session = await requireAuthSession();
     const isSalesNavigatorPeopleSearch = new URL(linkedInUrl).pathname.includes("/sales/search/people");
     let localResults;
@@ -14325,7 +14519,7 @@ function addAffiliateAudienceOptions(command) {
     return command
         .requiredOption("--affiliate-link <url>", "Affiliate link to promote")
         .requiredOption("--linkedin-url <url>", "Sales Navigator people or LinkedIn content search URL")
-        .option("--max-results <number>", "Maximum audience results (people searches: 2000)", String(SALES_NAVIGATOR_PEOPLE_MAX_RESULTS))
+        .option("--max-results <number>", "Maximum audience results (people searches: 2500; Connections of: 1000)")
         .option("--curl-file <path>", "Optional copied Sales Navigator Lead Search curl request")
         .option("--browser-relay-port <number>", "Use the signed-in browser through a loopback relay")
         .option("--page-size <number>", "Direct Sales Navigator page size", "100")
@@ -14339,7 +14533,8 @@ program
     .alias("leads:collect")
     .description("Collect a Sales Navigator people search into the active workspace.")
     .requiredOption("--linkedin-url <url>", "Sales Navigator people search URL")
-    .option("--max-results <number>", "Maximum people to collect", String(SALES_NAVIGATOR_PEOPLE_MAX_RESULTS))
+    .option("--max-results <number>", "Bounded collection size; defaults to exhaustive collection with a 2500-person window")
+    .option("--complete", "Split oversized searches by company headcount and deduplicate the complete result", false)
     .option("--curl-file <path>", "Optional copied Sales Navigator Lead Search curl request")
     .option("--browser-relay-port <number>", "Use the signed-in browser through a loopback relay")
     .option("--page-size <number>", "Direct Sales Navigator page size", "100")

package/dist/sales-navigator.js

@@ -92,6 +92,45 @@ export const DEFAULT_SALES_NAVIGATOR_PEOPLE_SLICE_FILTERS = [
 ];
 const LINKEDIN_SALES_NAVIGATOR_PEOPLE_SEARCH_URL = "https://www.linkedin.com/sales/search/people";
 const LINKEDIN_SALES_NAVIGATOR_ACCOUNT_SEARCH_URL = "https://www.linkedin.com/sales/search/company";
+export const SALES_NAVIGATOR_PEOPLE_MAX_RESULTS = 2_500;
+export const SALES_NAVIGATOR_CONNECTIONS_OF_MAX_RESULTS = 1_000;
+export const SALES_NAVIGATOR_ACCOUNT_MAX_RESULTS = 1_000;
+function decodeSalesNavigatorQueryValue(value) {
+    let decoded = value;
+    for (let attempt = 0; attempt < 3; attempt += 1) {
+        try {
+            const next = decodeURIComponent(decoded);
+            if (next === decoded)
+                break;
+            decoded = next;
+        }
+        catch {
+            break;
+        }
+    }
+    return decoded;
+}
+function getSalesNavigatorQueryValue(sourceQueryUrl) {
+    const url = new URL(sourceQueryUrl);
+    const queryFromSearch = url.searchParams.get("query");
+    const queryFromHash = url.hash.startsWith("#query=")
+        ? new URLSearchParams(url.hash.slice(1)).get("query")
+        : null;
+    return decodeSalesNavigatorQueryValue(queryFromSearch ?? queryFromHash ?? "");
+}
+export function isSalesNavigatorConnectionsOfSearch(sourceQueryUrl) {
+    const queryValue = getSalesNavigatorQueryValue(sourceQueryUrl);
+    return /(?:type\s*:\s*)?(?:CONNECTIONS?_OF|CONNECTION_OF|TEAMLINK_CONNECTION_OF)\b/i.test(queryValue) || /\bconnections?\s+of\b/i.test(queryValue);
+}
+export function getSalesNavigatorPeopleAccessibleResultLimit(sourceQueryUrl) {
+    return isSalesNavigatorConnectionsOfSearch(sourceQueryUrl)
+        ? SALES_NAVIGATOR_CONNECTIONS_OF_MAX_RESULTS
+        : SALES_NAVIGATOR_PEOPLE_MAX_RESULTS;
+}
+export function salesNavigatorPeopleSearchHasFilter(sourceQueryUrl, filterType) {
+    const escapedFilterType = filterType.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    return new RegExp(`\\btype\\s*:\\s*${escapedFilterType}\\b`, "i").test(getSalesNavigatorQueryValue(sourceQueryUrl));
+}
 export const DEFAULT_SALES_NAVIGATOR_CRAWL_BASELINE_FILTERS = [
     {
         type: "FUNCTION",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "salesprompter-cli",
-  "version": "0.1.56",
+  "version": "0.1.58",
   "description": "Sales workflow CLI for guided lead generation, enrichment, scoring, and sync.",
   "author": "Daniel Sinewe <hello@danielsinewe.com>",
   "type": "module",