salesprompter-cli 0.1.58 → 0.1.60

package/README.md

@@ -66,7 +66,8 @@ salesprompter companies:resolve-linkedin-urls --in ./companies.txt --out ./compa
 salesprompter contacts:resolve-emails --in ./contacts.tsv --out-dir ./email-run --dry-run
 
 # Collect a Sales Navigator people search into the active workspace
-# Oversized or pagination-short searches are split by company headcount and deduplicated.
+# Normal people searches use 2,500-result slices; Connections-of searches use 1,000.
+# Oversized searches split by company headcount and deduplicate by canonical profile URL.
 salesprompter leads:collect \
   --linkedin-url "$SALES_NAV_PEOPLE_URL"
 

package/dist/cli.js

@@ -8019,7 +8019,7 @@ async function createLocalAccountSearchBrowserRelay(requestedPort) {
         },
     };
 }
-const LOCAL_ACCOUNT_SEARCH_COLLECTOR_RESULT_WINDOW = 1600;
+const LOCAL_ACCOUNT_SEARCH_COLLECTOR_RESULT_WINDOW = 1000;
 const LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS_VERSION = 3;
 const LOCAL_ACCOUNT_SEARCH_COLLECTOR_COVERAGE_PASS = "coverage-complete";
 const LOCAL_ACCOUNT_SEARCH_COLLECTOR_PASSES = [
@@ -8724,6 +8724,7 @@ async function fetchAllLocalSalesNavigatorPeople(parsedRequest, options) {
         },
     };
 }
+const LOCAL_SALES_NAVIGATOR_REPORTED_COUNT_DRIFT_LIMIT = 5;
 function canonicalLocalSalesNavigatorLeadKey(person) {
     try {
         const url = new URL(person.profileUrl);
@@ -8752,111 +8753,161 @@ async function fetchCompleteLocalSalesNavigatorPeople(sourceQueryUrl, requestHea
     const dimensions = localPeopleSplitDimensions(root.sourceQueryUrl);
     const queue = [root];
     const queuedUrls = new Set([root.slicedQueryUrl]);
+    const rootRecoveryDimensionKeys = new Set();
     const peopleByKey = new Map();
     const slices = [];
     let rootTotalResults = null;
     let fetchedPages = 0;
     let totalDelayMs = 0;
     let retryCount = 0;
-    while (queue.length > 0) {
-        const attempt = queue.shift();
-        const parsedRequest = {
-            url: buildSalesNavigatorLeadApiUrlFromSearchUrl(attempt.slicedQueryUrl, options.pageSize),
-            headers: requestHeaders,
-        };
-        let collected = null;
-        let tooBroadTotal = null;
-        let probeFetchedPages = 0;
-        try {
-            collected = await fetchAllLocalSalesNavigatorPeople(parsedRequest, {
-                requestedProfiles: options.maxResultsPerSearch,
-                maxAllowedResults: options.maxResultsPerSearch,
-                pageSize: options.pageSize,
-                pageDelayMinMs: options.pageDelayMinMs,
-                pageDelayMaxMs: options.pageDelayMaxMs,
-                retry: options.retry,
-                executeRequest: options.executeRequest,
-            });
-            fetchedPages += collected.fetchedPages;
-            totalDelayMs += collected.pacing.totalDelayMs;
-            retryCount += collected.pacing.retryCount;
-            if (attempt.depth === 0) {
-                rootTotalResults = collected.totalResults;
-            }
-        }
-        catch (error) {
-            if (!(error instanceof SalesNavigatorSliceTooBroadError)) {
-                throw error;
+    while (true) {
+        while (queue.length > 0) {
+            const attempt = queue.shift();
+            const parsedRequest = {
+                url: buildSalesNavigatorLeadApiUrlFromSearchUrl(attempt.slicedQueryUrl, options.pageSize),
+                headers: requestHeaders,
+            };
+            let collected = null;
+            let tooBroadTotal = null;
+            let probeFetchedPages = 0;
+            try {
+                collected = await fetchAllLocalSalesNavigatorPeople(parsedRequest, {
+                    requestedProfiles: options.maxResultsPerSearch,
+                    maxAllowedResults: options.maxResultsPerSearch,
+                    pageSize: options.pageSize,
+                    pageDelayMinMs: options.pageDelayMinMs,
+                    pageDelayMaxMs: options.pageDelayMaxMs,
+                    retry: options.retry,
+                    executeRequest: options.executeRequest,
+                });
+                fetchedPages += collected.fetchedPages;
+                totalDelayMs += collected.pacing.totalDelayMs;
+                retryCount += collected.pacing.retryCount;
+                if (attempt.depth === 0) {
+                    rootTotalResults = collected.totalResults;
+                }
             }
-            tooBroadTotal = error.totalResults;
-            const details = error.details && typeof error.details === "object"
-                ? error.details
-                : {};
-            probeFetchedPages = Number(details.fetchedPages ?? 1);
-            fetchedPages += probeFetchedPages;
-            retryCount += Number(details.retryCount ?? 0);
-            totalDelayMs += Number(details.totalDelayMs ?? 0);
-            if (attempt.depth === 0) {
-                rootTotalResults = tooBroadTotal;
+            catch (error) {
+                if (!(error instanceof SalesNavigatorSliceTooBroadError)) {
+                    throw error;
+                }
+                tooBroadTotal = error.totalResults;
+                const details = error.details && typeof error.details === "object"
+                    ? error.details
+                    : {};
+                probeFetchedPages = Number(details.fetchedPages ?? 1);
+                fetchedPages += probeFetchedPages;
+                retryCount += Number(details.retryCount ?? 0);
+                totalDelayMs += Number(details.totalDelayMs ?? 0);
+                if (attempt.depth === 0) {
+                    rootTotalResults = tooBroadTotal;
+                }
             }
-        }
-        const needsSplit = tooBroadTotal != null ||
-            (collected?.totalResults == null &&
-                collected != null &&
-                collected.people.length >= options.maxResultsPerSearch) ||
-            (collected?.totalResults != null &&
-                collected.people.length < collected.totalResults);
-        if (needsSplit) {
-            const usedDimensionKeys = new Set(attempt.splitTrail.map((entry) => entry.key.replace(/-residual$/, "")));
-            const nextDimension = dimensions.find((dimension) => !usedDimensionKeys.has(dimension.key));
-            if (!nextDimension) {
-                throw new Error(`Sales Navigator search still exceeds the ${options.maxResultsPerSearch}-person window after all safe split dimensions were used. Narrow the source search further.`);
+            const needsSplit = tooBroadTotal != null ||
+                (collected?.totalResults == null &&
+                    collected != null &&
+                    collected.people.length >= options.maxResultsPerSearch) ||
+                (attempt.depth === 0 &&
+                    collected?.totalResults != null &&
+                    collected.people.length < collected.totalResults &&
+                    collected.totalResults - collected.people.length >
+                        LOCAL_SALES_NAVIGATOR_REPORTED_COUNT_DRIFT_LIMIT);
+            if (needsSplit) {
+                const usedDimensionKeys = new Set(attempt.splitTrail.map((entry) => entry.key.replace(/-residual$/, "")));
+                const nextDimension = dimensions.find((dimension) => !usedDimensionKeys.has(dimension.key));
+                if (!nextDimension) {
+                    throw new Error(`Sales Navigator search could not be completed within the ${options.maxResultsPerSearch}-person window after all safe split dimensions were used. Narrow the source search further.`);
+                }
+                if (attempt.depth === 0) {
+                    rootRecoveryDimensionKeys.add(nextDimension.key);
+                }
+                const children = expandSalesNavigatorCrawlAttempt(attempt, nextDimension, "people");
+                if (nextDimension.supportsResidualExclusion !== false) {
+                    children.push(buildSalesNavigatorResidualCrawlAttempt(attempt, nextDimension, "people"));
+                }
+                for (const child of children) {
+                    if (queuedUrls.has(child.slicedQueryUrl))
+                        continue;
+                    queuedUrls.add(child.slicedQueryUrl);
+                    queue.push(child);
+                }
+                slices.push({
+                    depth: attempt.depth,
+                    splitTrail: attempt.splitTrail,
+                    totalResults: tooBroadTotal ?? collected?.totalResults ?? null,
+                    collected: 0,
+                    fetchedPages: probeFetchedPages || collected?.fetchedPages || 0,
+                    status: "split",
+                    splitDimension: nextDimension.key,
+                });
+                continue;
             }
-            const children = expandSalesNavigatorCrawlAttempt(attempt, nextDimension, "people");
-            if (nextDimension.supportsResidualExclusion !== false) {
-                children.push(buildSalesNavigatorResidualCrawlAttempt(attempt, nextDimension, "people"));
+            if (!collected) {
+                throw new Error("Sales Navigator slice did not return a collection result.");
             }
-            for (const child of children) {
-                if (queuedUrls.has(child.slicedQueryUrl))
-                    continue;
-                queuedUrls.add(child.slicedQueryUrl);
-                queue.push(child);
+            for (const person of collected.people) {
+                peopleByKey.set(canonicalLocalSalesNavigatorLeadKey(person), person);
             }
             slices.push({
                 depth: attempt.depth,
                 splitTrail: attempt.splitTrail,
-                totalResults: tooBroadTotal ?? collected?.totalResults ?? null,
-                collected: 0,
-                fetchedPages: probeFetchedPages || collected?.fetchedPages || 0,
-                status: "split",
-                splitDimension: nextDimension.key,
+                totalResults: collected.totalResults,
+                collected: collected.people.length,
+                fetchedPages: collected.fetchedPages,
+                status: "collected",
+                splitDimension: null,
             });
-            continue;
         }
-        if (!collected) {
-            throw new Error("Sales Navigator slice did not return a collection result.");
+        const currentReportedCountDrift = rootTotalResults == null
+            ? 0
+            : Math.max(0, rootTotalResults - peopleByKey.size);
+        const currentReportedCountDriftAccepted = rootTotalResults != null &&
+            rootTotalResults <= options.maxResultsPerSearch &&
+            currentReportedCountDrift <=
+                LOCAL_SALES_NAVIGATOR_REPORTED_COUNT_DRIFT_LIMIT;
+        if (rootTotalResults == null ||
+            peopleByKey.size >= rootTotalResults ||
+            currentReportedCountDriftAccepted) {
+            break;
         }
-        for (const person of collected.people) {
-            peopleByKey.set(canonicalLocalSalesNavigatorLeadKey(person), person);
+        const nextRecoveryDimension = dimensions.find((dimension) => !rootRecoveryDimensionKeys.has(dimension.key));
+        if (!nextRecoveryDimension) {
+            break;
+        }
+        rootRecoveryDimensionKeys.add(nextRecoveryDimension.key);
+        const recoveryChildren = expandSalesNavigatorCrawlAttempt(root, nextRecoveryDimension, "people");
+        if (nextRecoveryDimension.supportsResidualExclusion !== false) {
+            recoveryChildren.push(buildSalesNavigatorResidualCrawlAttempt(root, nextRecoveryDimension, "people"));
+        }
+        for (const child of recoveryChildren) {
+            if (queuedUrls.has(child.slicedQueryUrl))
+                continue;
+            queuedUrls.add(child.slicedQueryUrl);
+            queue.push(child);
         }
         slices.push({
-            depth: attempt.depth,
-            splitTrail: attempt.splitTrail,
-            totalResults: collected.totalResults,
-            collected: collected.people.length,
-            fetchedPages: collected.fetchedPages,
-            status: "collected",
-            splitDimension: null,
+            depth: 0,
+            splitTrail: [],
+            totalResults: rootTotalResults,
+            collected: 0,
+            fetchedPages: 0,
+            status: "split",
+            splitDimension: nextRecoveryDimension.key,
         });
     }
     const people = [...peopleByKey.values()];
-    if (rootTotalResults != null && people.length < rootTotalResults) {
+    const reportedCountDrift = rootTotalResults == null ? 0 : Math.max(0, rootTotalResults - people.length);
+    const acceptedReportedCountDrift = rootTotalResults != null &&
+        rootTotalResults <= options.maxResultsPerSearch &&
+        reportedCountDrift <= LOCAL_SALES_NAVIGATOR_REPORTED_COUNT_DRIFT_LIMIT;
+    if (reportedCountDrift > 0 && !acceptedReportedCountDrift) {
         throw new Error(`Adaptive collection found ${people.length} unique people but the root search reported ${rootTotalResults}. No import was written because coverage is incomplete.`);
     }
     return {
         people,
         totalResults: rootTotalResults,
         rootTotalResults,
+        reportedCountDrift,
         fetchedPages,
         sourceQueryUrl,
         collectionMode: "adaptive",
@@ -13645,6 +13696,7 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
             ? {
                 collectionMode: adaptiveCollection.collectionMode,
                 accessibleResultLimit,
+                reportedCountDrift: adaptiveCollection.reportedCountDrift,
                 sliceCount: adaptiveCollection.slices.filter((slice) => slice.status === "collected").length,
                 slices: adaptiveCollection.slices,
             }
@@ -13658,14 +13710,19 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
         dryRun: false,
         collected: collected.people.length,
         totalResults: collected.totalResults,
-        collectionComplete: collected.totalResults != null &&
-            collected.people.length >= collected.totalResults,
-        truncated: collected.totalResults != null &&
+        collectionComplete: adaptiveCollection != null ||
+            (collected.totalResults != null &&
+                collected.people.length >= collected.totalResults),
+        truncated: adaptiveCollection == null &&
+            collected.totalResults != null &&
             collected.totalResults > collected.people.length &&
             collected.people.length >= maxResults,
-        remainingResults: collected.totalResults == null
-            ? null
-            : Math.max(0, collected.totalResults - collected.people.length),
+        remainingResults: adaptiveCollection != null
+            ? 0
+            : collected.totalResults == null
+                ? null
+                : Math.max(0, collected.totalResults - collected.people.length),
+        reportedCountDrift: adaptiveCollection?.reportedCountDrift ?? 0,
         fetchedPages: collected.fetchedPages,
         accessibleResultLimit,
         collectionMode: adaptiveCollection ? "adaptive" : "bounded",
@@ -15563,10 +15620,10 @@ program
         if (normalizedCheckpointUrl !== normalizedSourceQueryUrl) {
             throw new Error("The checkpoint belongs to a different Sales Navigator account search.");
         }
-        if (checkpoint.pageSize !== pageSize || checkpoint.cap > cap) {
-            throw new Error(`Checkpoint paging does not match this invocation. Use --max-results-per-search ${checkpoint.cap} --page-size ${checkpoint.pageSize}.`);
+        if (checkpoint.pageSize !== pageSize) {
+            throw new Error(`Checkpoint page size does not match this invocation. Use --page-size ${checkpoint.pageSize}.`);
         }
-        if (checkpoint.cap < cap) {
+        if (checkpoint.cap !== cap) {
             checkpoint.cap = cap;
         }
         checkpoint.delayMinMs = delayMinMs;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "salesprompter-cli",
-  "version": "0.1.58",
+  "version": "0.1.60",
   "description": "Sales workflow CLI for guided lead generation, enrichment, scoring, and sync.",
   "author": "Daniel Sinewe <hello@danielsinewe.com>",
   "type": "module",