salesprompter-cli 0.1.65 → 0.1.67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -73,6 +73,11 @@ salesprompter contacts:resolve-emails --in ./contacts.tsv --out-dir ./email-run
73
73
  salesprompter leads:collect \
74
74
  --linkedin-url "$SALES_NAV_PEOPLE_URL"
75
75
 
76
+ # For large discovery runs, store the leads now and enrich selected imports later
77
+ salesprompter leads:collect \
78
+ --linkedin-url "$SALES_NAV_PEOPLE_URL" \
79
+ --defer-email-enrichment
80
+
76
81
  # Enrich one or more stored imports: Sales Navigator company -> official domain -> Hunter email
77
82
  # This updates workspace data only. It does not create or start outreach.
78
83
  # Stale ids and raw legal-name misses use conservative exact-name recovery.
@@ -120,6 +125,19 @@ salesprompter packs:list
120
125
  salesprompter --help
121
126
  ```
122
127
 
128
+ Collect an exact signed-in LinkedIn product-search order locally, without uploading it:
129
+
130
+ ```bash
131
+ salesprompter products:collect \
132
+ --query-url "$LINKEDIN_PRODUCTS_URL" \
133
+ --checkpoint ./data/products.checkpoint.json \
134
+ --raw-jsonl ./data/products.raw.jsonl \
135
+ --out ./data/products.complete.json \
136
+ --relay-port 43117
137
+ ```
138
+
139
+ Use a signed-in in-app-browser worker with the loopback `GET /task` and `POST /page` protocol. Interrupted, bounded, challenged, or rate-limited runs remain checkpointed and do not create the complete artifact.
140
+
123
141
  Download stored workspace leads without starting a new Sales Navigator scrape:
124
142
 
125
143
  ```bash
package/dist/cli.js CHANGED
@@ -28,6 +28,7 @@ import { buildHistoricalVendorIcp, buildVendorIcp } from "./icp-templates.js";
28
28
  import { InstantlySyncProvider } from "./instantly.js";
29
29
  import { backfillLinkedInCompanies } from "./linkedin-companies.js";
30
30
  import { parseLinkedInCompanyPage } from "./linkedin-companies.js";
31
+ import { collectLinkedInProductsViaBrowserRelay } from "./linkedin-product-search.js";
31
32
  import { crawlLinkedInProductCategory } from "./linkedin-products.js";
32
33
  import { claimLinkedInSessionCookieForCli, claimValidatedSalesNavigatorSessionCookieForCli, createLinkedInSessionSupabaseClient, recordLinkedInSessionCookieAudit, resolveConfiguredEnvValue } from "./linkedin-session.js";
33
34
  import { buildLeadlistsFunnelQueries } from "./leadlists-funnel.js";
@@ -296,6 +297,23 @@ const SalesNavigatorLocalImportResponseSchema = z.object({
296
297
  })
297
298
  .nullable()
298
299
  .optional(),
300
+ emailEnrichment: z
301
+ .object({
302
+ scheduled: z.boolean(),
303
+ deferred: z.boolean().optional(),
304
+ provider: z.string().min(1).optional(),
305
+ fallbackProvider: z.string().min(1).optional(),
306
+ })
307
+ .optional(),
308
+ enrichmentDeferred: z.boolean().optional(),
309
+ outreachStarted: z.literal(false).optional(),
310
+ });
311
+ const SalesNavigatorLocalImportCapabilitiesResponseSchema = z.object({
312
+ status: z.literal("ok"),
313
+ capabilities: z.object({
314
+ deferredEmailEnrichment: z.literal(true),
315
+ }),
316
+ outreachStarted: z.literal(false),
299
317
  });
300
318
  const PhantombusterContainersSyncResponseSchema = z.object({
301
319
  status: z.literal("ok"),
@@ -581,7 +599,7 @@ const cliPacks = [
581
599
  slug: "research",
582
600
  title: "Research",
583
601
  summary: "Scrape markets and enrich companies before outreach.",
584
- commands: ["market:scrape", "companies:enrich"],
602
+ commands: ["products:collect", "market:scrape", "companies:enrich"],
585
603
  installStatus: "included"
586
604
  },
587
605
  {
@@ -631,6 +649,7 @@ const helpAliasByCommandName = new Map([
631
649
  ["linkedin-companies:backfill", "companies:enrich"],
632
650
  ["linkedin-companies:scrape-local", "companies:scrape-linkedin"],
633
651
  ["dealroom-companies:scrape-local", "companies:scrape-dealroom"],
652
+ ["linkedin-products:collect", "products:collect"],
634
653
  ["linkedin-products:scrape", "market:scrape"],
635
654
  ["salesnav:from-product-category", "leads:discover"],
636
655
  ["salesnav:people:collect", "leads:collect"],
@@ -681,6 +700,7 @@ const helpVisibleCommandNames = new Set([
681
700
  "linkedin-companies:backfill",
682
701
  "linkedin-companies:scrape-local",
683
702
  "dealroom-companies:scrape-local",
703
+ "linkedin-products:collect",
684
704
  "linkedin-products:scrape",
685
705
  "salesnav:from-product-category",
686
706
  "salesnav:people:collect",
@@ -9275,6 +9295,9 @@ async function importLocalSalesNavigatorPeopleViaApp(session, payload) {
9275
9295
  totalResults: payload.totalResults,
9276
9296
  fetchedPages: payload.fetchedPages,
9277
9297
  people: payload.people,
9298
+ ...(payload.startEmailEnrichment === false
9299
+ ? { startEmailEnrichment: false }
9300
+ : {}),
9278
9301
  rawPayload: {
9279
9302
  ...payload.rawPayload,
9280
9303
  clientId: workspaceClientId ?? undefined,
@@ -9305,6 +9328,20 @@ async function importLocalSalesNavigatorPeopleViaApp(session, payload) {
9305
9328
  }, SalesNavigatorLocalImportResponseSchema);
9306
9329
  return value;
9307
9330
  }
9331
+ async function requireDeferredEmailEnrichmentCapability(session) {
9332
+ try {
9333
+ const result = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/salesnav/local-import`, {
9334
+ method: "GET",
9335
+ headers: {
9336
+ Authorization: `Bearer ${currentSession.accessToken}`,
9337
+ },
9338
+ }), SalesNavigatorLocalImportCapabilitiesResponseSchema);
9339
+ return result.session;
9340
+ }
9341
+ catch {
9342
+ throw new Error("This Salesprompter workspace cannot safely defer email enrichment yet. Update the app before collecting leads with --defer-email-enrichment.");
9343
+ }
9344
+ }
9308
9345
  const SALESPROMPTER_SUPABASE_TABLES = {
9309
9346
  accountSearchRuns: "salesprompter_linkedin_sales_nav_account_search_runs",
9310
9347
  accountSearchSlices: "salesprompter_linkedin_sales_nav_account_search_slices",
@@ -13813,6 +13850,7 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
13813
13850
  .max(accessibleResultLimit)
13814
13851
  .parse(options.maxResults);
13815
13852
  const complete = Boolean(options.complete || options.maxResults == null);
13853
+ const emailEnrichmentDeferred = Boolean(options.deferEmailEnrichment);
13816
13854
  const checkpointPath = complete
13817
13855
  ? options.checkpoint
13818
13856
  ? path.resolve(options.checkpoint)
@@ -13827,13 +13865,18 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
13827
13865
  accessibleResultLimit,
13828
13866
  complete,
13829
13867
  resumable: complete,
13868
+ emailEnrichmentDeferred,
13869
+ outreachStarted: false,
13830
13870
  destination: "/leads/cli-imports"
13831
13871
  };
13832
13872
  if (options.out)
13833
13873
  await writeJsonFile(options.out, payload);
13834
13874
  return payload;
13835
13875
  }
13836
- const session = await requireAuthSession();
13876
+ let session = await requireAuthSession();
13877
+ if (emailEnrichmentDeferred) {
13878
+ session = await requireDeferredEmailEnrichmentCapability(session);
13879
+ }
13837
13880
  const pageDelayMinMs = z.coerce.number().int().min(0).parse(options.pageDelayMinMs);
13838
13881
  const pageDelayMaxMs = z.coerce.number().int().min(pageDelayMinMs).parse(options.pageDelayMaxMs);
13839
13882
  const browserRelayPort = options.browserRelayPort == null
@@ -13899,6 +13942,9 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
13899
13942
  slicePreset: adaptiveCollection
13900
13943
  ? "local-salesnav-people-adaptive"
13901
13944
  : "local-salesnav-people",
13945
+ ...(emailEnrichmentDeferred
13946
+ ? { startEmailEnrichment: false }
13947
+ : {}),
13902
13948
  rawPayload: adaptiveCollection
13903
13949
  ? {
13904
13950
  collectionMode: adaptiveCollection.collectionMode,
@@ -13914,6 +13960,13 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
13914
13960
  accessibleResultLimit,
13915
13961
  },
13916
13962
  });
13963
+ if (emailEnrichmentDeferred &&
13964
+ !(imported.enrichmentDeferred === true &&
13965
+ imported.emailEnrichment?.deferred === true &&
13966
+ imported.emailEnrichment.scheduled === false &&
13967
+ imported.outreachStarted === false)) {
13968
+ throw new Error(`Import ${imported.runId} was stored, but the workspace did not confirm that email enrichment was deferred. Do not repeat the collection until that run's enrichment state is verified.`);
13969
+ }
13917
13970
  const payload = {
13918
13971
  status: "ok",
13919
13972
  dryRun: false,
@@ -13942,6 +13995,10 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
13942
13995
  runId: imported.runId,
13943
13996
  imported: imported.imported,
13944
13997
  upserted: imported.upserted,
13998
+ emailEnrichmentDeferred: imported.enrichmentDeferred ??
13999
+ imported.emailEnrichment?.deferred ??
14000
+ false,
14001
+ outreachStarted: imported.outreachStarted ?? false,
13945
14002
  workspaceUrl: `${session.apiBaseUrl}/leads/cli-imports?runId=` +
13946
14003
  encodeURIComponent(imported.runId)
13947
14004
  };
@@ -14807,6 +14864,7 @@ program
14807
14864
  .option("--max-results <number>", "Bounded collection size; defaults to exhaustive collection with a 2500-person window")
14808
14865
  .option("--complete", "Split oversized searches by company headcount and deduplicate the complete result", false)
14809
14866
  .option("--checkpoint <path>", "Optional resume checkpoint; exhaustive runs use a per-query checkpoint by default")
14867
+ .option("--defer-email-enrichment", "Store leads without starting email enrichment", false)
14810
14868
  .option("--curl-file <path>", "Optional copied Sales Navigator Lead Search curl request")
14811
14869
  .option("--browser-relay-port <number>", "Use the signed-in browser through a loopback relay")
14812
14870
  .option("--page-size <number>", "Direct Sales Navigator page size", "100")
@@ -15128,6 +15186,43 @@ program
15128
15186
  profile,
15129
15187
  });
15130
15188
  });
15189
+ program
15190
+ .command("linkedin-products:collect")
15191
+ .alias("products:collect")
15192
+ .description("Collect an exact LinkedIn product-search order locally through a loopback browser relay.")
15193
+ .requiredOption("--query-url <url>", "Exact LinkedIn /search/results/products/ query URL")
15194
+ .requiredOption("--checkpoint <path>", "Private resumable checkpoint JSON path")
15195
+ .requiredOption("--raw-jsonl <path>", "Private append-only normalized page evidence path")
15196
+ .requiredOption("--out <path>", "Complete public-fields-only ranked artifact path")
15197
+ .requiredOption("--relay-port <number>", "Loopback relay port for GET /task and POST /page")
15198
+ .option("--max-pages <number>", "Optional highest page number to accept in this run")
15199
+ .option("--max-requests <number>", "Optional number of unique browser tasks to issue in this run")
15200
+ .action(async (options) => {
15201
+ const relayPort = z.coerce.number().int().min(1).max(65_535).parse(options.relayPort);
15202
+ const maxPages = options.maxPages === undefined
15203
+ ? undefined
15204
+ : z.coerce.number().int().min(1).max(100_000).parse(options.maxPages);
15205
+ const maxRequests = options.maxRequests === undefined
15206
+ ? undefined
15207
+ : z.coerce.number().int().min(1).max(100_000).parse(options.maxRequests);
15208
+ const idleTimeoutMs = process.env.SALESPROMPTER_LINKEDIN_PRODUCTS_RELAY_IDLE_TIMEOUT_MS
15209
+ ? z.coerce.number().int().min(100).max(86_400_000).parse(process.env.SALESPROMPTER_LINKEDIN_PRODUCTS_RELAY_IDLE_TIMEOUT_MS)
15210
+ : undefined;
15211
+ const result = await collectLinkedInProductsViaBrowserRelay({
15212
+ queryUrl: String(options.queryUrl),
15213
+ checkpointPath: path.resolve(String(options.checkpoint)),
15214
+ rawJsonlPath: path.resolve(String(options.rawJsonl)),
15215
+ outPath: path.resolve(String(options.out)),
15216
+ relayPort,
15217
+ maxPages,
15218
+ maxRequests,
15219
+ idleTimeoutMs,
15220
+ onListening: ({ taskUrl, pageUrl }) => {
15221
+ writeProgress(`LinkedIn product relay ready: GET ${taskUrl} and POST ${pageUrl}`);
15222
+ }
15223
+ });
15224
+ printOutput(result);
15225
+ });
15131
15226
  program
15132
15227
  .command("linkedin-products:scrape")
15133
15228
  .alias("market:scrape")
@@ -0,0 +1,1081 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ import { access, appendFile, chmod, mkdir, readFile, rename, writeFile } from "node:fs/promises";
3
+ import { createServer } from "node:http";
4
+ import path from "node:path";
5
+ import { z } from "zod";
6
+ const CHECKPOINT_SCHEMA_VERSION = 1;
7
+ const ARTIFACT_SCHEMA_VERSION = 1;
8
+ const MAX_RELAY_BODY_BYTES = 5 * 1024 * 1024;
9
+ const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
10
+ const nonEmptyText = (max) => z.string().trim().min(1).max(max);
11
+ const optionalPublicUrl = z.string().trim().url().max(4_096).optional();
12
+ export const visibleLinkedInProductCardSchema = z
13
+ .object({
14
+ identity: z.string().trim().min(1).max(256).optional(),
15
+ name: nonEmptyText(500),
16
+ linkedinUrl: nonEmptyText(4_096),
17
+ imageUrl: optionalPublicUrl,
18
+ vendor: nonEmptyText(500).optional(),
19
+ category: nonEmptyText(500).optional(),
20
+ description: nonEmptyText(10_000).optional()
21
+ })
22
+ .strict();
23
+ const pageSuccessSchema = z
24
+ .object({
25
+ pageNumber: z.number().int().min(1),
26
+ currentUrl: nonEmptyText(8_192),
27
+ reportedTotal: z.number().int().min(0).nullable().optional(),
28
+ hasNext: z.boolean(),
29
+ items: z.array(visibleLinkedInProductCardSchema).max(1_000)
30
+ })
31
+ .strict();
32
+ const terminalErrorSchema = z
33
+ .object({
34
+ pageNumber: z.number().int().min(1),
35
+ currentUrl: nonEmptyText(8_192),
36
+ error: z
37
+ .object({
38
+ code: z.enum([
39
+ "auth_required",
40
+ "challenge",
41
+ "rate_limited",
42
+ "interrupted",
43
+ "unexpected_page"
44
+ ]),
45
+ message: z.string().trim().max(1_000).optional()
46
+ })
47
+ .strict()
48
+ })
49
+ .strict();
50
+ export const linkedInProductPageSubmissionSchema = z.union([pageSuccessSchema, terminalErrorSchema]);
51
+ const rawVisibleProductCardSchema = visibleLinkedInProductCardSchema
52
+ .extend({ linkedinSlug: nonEmptyText(500) })
53
+ .strict();
54
+ const rawPageEvidenceSchema = z
55
+ .object({
56
+ type: z.literal("page"),
57
+ queryHash: z.string().regex(/^[a-f0-9]{64}$/),
58
+ pageNumber: z.number().int().min(1),
59
+ currentUrl: nonEmptyText(8_192),
60
+ reportedTotal: z.number().int().min(0).nullable(),
61
+ hasNext: z.boolean(),
62
+ items: z.array(rawVisibleProductCardSchema).max(1_000),
63
+ visibleItemCount: z.number().int().min(0),
64
+ newProductCount: z.number().int().min(0),
65
+ duplicateCount: z.number().int().min(0),
66
+ receivedAt: nonEmptyText(100)
67
+ })
68
+ .strict();
69
+ const rawTerminalEvidenceSchema = z
70
+ .object({
71
+ type: z.literal("terminal_error"),
72
+ queryHash: z.string().regex(/^[a-f0-9]{64}$/),
73
+ pageNumber: z.number().int().min(1),
74
+ currentUrl: nonEmptyText(8_192),
75
+ error: z.object({ code: terminalErrorSchema.shape.error.shape.code }).strict(),
76
+ receivedAt: nonEmptyText(100)
77
+ })
78
+ .strict();
79
+ const rawEvidenceRecordSchema = z.union([rawPageEvidenceSchema, rawTerminalEvidenceSchema]);
80
+ export class LinkedInProductCollectorInvariantError extends Error {
81
+ code;
82
+ constructor(code, message) {
83
+ super(message);
84
+ this.code = code;
85
+ this.name = "LinkedInProductCollectorInvariantError";
86
+ }
87
+ }
88
+ function sha256(value) {
89
+ return createHash("sha256").update(value, "utf8").digest("hex");
90
+ }
91
+ function cleanText(value) {
92
+ const cleaned = value?.replace(/\s+/g, " ").trim();
93
+ return cleaned || undefined;
94
+ }
95
+ function canonicalizePublicUrl(value) {
96
+ if (!value)
97
+ return undefined;
98
+ const parsed = new URL(value);
99
+ if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
100
+ throw new LinkedInProductCollectorInvariantError("invalid_public_url", `Unsupported public URL: ${value}`);
101
+ }
102
+ parsed.username = "";
103
+ parsed.password = "";
104
+ parsed.hash = "";
105
+ return parsed.toString();
106
+ }
107
+ export function normalizeLinkedInProductSearchUrl(input) {
108
+ let parsed;
109
+ try {
110
+ parsed = new URL(input);
111
+ }
112
+ catch {
113
+ throw new LinkedInProductCollectorInvariantError("invalid_query_url", "The query URL is not a valid URL.");
114
+ }
115
+ if (parsed.protocol !== "https:" ||
116
+ parsed.port !== "" ||
117
+ parsed.username !== "" ||
118
+ parsed.password !== "" ||
119
+ !["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
120
+ throw new LinkedInProductCollectorInvariantError("invalid_query_url", "The query URL must use https://www.linkedin.com.");
121
+ }
122
+ if (parsed.pathname !== "/search/results/products" && parsed.pathname !== "/search/results/products/") {
123
+ throw new LinkedInProductCollectorInvariantError("invalid_query_url", "The query URL must point to /search/results/products/.");
124
+ }
125
+ const suppliedPages = parsed.searchParams.getAll("page");
126
+ const suppliedPage = suppliedPages[0];
127
+ if (suppliedPages.length > 1 || (suppliedPage && suppliedPage !== "1")) {
128
+ throw new LinkedInProductCollectorInvariantError("invalid_query_url", "An exhaustive collection must begin on page 1; remove the page parameter.");
129
+ }
130
+ parsed.protocol = "https:";
131
+ parsed.hostname = "www.linkedin.com";
132
+ parsed.port = "";
133
+ parsed.username = "";
134
+ parsed.password = "";
135
+ parsed.pathname = "/search/results/products/";
136
+ parsed.hash = "";
137
+ parsed.searchParams.delete("page");
138
+ return parsed.toString();
139
+ }
140
+ export function buildLinkedInProductSearchPageUrl(queryUrl, pageNumber) {
141
+ if (!Number.isInteger(pageNumber) || pageNumber < 1) {
142
+ throw new LinkedInProductCollectorInvariantError("invalid_page", "Page numbers must be positive integers.");
143
+ }
144
+ const parsed = new URL(normalizeLinkedInProductSearchUrl(queryUrl));
145
+ if (pageNumber > 1)
146
+ parsed.searchParams.set("page", String(pageNumber));
147
+ return parsed.toString();
148
+ }
149
+ function assertExpectedLinkedInSearchPage(currentUrl, queryUrl, pageNumber) {
150
+ let actual;
151
+ try {
152
+ actual = new URL(currentUrl);
153
+ }
154
+ catch {
155
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported an invalid page URL.");
156
+ }
157
+ const source = new URL(normalizeLinkedInProductSearchUrl(queryUrl));
158
+ if (actual.protocol !== "https:" ||
159
+ actual.port !== "" ||
160
+ actual.username !== "" ||
161
+ actual.password !== "" ||
162
+ !["linkedin.com", "www.linkedin.com"].includes(actual.hostname.toLowerCase()) ||
163
+ (actual.pathname !== "/search/results/products" && actual.pathname !== "/search/results/products/")) {
164
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser is not on the expected LinkedIn product-search page.");
165
+ }
166
+ for (const key of new Set(source.searchParams.keys())) {
167
+ const expectedValues = source.searchParams.getAll(key);
168
+ const actualValues = actual.searchParams.getAll(key);
169
+ if (expectedValues.length !== actualValues.length || expectedValues.some((value, index) => value !== actualValues[index])) {
170
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", `The browser page changed the original ${key} query filter.`);
171
+ }
172
+ }
173
+ const sourceKeys = new Set(source.searchParams.keys());
174
+ for (const key of new Set(actual.searchParams.keys())) {
175
+ if (!sourceKeys.has(key) && key !== "page" && key !== "spellCorrectionEnabled") {
176
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", `The browser page added the unapproved ${key} query parameter.`);
177
+ }
178
+ }
179
+ const spellCorrectionValues = actual.searchParams.getAll("spellCorrectionEnabled");
180
+ if (spellCorrectionValues.length > 1 ||
181
+ spellCorrectionValues.some((value) => value !== "true" && value !== "false")) {
182
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported an invalid spellCorrectionEnabled parameter.");
183
+ }
184
+ const actualPages = actual.searchParams.getAll("page");
185
+ const actualPage = actualPages[0] ?? null;
186
+ if (actualPages.length > 1) {
187
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported multiple page numbers.");
188
+ }
189
+ if (pageNumber === 1) {
190
+ if (actualPage && actualPage !== "1") {
191
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported the wrong page number.");
192
+ }
193
+ }
194
+ else if (actualPage !== String(pageNumber)) {
195
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported the wrong page number.");
196
+ }
197
+ return buildLinkedInProductSearchPageUrl(queryUrl, pageNumber);
198
+ }
199
+ export function canonicalizeLinkedInProductUrl(value) {
200
+ let parsed;
201
+ try {
202
+ parsed = new URL(value, "https://www.linkedin.com");
203
+ }
204
+ catch {
205
+ throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid LinkedIn product URL: ${value}`);
206
+ }
207
+ if (parsed.protocol !== "https:" ||
208
+ parsed.port !== "" ||
209
+ parsed.username !== "" ||
210
+ parsed.password !== "" ||
211
+ !["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
212
+ throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Product URL is outside LinkedIn: ${value}`);
213
+ }
214
+ const match = parsed.pathname.match(/^\/products\/([^/]+)\/?$/i);
215
+ if (!match?.[1]) {
216
+ throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Product URL must match /products/<slug>/: ${value}`);
217
+ }
218
+ let decodedSlug;
219
+ try {
220
+ decodedSlug = decodeURIComponent(match[1]).trim().normalize("NFC").toLowerCase();
221
+ }
222
+ catch {
223
+ throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid encoded product slug: ${value}`);
224
+ }
225
+ if (!decodedSlug || /[\u0000-\u001f\u007f/\\]/.test(decodedSlug)) {
226
+ throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid LinkedIn product slug: ${match[1]}`);
227
+ }
228
+ const slug = encodeURIComponent(decodedSlug)
229
+ .replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).padStart(2, "0")}`)
230
+ .toLowerCase();
231
+ if (slug.length > 720 || !/^(?:[a-z0-9._~-]|%[0-9a-f]{2})+$/.test(slug)) {
232
+ throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid LinkedIn product slug: ${slug}`);
233
+ }
234
+ return {
235
+ linkedinUrl: `https://www.linkedin.com/products/${slug}`,
236
+ linkedinSlug: slug
237
+ };
238
+ }
239
+ function normalizeIdentity(value) {
240
+ const identity = cleanText(value);
241
+ if (!identity)
242
+ return undefined;
243
+ if (!/^[A-Za-z0-9:._-]+$/.test(identity)) {
244
+ throw new LinkedInProductCollectorInvariantError("invalid_identity", "Product identity contains unsupported characters.");
245
+ }
246
+ return identity;
247
+ }
248
+ function normalizeVisibleProductCard(card, rank) {
249
+ const { linkedinUrl, linkedinSlug } = canonicalizeLinkedInProductUrl(card.linkedinUrl);
250
+ const name = cleanText(card.name);
251
+ if (!name) {
252
+ throw new LinkedInProductCollectorInvariantError("invalid_product", "A visible product card is missing its name.");
253
+ }
254
+ const imageUrl = canonicalizePublicUrl(card.imageUrl);
255
+ const vendor = cleanText(card.vendor);
256
+ const category = cleanText(card.category);
257
+ const description = cleanText(card.description);
258
+ const identity = normalizeIdentity(card.identity);
259
+ return {
260
+ rank,
261
+ name,
262
+ linkedinUrl,
263
+ linkedinSlug,
264
+ ...(imageUrl ? { imageUrl } : {}),
265
+ ...(vendor ? { vendor } : {}),
266
+ ...(category ? { category } : {}),
267
+ ...(description ? { description } : {}),
268
+ ...(identity ? { identity } : {})
269
+ };
270
+ }
271
+ export function createLinkedInProductCollectorCheckpoint(queryUrl, now = new Date()) {
272
+ const normalizedQueryUrl = normalizeLinkedInProductSearchUrl(queryUrl);
273
+ const timestamp = now.toISOString();
274
+ return {
275
+ schemaVersion: CHECKPOINT_SCHEMA_VERSION,
276
+ queryUrl: normalizedQueryUrl,
277
+ queryHash: sha256(normalizedQueryUrl),
278
+ startedAt: timestamp,
279
+ updatedAt: timestamp,
280
+ status: "collecting",
281
+ nextPageNumber: 1,
282
+ pagesAccepted: 0,
283
+ tasksIssued: 0,
284
+ approximateReportedTotal: null,
285
+ products: []
286
+ };
287
+ }
288
+ export function applyLinkedInProductSearchPage(checkpoint, submission, now = new Date()) {
289
+ if (checkpoint.status !== "collecting") {
290
+ throw new LinkedInProductCollectorInvariantError("checkpoint_closed", "The checkpoint is not collecting.");
291
+ }
292
+ if (submission.pageNumber !== checkpoint.nextPageNumber) {
293
+ throw new LinkedInProductCollectorInvariantError("out_of_order_page", `Expected page ${checkpoint.nextPageNumber}, received page ${submission.pageNumber}.`);
294
+ }
295
+ const currentUrl = assertExpectedLinkedInSearchPage(submission.currentUrl, checkpoint.queryUrl, submission.pageNumber);
296
+ if (submission.hasNext && submission.items.length === 0) {
297
+ throw new LinkedInProductCollectorInvariantError("empty_page_with_next", "A page that claims to have a next page must contain at least one visible product card.");
298
+ }
299
+ const products = checkpoint.products.map((product) => ({ ...product }));
300
+ const byUrl = new Map(products.map((product) => [product.linkedinUrl, product]));
301
+ const byIdentity = new Map();
302
+ for (const product of products) {
303
+ if (product.identity)
304
+ byIdentity.set(product.identity, product);
305
+ }
306
+ let duplicateCount = 0;
307
+ let newProductCount = 0;
308
+ for (const input of submission.items) {
309
+ const candidate = normalizeVisibleProductCard(input, products.length + 1);
310
+ const existingByUrl = byUrl.get(candidate.linkedinUrl);
311
+ const existingByIdentity = candidate.identity ? byIdentity.get(candidate.identity) : undefined;
312
+ if (existingByIdentity && existingByIdentity.linkedinUrl !== candidate.linkedinUrl) {
313
+ throw new LinkedInProductCollectorInvariantError("identity_drift", `Product identity ${candidate.identity} moved to a different canonical URL.`);
314
+ }
315
+ if (existingByUrl) {
316
+ if (candidate.identity && existingByUrl.identity && candidate.identity !== existingByUrl.identity) {
317
+ throw new LinkedInProductCollectorInvariantError("identity_drift", `Canonical URL ${candidate.linkedinUrl} changed product identity.`);
318
+ }
319
+ if (existingByUrl.name !== candidate.name) {
320
+ throw new LinkedInProductCollectorInvariantError("identity_drift", `Canonical URL ${candidate.linkedinUrl} changed product name.`);
321
+ }
322
+ if (!existingByUrl.identity && candidate.identity) {
323
+ existingByUrl.identity = candidate.identity;
324
+ byIdentity.set(candidate.identity, existingByUrl);
325
+ }
326
+ duplicateCount += 1;
327
+ continue;
328
+ }
329
+ products.push(candidate);
330
+ byUrl.set(candidate.linkedinUrl, candidate);
331
+ if (candidate.identity)
332
+ byIdentity.set(candidate.identity, candidate);
333
+ newProductCount += 1;
334
+ }
335
+ products.forEach((product, index) => {
336
+ if (product.rank !== index + 1) {
337
+ throw new LinkedInProductCollectorInvariantError("rank_drift", "Checkpoint ranks are not contiguous.");
338
+ }
339
+ });
340
+ const capturedAt = now.toISOString();
341
+ const reportedTotal = submission.reportedTotal ?? checkpoint.approximateReportedTotal;
342
+ const terminalPage = submission.hasNext
343
+ ? undefined
344
+ : {
345
+ pageNumber: submission.pageNumber,
346
+ currentUrl,
347
+ reportedTotal: submission.reportedTotal ?? null,
348
+ hasNext: false,
349
+ visibleItemCount: submission.items.length,
350
+ newProductCount,
351
+ capturedAt
352
+ };
353
+ const nextCheckpoint = {
354
+ ...checkpoint,
355
+ updatedAt: capturedAt,
356
+ status: submission.hasNext ? "collecting" : "complete",
357
+ nextPageNumber: submission.hasNext ? submission.pageNumber + 1 : submission.pageNumber,
358
+ pagesAccepted: checkpoint.pagesAccepted + 1,
359
+ tasksIssued: Math.max(checkpoint.tasksIssued, checkpoint.pagesAccepted + 1),
360
+ approximateReportedTotal: reportedTotal ?? null,
361
+ products,
362
+ ...(terminalPage ? { terminalPage, completedAt: capturedAt } : {}),
363
+ lastStop: undefined
364
+ };
365
+ if (terminalPage)
366
+ nextCheckpoint.orderChecksum = computeLinkedInProductOrderChecksum(products);
367
+ return { checkpoint: nextCheckpoint, newProductCount, duplicateCount };
368
+ }
369
+ export function computeLinkedInProductOrderChecksum(products) {
370
+ const manifest = products.map((product) => `${product.rank}\t${product.linkedinUrl}`).join("\n");
371
+ return sha256(manifest);
372
+ }
373
+ export function buildLinkedInProductCollectionArtifact(checkpoint) {
374
+ assertValidCheckpoint(checkpoint);
375
+ if (checkpoint.status !== "complete" || !checkpoint.terminalPage || !checkpoint.completedAt) {
376
+ throw new LinkedInProductCollectorInvariantError("incomplete_checkpoint", "A complete artifact requires natural terminal-page evidence.");
377
+ }
378
+ const products = checkpoint.products.map((stored) => {
379
+ const canonical = canonicalizeLinkedInProductUrl(stored.linkedinUrl);
380
+ if (canonical.linkedinSlug !== stored.linkedinSlug) {
381
+ throw new LinkedInProductCollectorInvariantError("canonical_url_drift", "Checkpoint product URL and slug disagree.");
382
+ }
383
+ const name = cleanText(stored.name);
384
+ if (!name)
385
+ throw new LinkedInProductCollectorInvariantError("invalid_product", "Checkpoint product name is empty.");
386
+ const imageUrl = canonicalizePublicUrl(stored.imageUrl);
387
+ const vendor = cleanText(stored.vendor);
388
+ const category = cleanText(stored.category);
389
+ const description = cleanText(stored.description);
390
+ return {
391
+ rank: stored.rank,
392
+ name,
393
+ linkedinUrl: canonical.linkedinUrl,
394
+ linkedinSlug: canonical.linkedinSlug,
395
+ ...(imageUrl ? { imageUrl } : {}),
396
+ ...(vendor ? { vendor } : {}),
397
+ ...(category ? { category } : {}),
398
+ ...(description ? { description } : {})
399
+ };
400
+ });
401
+ products.forEach((product, index) => {
402
+ if (product.rank !== index + 1) {
403
+ throw new LinkedInProductCollectorInvariantError("rank_drift", "Artifact ranks are not contiguous.");
404
+ }
405
+ });
406
+ const orderChecksum = computeLinkedInProductOrderChecksum(products);
407
+ if (checkpoint.orderChecksum && checkpoint.orderChecksum !== orderChecksum) {
408
+ throw new LinkedInProductCollectorInvariantError("checksum_drift", "Checkpoint order checksum changed.");
409
+ }
410
+ const terminalPage = {
411
+ pageNumber: checkpoint.terminalPage.pageNumber,
412
+ currentUrl: checkpoint.terminalPage.currentUrl,
413
+ reportedTotal: checkpoint.terminalPage.reportedTotal,
414
+ hasNext: false,
415
+ visibleItemCount: checkpoint.terminalPage.visibleItemCount,
416
+ newProductCount: checkpoint.terminalPage.newProductCount,
417
+ capturedAt: checkpoint.terminalPage.capturedAt
418
+ };
419
+ return {
420
+ schemaVersion: ARTIFACT_SCHEMA_VERSION,
421
+ kind: "linkedin_product_search_snapshot",
422
+ complete: true,
423
+ source: {
424
+ queryUrl: checkpoint.queryUrl,
425
+ queryHash: checkpoint.queryHash
426
+ },
427
+ capture: {
428
+ startedAt: checkpoint.startedAt,
429
+ completedAt: checkpoint.completedAt,
430
+ approximateReportedTotal: checkpoint.approximateReportedTotal,
431
+ pagesAccepted: checkpoint.pagesAccepted,
432
+ terminalPage
433
+ },
434
+ productCount: products.length,
435
+ orderChecksumAlgorithm: "sha256",
436
+ orderChecksum,
437
+ products
438
+ };
439
+ }
440
+ function rawVisibleProductCard(card) {
441
+ const { rank: _rank, ...normalized } = normalizeVisibleProductCard(card, 1);
442
+ return normalized;
443
+ }
444
+ function sanitizeTerminalErrorUrl(currentUrl) {
445
+ let parsed;
446
+ try {
447
+ parsed = new URL(currentUrl);
448
+ }
449
+ catch {
450
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported an invalid terminal URL.");
451
+ }
452
+ if (parsed.protocol !== "https:" ||
453
+ parsed.port !== "" ||
454
+ parsed.username !== "" ||
455
+ parsed.password !== "" ||
456
+ !["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
457
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", "A terminal browser error must come from a LinkedIn page.");
458
+ }
459
+ return `https://www.linkedin.com${parsed.pathname}`;
460
+ }
461
+ function terminalStopMessage(code) {
462
+ switch (code) {
463
+ case "auth_required":
464
+ return "The signed-in LinkedIn session is no longer available.";
465
+ case "challenge":
466
+ return "LinkedIn displayed a checkpoint or challenge.";
467
+ case "rate_limited":
468
+ return "LinkedIn rate-limited the product search.";
469
+ case "interrupted":
470
+ return "The browser worker was interrupted.";
471
+ case "unexpected_page":
472
+ return "The browser reached an unexpected LinkedIn page.";
473
+ }
474
+ }
475
+ async function ensureParent(filePath) {
476
+ await mkdir(path.dirname(path.resolve(filePath)), { recursive: true });
477
+ }
478
+ async function writePrivateJsonAtomic(filePath, value) {
479
+ const resolved = path.resolve(filePath);
480
+ await ensureParent(resolved);
481
+ const temporary = `${resolved}.${process.pid}.${randomUUID()}.tmp`;
482
+ await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
483
+ await chmod(temporary, 0o600);
484
+ await rename(temporary, resolved);
485
+ await chmod(resolved, 0o600);
486
+ }
487
+ async function appendPrivateJsonLine(filePath, value) {
488
+ const resolved = path.resolve(filePath);
489
+ await ensureParent(resolved);
490
+ await appendFile(resolved, `${JSON.stringify(value)}\n`, { encoding: "utf8", mode: 0o600 });
491
+ await chmod(resolved, 0o600);
492
+ }
493
+ async function readCheckpoint(filePath) {
494
+ try {
495
+ const value = JSON.parse(await readFile(path.resolve(filePath), "utf8"));
496
+ if (value.schemaVersion !== CHECKPOINT_SCHEMA_VERSION || !Array.isArray(value.products)) {
497
+ throw new Error("Unsupported LinkedIn product collector checkpoint.");
498
+ }
499
+ assertValidCheckpoint(value);
500
+ return value;
501
+ }
502
+ catch (error) {
503
+ if (error.code === "ENOENT")
504
+ return null;
505
+ throw error;
506
+ }
507
+ }
508
+ function assertValidCheckpoint(checkpoint) {
509
+ if (normalizeLinkedInProductSearchUrl(checkpoint.queryUrl) !== checkpoint.queryUrl) {
510
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint query URL is invalid.");
511
+ }
512
+ if (!/^[a-f0-9]{64}$/.test(checkpoint.queryHash)) {
513
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint query hash is invalid.");
514
+ }
515
+ if (!["collecting", "complete", "failed"].includes(checkpoint.status)) {
516
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint status is invalid.");
517
+ }
518
+ if (!Number.isInteger(checkpoint.nextPageNumber) ||
519
+ checkpoint.nextPageNumber < 1 ||
520
+ !Number.isInteger(checkpoint.pagesAccepted) ||
521
+ checkpoint.pagesAccepted < 0 ||
522
+ !Number.isInteger(checkpoint.tasksIssued) ||
523
+ checkpoint.tasksIssued < 0) {
524
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint counters are invalid.");
525
+ }
526
+ if (checkpoint.tasksIssued < checkpoint.pagesAccepted) {
527
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint has more accepted pages than issued browser tasks.");
528
+ }
529
+ assertIsoTimestamp(checkpoint.startedAt, "startedAt");
530
+ assertIsoTimestamp(checkpoint.updatedAt, "updatedAt");
531
+ if (checkpoint.updatedAt < checkpoint.startedAt) {
532
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint timestamps are out of order.");
533
+ }
534
+ if (checkpoint.approximateReportedTotal !== null &&
535
+ (!Number.isInteger(checkpoint.approximateReportedTotal) || checkpoint.approximateReportedTotal < 0)) {
536
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint approximate reported total is invalid.");
537
+ }
538
+ const seenUrls = new Set();
539
+ const seenIdentities = new Set();
540
+ checkpoint.products.forEach((product, index) => {
541
+ if (product.rank !== index + 1 || !Number.isInteger(product.rank)) {
542
+ throw new LinkedInProductCollectorInvariantError("rank_drift", "Checkpoint ranks are not contiguous.");
543
+ }
544
+ const canonical = canonicalizeLinkedInProductUrl(product.linkedinUrl);
545
+ if (canonical.linkedinUrl !== product.linkedinUrl || canonical.linkedinSlug !== product.linkedinSlug) {
546
+ throw new LinkedInProductCollectorInvariantError("canonical_url_drift", "Checkpoint contains a non-canonical product URL.");
547
+ }
548
+ if (seenUrls.has(product.linkedinUrl)) {
549
+ throw new LinkedInProductCollectorInvariantError("duplicate_product", "Checkpoint contains a duplicate product URL.");
550
+ }
551
+ seenUrls.add(product.linkedinUrl);
552
+ if (product.identity) {
553
+ const identity = normalizeIdentity(product.identity);
554
+ if (!identity || seenIdentities.has(identity)) {
555
+ throw new LinkedInProductCollectorInvariantError("identity_drift", "Checkpoint contains a duplicate or invalid product identity.");
556
+ }
557
+ seenIdentities.add(identity);
558
+ }
559
+ });
560
+ if (checkpoint.status === "complete") {
561
+ if (checkpoint.pagesAccepted < 1 ||
562
+ checkpoint.nextPageNumber !== checkpoint.pagesAccepted ||
563
+ !checkpoint.terminalPage ||
564
+ !checkpoint.completedAt ||
565
+ checkpoint.lastStop !== undefined ||
566
+ !/^[a-f0-9]{64}$/.test(checkpoint.orderChecksum ?? "")) {
567
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "A complete checkpoint is missing terminal evidence or its checksum.");
568
+ }
569
+ const terminal = checkpoint.terminalPage;
570
+ if (terminal.hasNext !== false ||
571
+ !Number.isInteger(terminal.pageNumber) ||
572
+ terminal.pageNumber !== checkpoint.pagesAccepted ||
573
+ terminal.currentUrl !== buildLinkedInProductSearchPageUrl(checkpoint.queryUrl, terminal.pageNumber) ||
574
+ !Number.isInteger(terminal.visibleItemCount) ||
575
+ terminal.visibleItemCount < 0 ||
576
+ !Number.isInteger(terminal.newProductCount) ||
577
+ terminal.newProductCount < 0 ||
578
+ terminal.newProductCount > terminal.visibleItemCount ||
579
+ (terminal.reportedTotal !== null &&
580
+ (!Number.isInteger(terminal.reportedTotal) || terminal.reportedTotal < 0))) {
581
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Complete checkpoint terminal-page evidence is invalid.");
582
+ }
583
+ assertIsoTimestamp(terminal.capturedAt, "terminalPage.capturedAt");
584
+ assertIsoTimestamp(checkpoint.completedAt, "completedAt");
585
+ if (terminal.capturedAt !== checkpoint.completedAt || checkpoint.completedAt !== checkpoint.updatedAt) {
586
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Complete checkpoint capture timestamps disagree.");
587
+ }
588
+ if (computeLinkedInProductOrderChecksum(checkpoint.products) !== checkpoint.orderChecksum) {
589
+ throw new LinkedInProductCollectorInvariantError("checksum_drift", "Checkpoint order checksum changed.");
590
+ }
591
+ }
592
+ else {
593
+ if (checkpoint.nextPageNumber !== checkpoint.pagesAccepted + 1 ||
594
+ checkpoint.terminalPage !== undefined ||
595
+ checkpoint.completedAt !== undefined ||
596
+ checkpoint.orderChecksum !== undefined) {
597
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Incomplete checkpoint state is internally inconsistent.");
598
+ }
599
+ if (checkpoint.lastStop) {
600
+ if (typeof checkpoint.lastStop.reason !== "string" ||
601
+ checkpoint.lastStop.reason.trim() === "" ||
602
+ !Number.isInteger(checkpoint.lastStop.pageNumber) ||
603
+ checkpoint.lastStop.pageNumber !== checkpoint.nextPageNumber) {
604
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint stop evidence is invalid.");
605
+ }
606
+ assertIsoTimestamp(checkpoint.lastStop.at, "lastStop.at");
607
+ }
608
+ }
609
+ }
610
+ function assertIsoTimestamp(value, field) {
611
+ if (typeof value !== "string") {
612
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", `Checkpoint ${field} is not a timestamp.`);
613
+ }
614
+ const parsed = new Date(value);
615
+ if (Number.isNaN(parsed.valueOf()) || parsed.toISOString() !== value) {
616
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", `Checkpoint ${field} is not canonical ISO time.`);
617
+ }
618
+ }
619
+ async function fileExists(filePath) {
620
+ try {
621
+ await access(path.resolve(filePath));
622
+ return true;
623
+ }
624
+ catch {
625
+ return false;
626
+ }
627
+ }
628
+ async function assertRawEvidenceMatchesCheckpoint(filePath, checkpoint) {
629
+ const raw = await readFile(path.resolve(filePath), "utf8");
630
+ const lines = raw.split(/\r?\n/).filter((line) => line.trim() !== "");
631
+ let replay = createLinkedInProductCollectorCheckpoint(checkpoint.queryUrl, new Date(checkpoint.startedAt));
632
+ let acceptedPageRecords = 0;
633
+ for (const [index, line] of lines.entries()) {
634
+ let parsedJson;
635
+ try {
636
+ parsedJson = JSON.parse(line);
637
+ }
638
+ catch {
639
+ throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw JSONL line ${index + 1} is not valid JSON.`);
640
+ }
641
+ const record = rawEvidenceRecordSchema.parse(parsedJson);
642
+ if (record.queryHash !== checkpoint.queryHash) {
643
+ throw new LinkedInProductCollectorInvariantError("raw_query_mismatch", `Raw JSONL line ${index + 1} belongs to another query.`);
644
+ }
645
+ assertIsoTimestamp(record.receivedAt, `raw line ${index + 1} receivedAt`);
646
+ if (record.type === "terminal_error") {
647
+ if (replay.status !== "collecting" || record.pageNumber !== replay.nextPageNumber) {
648
+ throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw terminal error on line ${index + 1} is out of sequence.`);
649
+ }
650
+ continue;
651
+ }
652
+ if (record.pageNumber !== replay.nextPageNumber) {
653
+ throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw page on line ${index + 1} is duplicate or out of sequence.`);
654
+ }
655
+ const items = record.items.map(({ linkedinSlug, ...item }) => {
656
+ const canonical = canonicalizeLinkedInProductUrl(item.linkedinUrl);
657
+ if (canonical.linkedinSlug !== linkedinSlug) {
658
+ throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw product slug on line ${index + 1} disagrees with its URL.`);
659
+ }
660
+ return item;
661
+ });
662
+ const applied = applyLinkedInProductSearchPage(replay, {
663
+ pageNumber: record.pageNumber,
664
+ currentUrl: record.currentUrl,
665
+ reportedTotal: record.reportedTotal,
666
+ hasNext: record.hasNext,
667
+ items
668
+ }, new Date(record.receivedAt));
669
+ if (record.visibleItemCount !== record.items.length ||
670
+ record.newProductCount !== applied.newProductCount ||
671
+ record.duplicateCount !== applied.duplicateCount) {
672
+ throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw page counts on line ${index + 1} do not match its normalized items.`);
673
+ }
674
+ replay = applied.checkpoint;
675
+ acceptedPageRecords += 1;
676
+ }
677
+ if (acceptedPageRecords !== checkpoint.pagesAccepted) {
678
+ throw new LinkedInProductCollectorInvariantError("missing_raw_evidence", `Raw JSONL proves ${acceptedPageRecords} pages, but the checkpoint claims ${checkpoint.pagesAccepted}.`);
679
+ }
680
+ if (JSON.stringify(replay.products) !== JSON.stringify(checkpoint.products)) {
681
+ throw new LinkedInProductCollectorInvariantError("raw_order_mismatch", "Raw JSONL product order does not reproduce the checkpoint.");
682
+ }
683
+ if (replay.approximateReportedTotal !== checkpoint.approximateReportedTotal ||
684
+ (checkpoint.status === "complete") !== (replay.status === "complete")) {
685
+ throw new LinkedInProductCollectorInvariantError("raw_state_mismatch", "Raw JSONL completion metadata does not reproduce the checkpoint.");
686
+ }
687
+ }
688
+ async function readRelayJson(req) {
689
+ const chunks = [];
690
+ let size = 0;
691
+ for await (const chunk of req) {
692
+ const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
693
+ size += buffer.length;
694
+ if (size > MAX_RELAY_BODY_BYTES) {
695
+ throw new LinkedInProductCollectorInvariantError("body_too_large", "Relay request body is too large.");
696
+ }
697
+ chunks.push(buffer);
698
+ }
699
+ if (chunks.length === 0)
700
+ throw new Error("Relay request body is empty.");
701
+ return JSON.parse(Buffer.concat(chunks).toString("utf8"));
702
+ }
703
+ function writeRelayJson(res, statusCode, value, origin) {
704
+ const body = `${JSON.stringify(value)}\n`;
705
+ res.writeHead(statusCode, {
706
+ "Content-Type": "application/json; charset=utf-8",
707
+ "Content-Length": Buffer.byteLength(body),
708
+ "Cache-Control": "no-store",
709
+ "Access-Control-Allow-Origin": origin === "https://www.linkedin.com" ? origin : "https://www.linkedin.com",
710
+ "Access-Control-Allow-Private-Network": "true",
711
+ Vary: "Origin"
712
+ });
713
+ res.end(body);
714
+ }
715
+ function isAllowedRelayOrigin(req) {
716
+ const origin = req.headers.origin;
717
+ return origin === undefined || origin === "https://www.linkedin.com";
718
+ }
719
+ function closeServer(server) {
720
+ return new Promise((resolve) => server.close(() => resolve()));
721
+ }
722
+ export async function collectLinkedInProductsViaBrowserRelay(options) {
723
+ const queryUrl = normalizeLinkedInProductSearchUrl(options.queryUrl);
724
+ const expectedQueryHash = sha256(queryUrl);
725
+ const resolvedPaths = [options.checkpointPath, options.rawJsonlPath, options.outPath].map((value) => path.resolve(value));
726
+ if (new Set(resolvedPaths).size !== resolvedPaths.length) {
727
+ throw new LinkedInProductCollectorInvariantError("artifact_path_conflict", "Checkpoint, raw JSONL, and complete artifact paths must be distinct.");
728
+ }
729
+ const existingCheckpoint = await readCheckpoint(options.checkpointPath);
730
+ const [rawExists, outExists] = await Promise.all([
731
+ fileExists(options.rawJsonlPath),
732
+ fileExists(options.outPath)
733
+ ]);
734
+ if (!existingCheckpoint && (rawExists || outExists)) {
735
+ throw new LinkedInProductCollectorInvariantError("orphaned_artifact", "A new collection requires unused raw JSONL and output paths. Choose new paths or restore the matching checkpoint.");
736
+ }
737
+ if (existingCheckpoint && existingCheckpoint.pagesAccepted > 0 && !rawExists) {
738
+ throw new LinkedInProductCollectorInvariantError("missing_raw_evidence", "The checkpoint has accepted pages but its raw JSONL evidence is missing.");
739
+ }
740
+ if (existingCheckpoint && rawExists) {
741
+ await assertRawEvidenceMatchesCheckpoint(options.rawJsonlPath, existingCheckpoint);
742
+ }
743
+ if (existingCheckpoint && existingCheckpoint.status !== "complete" && outExists) {
744
+ throw new LinkedInProductCollectorInvariantError("stale_complete_artifact", "An incomplete checkpoint cannot share a path with an existing complete artifact. Choose a new output path.");
745
+ }
746
+ let checkpoint = existingCheckpoint ?? createLinkedInProductCollectorCheckpoint(queryUrl);
747
+ if (checkpoint.queryUrl !== queryUrl || checkpoint.queryHash !== expectedQueryHash) {
748
+ throw new LinkedInProductCollectorInvariantError("checkpoint_query_mismatch", "The checkpoint belongs to a different LinkedIn product search URL.");
749
+ }
750
+ await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
751
+ if (checkpoint.status === "complete") {
752
+ const artifact = buildLinkedInProductCollectionArtifact(checkpoint);
753
+ await writePrivateJsonAtomic(options.outPath, artifact);
754
+ return {
755
+ status: "complete",
756
+ complete: true,
757
+ queryUrl,
758
+ queryHash: expectedQueryHash,
759
+ checkpointPath: path.resolve(options.checkpointPath),
760
+ rawJsonlPath: path.resolve(options.rawJsonlPath),
761
+ outPath: path.resolve(options.outPath),
762
+ pagesAccepted: checkpoint.pagesAccepted,
763
+ productsCollected: checkpoint.products.length,
764
+ nextPageNumber: checkpoint.nextPageNumber,
765
+ orderChecksum: checkpoint.orderChecksum,
766
+ terminalPage: checkpoint.terminalPage
767
+ };
768
+ }
769
+ if (checkpoint.status === "failed") {
770
+ throw new LinkedInProductCollectorInvariantError("failed_checkpoint", "The checkpoint is failed. Inspect it and choose a new checkpoint path before retrying.");
771
+ }
772
+ let uniqueTasksIssuedThisRun = 0;
773
+ let lastIssuedPage = null;
774
+ let settled = false;
775
+ let acceptingRequests = true;
776
+ let idleTimer;
777
+ let incompleteStopPromise;
778
+ let resolveResult;
779
+ let rejectResult;
780
+ const resultPromise = new Promise((resolve, reject) => {
781
+ resolveResult = resolve;
782
+ rejectResult = reject;
783
+ });
784
+ let requestQueue = Promise.resolve();
785
+ const enqueueExclusive = (operation) => {
786
+ const queued = requestQueue.then(operation, operation);
787
+ requestQueue = queued.catch(() => undefined);
788
+ return queued;
789
+ };
790
+ const server = createServer((req, res) => {
791
+ if (!acceptingRequests) {
792
+ writeRelayJson(res, 410, { status: "closed" }, req.headers.origin);
793
+ return;
794
+ }
795
+ const execute = async () => {
796
+ try {
797
+ await handleRequest(req, res);
798
+ }
799
+ catch (error) {
800
+ const browserInputError = error instanceof LinkedInProductCollectorInvariantError || error instanceof z.ZodError || error instanceof SyntaxError;
801
+ if (!res.headersSent) {
802
+ writeRelayJson(res, browserInputError ? 409 : 500, {
803
+ status: "rejected",
804
+ error: error instanceof Error ? error.message : String(error)
805
+ }, req.headers.origin);
806
+ }
807
+ if (!browserInputError)
808
+ await abortWithError(error);
809
+ }
810
+ };
811
+ void enqueueExclusive(execute);
812
+ });
813
+ const makeResult = (status, reason) => ({
814
+ status,
815
+ complete: status === "complete",
816
+ ...(reason ? { reason } : {}),
817
+ queryUrl,
818
+ queryHash: expectedQueryHash,
819
+ checkpointPath: path.resolve(options.checkpointPath),
820
+ rawJsonlPath: path.resolve(options.rawJsonlPath),
821
+ outPath: status === "complete" ? path.resolve(options.outPath) : null,
822
+ pagesAccepted: checkpoint.pagesAccepted,
823
+ productsCollected: checkpoint.products.length,
824
+ nextPageNumber: checkpoint.nextPageNumber,
825
+ ...(checkpoint.orderChecksum ? { orderChecksum: checkpoint.orderChecksum } : {}),
826
+ ...(checkpoint.terminalPage ? { terminalPage: checkpoint.terminalPage } : {})
827
+ });
828
+ const finish = async (status, reason) => {
829
+ if (settled)
830
+ return;
831
+ settled = true;
832
+ acceptingRequests = false;
833
+ if (idleTimer)
834
+ clearTimeout(idleTimer);
835
+ process.off("SIGINT", handleSigint);
836
+ process.off("SIGTERM", handleSigterm);
837
+ await closeServer(server);
838
+ resolveResult?.(makeResult(status, reason));
839
+ };
840
+ const abortWithError = async (error) => {
841
+ if (settled)
842
+ return;
843
+ settled = true;
844
+ acceptingRequests = false;
845
+ if (idleTimer)
846
+ clearTimeout(idleTimer);
847
+ process.off("SIGINT", handleSigint);
848
+ process.off("SIGTERM", handleSigterm);
849
+ await closeServer(server);
850
+ rejectResult?.(error);
851
+ };
852
+ const stopIncomplete = (reason, message) => {
853
+ acceptingRequests = false;
854
+ if (incompleteStopPromise)
855
+ return incompleteStopPromise;
856
+ incompleteStopPromise = (async () => {
857
+ if (settled)
858
+ return;
859
+ const now = new Date().toISOString();
860
+ checkpoint = {
861
+ ...checkpoint,
862
+ updatedAt: now,
863
+ lastStop: {
864
+ reason,
865
+ at: now,
866
+ pageNumber: checkpoint.nextPageNumber,
867
+ ...(message ? { message } : {})
868
+ }
869
+ };
870
+ await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
871
+ await finish("incomplete", reason);
872
+ })();
873
+ return incompleteStopPromise;
874
+ };
875
+ const requestIncompleteStop = (reason, message) => {
876
+ acceptingRequests = false;
877
+ return enqueueExclusive(async () => {
878
+ if (settled)
879
+ return;
880
+ if (checkpoint.status === "complete") {
881
+ await finish("complete");
882
+ return;
883
+ }
884
+ await stopIncomplete(reason, message);
885
+ });
886
+ };
887
+ function resetIdleTimer() {
888
+ if (idleTimer)
889
+ clearTimeout(idleTimer);
890
+ idleTimer = setTimeout(() => {
891
+ void requestIncompleteStop("idle_timeout", "No browser relay activity arrived before the local timeout.").catch((error) => {
892
+ rejectResult?.(error);
893
+ });
894
+ }, options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS);
895
+ idleTimer.unref();
896
+ }
897
+ async function handleRequest(req, res) {
898
+ if (!acceptingRequests) {
899
+ writeRelayJson(res, 410, { status: "closed" }, req.headers.origin);
900
+ return;
901
+ }
902
+ if (!isAllowedRelayOrigin(req)) {
903
+ writeRelayJson(res, 403, { status: "rejected", error: "Relay origin is not allowed." });
904
+ return;
905
+ }
906
+ resetIdleTimer();
907
+ const requestUrl = new URL(req.url ?? "/", "http://127.0.0.1");
908
+ if (req.method === "OPTIONS") {
909
+ res.writeHead(204, {
910
+ "Access-Control-Allow-Origin": "https://www.linkedin.com",
911
+ "Access-Control-Allow-Methods": "GET, POST, OPTIONS",
912
+ "Access-Control-Allow-Headers": "Content-Type",
913
+ "Access-Control-Allow-Private-Network": "true",
914
+ "Access-Control-Max-Age": "600",
915
+ Vary: "Origin"
916
+ });
917
+ res.end();
918
+ return;
919
+ }
920
+ if (req.method === "GET" && requestUrl.pathname === "/health") {
921
+ writeRelayJson(res, 200, { status: "ok", queryHash: expectedQueryHash }, req.headers.origin);
922
+ return;
923
+ }
924
+ if (req.method === "GET" && requestUrl.pathname === "/task") {
925
+ if (options.maxPages !== undefined && checkpoint.nextPageNumber > options.maxPages) {
926
+ acceptingRequests = false;
927
+ writeRelayJson(res, 200, { status: "incomplete", reason: "page_limit" }, req.headers.origin);
928
+ res.once("finish", () => void requestIncompleteStop("page_limit"));
929
+ return;
930
+ }
931
+ if (options.maxRequests !== undefined &&
932
+ uniqueTasksIssuedThisRun >= options.maxRequests &&
933
+ lastIssuedPage !== checkpoint.nextPageNumber) {
934
+ acceptingRequests = false;
935
+ writeRelayJson(res, 200, { status: "incomplete", reason: "request_limit" }, req.headers.origin);
936
+ res.once("finish", () => void requestIncompleteStop("request_limit"));
937
+ return;
938
+ }
939
+ if (lastIssuedPage !== checkpoint.nextPageNumber) {
940
+ uniqueTasksIssuedThisRun += 1;
941
+ lastIssuedPage = checkpoint.nextPageNumber;
942
+ checkpoint = { ...checkpoint, tasksIssued: checkpoint.tasksIssued + 1, updatedAt: new Date().toISOString() };
943
+ await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
944
+ }
945
+ writeRelayJson(res, 200, {
946
+ status: "task",
947
+ queryHash: expectedQueryHash,
948
+ pageNumber: checkpoint.nextPageNumber,
949
+ currentUrl: buildLinkedInProductSearchPageUrl(queryUrl, checkpoint.nextPageNumber),
950
+ productsAccepted: checkpoint.products.length
951
+ }, req.headers.origin);
952
+ return;
953
+ }
954
+ if (req.method === "POST" && requestUrl.pathname === "/page") {
955
+ const submission = linkedInProductPageSubmissionSchema.parse(await readRelayJson(req));
956
+ if (submission.pageNumber !== checkpoint.nextPageNumber) {
957
+ throw new LinkedInProductCollectorInvariantError("out_of_order_page", `Expected page ${checkpoint.nextPageNumber}, received page ${submission.pageNumber}.`);
958
+ }
959
+ if (lastIssuedPage !== submission.pageNumber) {
960
+ throw new LinkedInProductCollectorInvariantError("unleased_page", "Poll GET /task before posting the expected browser page.");
961
+ }
962
+ if ("error" in submission) {
963
+ acceptingRequests = false;
964
+ const sanitizedCurrentUrl = sanitizeTerminalErrorUrl(submission.currentUrl);
965
+ const receivedAt = new Date().toISOString();
966
+ await appendPrivateJsonLine(options.rawJsonlPath, {
967
+ type: "terminal_error",
968
+ queryHash: expectedQueryHash,
969
+ pageNumber: submission.pageNumber,
970
+ currentUrl: sanitizedCurrentUrl,
971
+ error: { code: submission.error.code },
972
+ receivedAt
973
+ });
974
+ writeRelayJson(res, 202, { status: "checkpointed", complete: false }, req.headers.origin);
975
+ res.once("finish", () => {
976
+ void requestIncompleteStop(submission.error.code, terminalStopMessage(submission.error.code)).catch((error) => rejectResult?.(error));
977
+ });
978
+ return;
979
+ }
980
+ const sanitizedCurrentUrl = assertExpectedLinkedInSearchPage(submission.currentUrl, checkpoint.queryUrl, submission.pageNumber);
981
+ let applied;
982
+ try {
983
+ applied = applyLinkedInProductSearchPage(checkpoint, submission);
984
+ }
985
+ catch (error) {
986
+ if (error instanceof LinkedInProductCollectorInvariantError && error.code === "identity_drift") {
987
+ acceptingRequests = false;
988
+ const now = new Date().toISOString();
989
+ checkpoint = {
990
+ ...checkpoint,
991
+ status: "failed",
992
+ updatedAt: now,
993
+ lastStop: { reason: error.code, at: now, pageNumber: checkpoint.nextPageNumber, message: error.message }
994
+ };
995
+ await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
996
+ writeRelayJson(res, 409, { status: "rejected", complete: false, error: error.message }, req.headers.origin);
997
+ res.once("finish", () => void finish("incomplete", error.code));
998
+ return;
999
+ }
1000
+ throw error;
1001
+ }
1002
+ await appendPrivateJsonLine(options.rawJsonlPath, {
1003
+ type: "page",
1004
+ queryHash: expectedQueryHash,
1005
+ pageNumber: submission.pageNumber,
1006
+ currentUrl: sanitizedCurrentUrl,
1007
+ reportedTotal: submission.reportedTotal ?? null,
1008
+ hasNext: submission.hasNext,
1009
+ items: submission.items.map(rawVisibleProductCard),
1010
+ visibleItemCount: submission.items.length,
1011
+ newProductCount: applied.newProductCount,
1012
+ duplicateCount: applied.duplicateCount,
1013
+ receivedAt: applied.checkpoint.updatedAt
1014
+ });
1015
+ checkpoint = applied.checkpoint;
1016
+ await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
1017
+ if (checkpoint.status === "complete") {
1018
+ const artifact = buildLinkedInProductCollectionArtifact(checkpoint);
1019
+ await writePrivateJsonAtomic(options.outPath, artifact);
1020
+ acceptingRequests = false;
1021
+ }
1022
+ lastIssuedPage = null;
1023
+ writeRelayJson(res, 202, {
1024
+ status: checkpoint.status === "complete" ? "complete" : "accepted",
1025
+ complete: checkpoint.status === "complete",
1026
+ pageNumber: submission.pageNumber,
1027
+ newProductCount: applied.newProductCount,
1028
+ duplicateCount: applied.duplicateCount,
1029
+ productsAccepted: checkpoint.products.length,
1030
+ nextPageNumber: checkpoint.nextPageNumber
1031
+ }, req.headers.origin);
1032
+ if (checkpoint.status === "complete") {
1033
+ res.once("finish", () => void finish("complete"));
1034
+ }
1035
+ else if (options.maxPages !== undefined && checkpoint.nextPageNumber > options.maxPages) {
1036
+ acceptingRequests = false;
1037
+ res.once("finish", () => void requestIncompleteStop("page_limit"));
1038
+ }
1039
+ else if (options.maxRequests !== undefined && uniqueTasksIssuedThisRun >= options.maxRequests) {
1040
+ acceptingRequests = false;
1041
+ res.once("finish", () => void requestIncompleteStop("request_limit"));
1042
+ }
1043
+ return;
1044
+ }
1045
+ writeRelayJson(res, 404, { status: "not_found" }, req.headers.origin);
1046
+ }
1047
+ const handleSigint = () => {
1048
+ void requestIncompleteStop("interrupted", "The collector received SIGINT.").catch((error) => rejectResult?.(error));
1049
+ };
1050
+ const handleSigterm = () => {
1051
+ void requestIncompleteStop("interrupted", "The collector received SIGTERM.").catch((error) => rejectResult?.(error));
1052
+ };
1053
+ process.once("SIGINT", handleSigint);
1054
+ process.once("SIGTERM", handleSigterm);
1055
+ try {
1056
+ await new Promise((resolve, reject) => {
1057
+ server.once("error", reject);
1058
+ server.listen(options.relayPort, "127.0.0.1", () => resolve());
1059
+ });
1060
+ resetIdleTimer();
1061
+ const address = server.address();
1062
+ if (!address || typeof address === "string")
1063
+ throw new Error("The browser relay did not bind to TCP.");
1064
+ options.onListening?.({
1065
+ host: "127.0.0.1",
1066
+ port: address.port,
1067
+ taskUrl: `http://127.0.0.1:${address.port}/task`,
1068
+ pageUrl: `http://127.0.0.1:${address.port}/page`
1069
+ });
1070
+ return await resultPromise;
1071
+ }
1072
+ catch (error) {
1073
+ if (idleTimer)
1074
+ clearTimeout(idleTimer);
1075
+ process.off("SIGINT", handleSigint);
1076
+ process.off("SIGTERM", handleSigterm);
1077
+ if (server.listening)
1078
+ await closeServer(server);
1079
+ throw error;
1080
+ }
1081
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "salesprompter-cli",
3
- "version": "0.1.65",
3
+ "version": "0.1.67",
4
4
  "description": "Sales workflow CLI for guided lead generation, enrichment, scoring, and sync.",
5
5
  "author": "Daniel Sinewe <hello@danielsinewe.com>",
6
6
  "type": "module",