salesprompter-cli 0.1.68 → 0.1.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -139,6 +139,19 @@ salesprompter products:collect \
139
139
 
140
140
  Use a signed-in in-app-browser worker with the loopback `GET /task` and `POST /page` protocol. Interrupted, bounded, challenged, or rate-limited runs remain checkpointed and do not create the complete artifact.
141
141
 
142
+ Enrich that proven order with public product-page details, without changing its ranks or uploading it:
143
+
144
+ ```bash
145
+ salesprompter products:collect-details \
146
+ --catalog ./data/products.complete.json \
147
+ --checkpoint ./data/products.details.checkpoint.json \
148
+ --raw-jsonl ./data/products.details.raw.jsonl \
149
+ --out ./data/products.with-details.json \
150
+ --relay-port 43118
151
+ ```
152
+
153
+ The detail worker uses the same local `GET /task` lease protocol and posts only allowlisted product fields to `POST /detail`. Public URLs must be HTTPS; product images are capped at 2,048 characters, other URLs at 4,096, descriptions at 10,000, other strings at 200, roles/features at 100 entries, and customer companies at 50. The final artifact is created only after every rank has either a complete detail page or explicit unavailable-page proof.
154
+
142
155
  Download stored workspace leads without starting a new Sales Navigator scrape:
143
156
 
144
157
  ```bash
package/dist/cli.js CHANGED
@@ -28,6 +28,7 @@ import { buildHistoricalVendorIcp, buildVendorIcp } from "./icp-templates.js";
28
28
  import { InstantlySyncProvider } from "./instantly.js";
29
29
  import { backfillLinkedInCompanies } from "./linkedin-companies.js";
30
30
  import { parseLinkedInCompanyPage } from "./linkedin-companies.js";
31
+ import { collectLinkedInProductDetailsViaBrowserRelay } from "./linkedin-product-details.js";
31
32
  import { collectLinkedInProductsViaBrowserRelay } from "./linkedin-product-search.js";
32
33
  import { crawlLinkedInProductCategory } from "./linkedin-products.js";
33
34
  import { claimLinkedInSessionCookieForCli, claimValidatedSalesNavigatorSessionCookieForCli, createLinkedInSessionSupabaseClient, recordLinkedInSessionCookieAudit, resolveConfiguredEnvValue } from "./linkedin-session.js";
@@ -626,7 +627,7 @@ const cliPacks = [
626
627
  slug: "research",
627
628
  title: "Research",
628
629
  summary: "Scrape markets and enrich companies before outreach.",
629
- commands: ["products:collect", "market:scrape", "companies:enrich"],
630
+ commands: ["products:collect", "products:collect-details", "market:scrape", "companies:enrich"],
630
631
  installStatus: "included"
631
632
  },
632
633
  {
@@ -677,6 +678,7 @@ const helpAliasByCommandName = new Map([
677
678
  ["linkedin-companies:scrape-local", "companies:scrape-linkedin"],
678
679
  ["dealroom-companies:scrape-local", "companies:scrape-dealroom"],
679
680
  ["linkedin-products:collect", "products:collect"],
681
+ ["linkedin-products:collect-details", "products:collect-details"],
680
682
  ["linkedin-products:scrape", "market:scrape"],
681
683
  ["salesnav:from-product-category", "leads:discover"],
682
684
  ["salesnav:people:collect", "leads:collect"],
@@ -728,6 +730,7 @@ const helpVisibleCommandNames = new Set([
728
730
  "linkedin-companies:scrape-local",
729
731
  "dealroom-companies:scrape-local",
730
732
  "linkedin-products:collect",
733
+ "linkedin-products:collect-details",
731
734
  "linkedin-products:scrape",
732
735
  "salesnav:from-product-category",
733
736
  "salesnav:people:collect",
@@ -15349,6 +15352,38 @@ program
15349
15352
  });
15350
15353
  printOutput(result);
15351
15354
  });
15355
+ program
15356
+ .command("linkedin-products:collect-details")
15357
+ .alias("products:collect-details")
15358
+ .description("Enrich a proven LinkedIn product ranking locally through a resumable loopback browser relay.")
15359
+ .requiredOption("--catalog <path>", "Complete products:collect ranking artifact path")
15360
+ .requiredOption("--checkpoint <path>", "Private resumable product-detail checkpoint JSON path")
15361
+ .requiredOption("--raw-jsonl <path>", "Private append-only normalized product-detail evidence path")
15362
+ .requiredOption("--out <path>", "Complete public-fields-only enriched artifact path")
15363
+ .requiredOption("--relay-port <number>", "Loopback relay port for GET /task and POST /detail")
15364
+ .option("--max-requests <number>", "Optional number of unique product-detail tasks to issue in this run")
15365
+ .action(async (options) => {
15366
+ const relayPort = z.coerce.number().int().min(1).max(65_535).parse(options.relayPort);
15367
+ const maxRequests = options.maxRequests === undefined
15368
+ ? undefined
15369
+ : z.coerce.number().int().min(1).max(100_000).parse(options.maxRequests);
15370
+ const idleTimeoutMs = process.env.SALESPROMPTER_LINKEDIN_PRODUCT_DETAILS_RELAY_IDLE_TIMEOUT_MS
15371
+ ? z.coerce.number().int().min(100).max(86_400_000).parse(process.env.SALESPROMPTER_LINKEDIN_PRODUCT_DETAILS_RELAY_IDLE_TIMEOUT_MS)
15372
+ : undefined;
15373
+ const result = await collectLinkedInProductDetailsViaBrowserRelay({
15374
+ catalogPath: path.resolve(String(options.catalog)),
15375
+ checkpointPath: path.resolve(String(options.checkpoint)),
15376
+ rawJsonlPath: path.resolve(String(options.rawJsonl)),
15377
+ outPath: path.resolve(String(options.out)),
15378
+ relayPort,
15379
+ maxRequests,
15380
+ idleTimeoutMs,
15381
+ onListening: ({ taskUrl, detailUrl }) => {
15382
+ writeProgress(`LinkedIn product detail relay ready: GET ${taskUrl} and POST ${detailUrl}`);
15383
+ }
15384
+ });
15385
+ printOutput(result);
15386
+ });
15352
15387
  program
15353
15388
  .command("linkedin-products:scrape")
15354
15389
  .alias("market:scrape")
@@ -0,0 +1,1203 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ import { access, appendFile, chmod, mkdir, readFile, rename, writeFile } from "node:fs/promises";
3
+ import { createServer } from "node:http";
4
+ import path from "node:path";
5
+ import { z } from "zod";
6
+ import { buildLinkedInProductSearchPageUrl, canonicalizeLinkedInProductUrl, computeLinkedInProductOrderChecksum, LinkedInProductCollectorInvariantError, normalizeLinkedInProductSearchUrl } from "./linkedin-product-search.js";
7
+ const CHECKPOINT_SCHEMA_VERSION = 1;
8
+ const ENRICHED_ARTIFACT_SCHEMA_VERSION = 2;
9
+ const MAX_RELAY_BODY_BYTES = 5 * 1024 * 1024;
10
+ const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
11
+ const nonEmptyText = (max) => z.string().trim().min(1).max(max);
12
+ const httpsUrl = (max) => z
13
+ .string()
14
+ .trim()
15
+ .url()
16
+ .max(max)
17
+ .refine((value) => {
18
+ const parsed = new URL(value);
19
+ return parsed.protocol === "https:" && parsed.username === "" && parsed.password === "";
20
+ }, "Public URLs must use HTTPS without embedded credentials.");
21
+ const optionalPublicUrl = httpsUrl(4_096).optional();
22
+ const optionalProductImageUrl = httpsUrl(2_048).optional();
23
+ const rankingProductSchema = z
24
+ .object({
25
+ rank: z.number().int().min(1),
26
+ name: nonEmptyText(200),
27
+ linkedinUrl: nonEmptyText(4_096),
28
+ linkedinSlug: nonEmptyText(720),
29
+ imageUrl: optionalProductImageUrl,
30
+ vendor: nonEmptyText(200).optional(),
31
+ category: nonEmptyText(200).optional(),
32
+ description: nonEmptyText(10_000).optional()
33
+ })
34
+ .strict();
35
+ const terminalPageSchema = z
36
+ .object({
37
+ pageNumber: z.number().int().min(1),
38
+ currentUrl: nonEmptyText(8_192),
39
+ reportedTotal: z.number().int().min(0).nullable(),
40
+ hasNext: z.literal(false),
41
+ visibleItemCount: z.number().int().min(0),
42
+ newProductCount: z.number().int().min(0),
43
+ capturedAt: nonEmptyText(100)
44
+ })
45
+ .strict();
46
+ const rankingArtifactSchema = z
47
+ .object({
48
+ schemaVersion: z.literal(1),
49
+ kind: z.literal("linkedin_product_search_snapshot"),
50
+ complete: z.literal(true),
51
+ source: z
52
+ .object({
53
+ queryUrl: nonEmptyText(8_192),
54
+ queryHash: z.string().regex(/^[a-f0-9]{64}$/)
55
+ })
56
+ .strict(),
57
+ capture: z
58
+ .object({
59
+ startedAt: nonEmptyText(100),
60
+ completedAt: nonEmptyText(100),
61
+ approximateReportedTotal: z.number().int().min(0).nullable(),
62
+ pagesAccepted: z.number().int().min(1),
63
+ terminalPage: terminalPageSchema
64
+ })
65
+ .strict(),
66
+ productCount: z.number().int().min(1),
67
+ orderChecksumAlgorithm: z.literal("sha256"),
68
+ orderChecksum: z.string().regex(/^[a-f0-9]{64}$/),
69
+ products: z.array(rankingProductSchema).min(1).max(100_000)
70
+ })
71
+ .strict();
72
+ const intendedRolesSchema = z.array(nonEmptyText(200)).max(100);
73
+ const featuresSchema = z.array(nonEmptyText(200)).max(100);
74
+ const usedByCompanySchema = z
75
+ .object({
76
+ name: nonEmptyText(200),
77
+ linkedinUrl: nonEmptyText(4_096).optional(),
78
+ logoUrl: optionalPublicUrl
79
+ })
80
+ .strict();
81
+ const completeDetailInputSchema = z
82
+ .object({
83
+ name: nonEmptyText(200),
84
+ imageUrl: optionalProductImageUrl,
85
+ vendor: nonEmptyText(200).optional(),
86
+ category: nonEmptyText(200).optional(),
87
+ description: nonEmptyText(10_000).optional(),
88
+ websiteUrl: optionalPublicUrl,
89
+ vendorLinkedinUrl: nonEmptyText(4_096).optional(),
90
+ vendorHandle: nonEmptyText(200).optional(),
91
+ intendedRoles: intendedRolesSchema,
92
+ usedBy: z.array(usedByCompanySchema).max(50),
93
+ features: featuresSchema
94
+ })
95
+ .strict();
96
+ export const linkedInProductUnavailableProofSchema = z.discriminatedUnion("kind", [
97
+ z
98
+ .object({
99
+ kind: z.literal("http_status"),
100
+ status: z.union([z.literal(404), z.literal(410)])
101
+ })
102
+ .strict(),
103
+ z
104
+ .object({
105
+ kind: z.literal("linkedin_marker"),
106
+ marker: z.enum(["page_not_found", "product_unavailable"])
107
+ })
108
+ .strict()
109
+ ]);
110
+ const relayErrorCodeSchema = z.enum([
111
+ "auth_required",
112
+ "challenge",
113
+ "rate_limited",
114
+ "interrupted",
115
+ "unexpected_page"
116
+ ]);
117
+ const completeDetailSubmissionSchema = z
118
+ .object({
119
+ rank: z.number().int().min(1),
120
+ currentUrl: nonEmptyText(8_192),
121
+ detail: completeDetailInputSchema
122
+ })
123
+ .strict();
124
+ const unavailableDetailSubmissionSchema = z
125
+ .object({
126
+ rank: z.number().int().min(1),
127
+ currentUrl: nonEmptyText(8_192),
128
+ unavailable: z
129
+ .object({
130
+ code: z.literal("product_unavailable"),
131
+ proof: linkedInProductUnavailableProofSchema
132
+ })
133
+ .strict()
134
+ })
135
+ .strict();
136
+ const terminalErrorSubmissionSchema = z
137
+ .object({
138
+ rank: z.number().int().min(1),
139
+ currentUrl: nonEmptyText(8_192),
140
+ error: z
141
+ .object({
142
+ code: relayErrorCodeSchema,
143
+ message: z.string().trim().max(1_000).optional()
144
+ })
145
+ .strict()
146
+ })
147
+ .strict();
148
+ export const linkedInProductDetailSubmissionSchema = z.union([
149
+ completeDetailSubmissionSchema,
150
+ unavailableDetailSubmissionSchema,
151
+ terminalErrorSubmissionSchema
152
+ ]);
153
+ const storedCompleteDetailSchema = completeDetailInputSchema
154
+ .extend({
155
+ status: z.literal("complete"),
156
+ capturedAt: nonEmptyText(100)
157
+ })
158
+ .strict();
159
+ const storedUnavailableDetailSchema = z
160
+ .object({
161
+ status: z.literal("unavailable"),
162
+ capturedAt: nonEmptyText(100),
163
+ proof: linkedInProductUnavailableProofSchema
164
+ })
165
+ .strict();
166
+ const storedDetailSchema = z.union([storedCompleteDetailSchema, storedUnavailableDetailSchema]);
167
+ const checkpointDetailSchema = z
168
+ .object({
169
+ rank: z.number().int().min(1),
170
+ linkedinUrl: nonEmptyText(4_096),
171
+ linkedinSlug: nonEmptyText(720),
172
+ detail: storedDetailSchema
173
+ })
174
+ .strict();
175
+ const checkpointSchema = z
176
+ .object({
177
+ schemaVersion: z.literal(CHECKPOINT_SCHEMA_VERSION),
178
+ source: z
179
+ .object({
180
+ queryHash: z.string().regex(/^[a-f0-9]{64}$/),
181
+ orderChecksum: z.string().regex(/^[a-f0-9]{64}$/),
182
+ productCount: z.number().int().min(1)
183
+ })
184
+ .strict(),
185
+ startedAt: nonEmptyText(100),
186
+ updatedAt: nonEmptyText(100),
187
+ status: z.enum(["collecting", "complete"]),
188
+ nextRank: z.number().int().min(1),
189
+ tasksIssued: z.number().int().min(0),
190
+ details: z.array(checkpointDetailSchema).max(100_000),
191
+ completedAt: nonEmptyText(100).optional(),
192
+ detailChecksum: z.string().regex(/^[a-f0-9]{64}$/).optional(),
193
+ lastStop: z
194
+ .object({
195
+ reason: nonEmptyText(200),
196
+ at: nonEmptyText(100),
197
+ rank: z.number().int().min(1),
198
+ message: z.string().trim().max(1_000).optional()
199
+ })
200
+ .strict()
201
+ .optional()
202
+ })
203
+ .strict();
204
+ const rawCompleteDetailResultSchema = z
205
+ .object({
206
+ status: z.literal("complete"),
207
+ detail: completeDetailInputSchema
208
+ })
209
+ .strict();
210
+ const rawUnavailableDetailResultSchema = z
211
+ .object({
212
+ status: z.literal("unavailable"),
213
+ proof: linkedInProductUnavailableProofSchema
214
+ })
215
+ .strict();
216
+ const rawDetailEvidenceSchema = z
217
+ .object({
218
+ type: z.literal("detail"),
219
+ orderChecksum: z.string().regex(/^[a-f0-9]{64}$/),
220
+ rank: z.number().int().min(1),
221
+ currentUrl: nonEmptyText(4_096),
222
+ result: z.union([rawCompleteDetailResultSchema, rawUnavailableDetailResultSchema]),
223
+ receivedAt: nonEmptyText(100)
224
+ })
225
+ .strict();
226
+ const rawTerminalEvidenceSchema = z
227
+ .object({
228
+ type: z.literal("terminal_error"),
229
+ orderChecksum: z.string().regex(/^[a-f0-9]{64}$/),
230
+ rank: z.number().int().min(1),
231
+ currentUrl: nonEmptyText(4_096),
232
+ error: z.object({ code: relayErrorCodeSchema }).strict(),
233
+ receivedAt: nonEmptyText(100)
234
+ })
235
+ .strict();
236
+ const rawEvidenceRecordSchema = z.union([rawDetailEvidenceSchema, rawTerminalEvidenceSchema]);
237
+ function sha256(value) {
238
+ return createHash("sha256").update(value, "utf8").digest("hex");
239
+ }
240
+ function canonicalJson(value) {
241
+ if (value === null || typeof value !== "object") {
242
+ return JSON.stringify(value);
243
+ }
244
+ if (Array.isArray(value)) {
245
+ return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`;
246
+ }
247
+ const record = value;
248
+ return `{${Object.keys(record)
249
+ .sort()
250
+ .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
251
+ .join(",")}}`;
252
+ }
253
+ function cleanText(value) {
254
+ const cleaned = value?.replace(/\s+/g, " ").trim();
255
+ return cleaned || undefined;
256
+ }
257
+ function uniqueText(values) {
258
+ const seen = new Set();
259
+ const output = [];
260
+ for (const value of values) {
261
+ const cleaned = cleanText(value);
262
+ if (!cleaned)
263
+ continue;
264
+ const key = cleaned.toLocaleLowerCase("en-US");
265
+ if (seen.has(key))
266
+ continue;
267
+ seen.add(key);
268
+ output.push(cleaned);
269
+ }
270
+ return output;
271
+ }
272
+ function canonicalizePublicUrl(value, maxLength = 4_096) {
273
+ const cleaned = cleanText(value);
274
+ if (!cleaned)
275
+ return undefined;
276
+ let parsed;
277
+ try {
278
+ parsed = new URL(cleaned);
279
+ }
280
+ catch {
281
+ throw new LinkedInProductCollectorInvariantError("invalid_public_url", `Invalid public URL: ${cleaned}`);
282
+ }
283
+ if (parsed.protocol !== "https:" ||
284
+ parsed.username !== "" ||
285
+ parsed.password !== "") {
286
+ throw new LinkedInProductCollectorInvariantError("invalid_public_url", `Unsupported public URL: ${cleaned}`);
287
+ }
288
+ parsed.hash = "";
289
+ const canonical = parsed.toString();
290
+ if (canonical.length > maxLength) {
291
+ throw new LinkedInProductCollectorInvariantError("invalid_public_url", `Public URL exceeds ${maxLength} characters.`);
292
+ }
293
+ return canonical;
294
+ }
295
+ export function canonicalizeLinkedInCompanyUrl(value) {
296
+ let parsed;
297
+ try {
298
+ parsed = new URL(value, "https://www.linkedin.com");
299
+ }
300
+ catch {
301
+ throw new LinkedInProductCollectorInvariantError("invalid_company_url", `Invalid LinkedIn company URL: ${value}`);
302
+ }
303
+ if (parsed.protocol !== "https:" ||
304
+ parsed.port !== "" ||
305
+ parsed.username !== "" ||
306
+ parsed.password !== "" ||
307
+ !["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
308
+ throw new LinkedInProductCollectorInvariantError("invalid_company_url", `Company URL is outside LinkedIn: ${value}`);
309
+ }
310
+ const match = parsed.pathname.match(/^\/company\/([^/]+)\/?$/i);
311
+ if (!match?.[1]) {
312
+ throw new LinkedInProductCollectorInvariantError("invalid_company_url", `Company URL must match /company/<handle>/: ${value}`);
313
+ }
314
+ let decodedHandle;
315
+ try {
316
+ decodedHandle = decodeURIComponent(match[1]).trim().normalize("NFC").toLowerCase();
317
+ }
318
+ catch {
319
+ throw new LinkedInProductCollectorInvariantError("invalid_company_url", `Invalid encoded company handle: ${value}`);
320
+ }
321
+ if (!decodedHandle || /[\s/?#\\]/u.test(decodedHandle)) {
322
+ throw new LinkedInProductCollectorInvariantError("invalid_company_url", `Invalid company handle: ${value}`);
323
+ }
324
+ const handle = encodeURIComponent(decodedHandle)
325
+ .replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).padStart(2, "0")}`)
326
+ .toLowerCase();
327
+ if (handle.length > 720 || !/^(?:[a-z0-9._~-]|%[0-9a-f]{2})+$/.test(handle)) {
328
+ throw new LinkedInProductCollectorInvariantError("invalid_company_url", `Invalid company handle: ${handle}`);
329
+ }
330
+ return {
331
+ linkedinUrl: `https://www.linkedin.com/company/${handle}`,
332
+ handle
333
+ };
334
+ }
335
+ function normalizeVendorHandle(value) {
336
+ const cleaned = cleanText(value)?.normalize("NFC").toLowerCase();
337
+ if (!cleaned)
338
+ return undefined;
339
+ if (cleaned.length > 200 || /[\s/?#\\]/u.test(cleaned)) {
340
+ throw new LinkedInProductCollectorInvariantError("invalid_vendor_handle", "Vendor handle is invalid.");
341
+ }
342
+ return cleaned;
343
+ }
344
+ function normalizeUsedBy(values) {
345
+ const seen = new Set();
346
+ const result = [];
347
+ for (const value of values) {
348
+ const name = cleanText(value.name);
349
+ if (!name)
350
+ continue;
351
+ const company = value.linkedinUrl ? canonicalizeLinkedInCompanyUrl(value.linkedinUrl) : undefined;
352
+ const logoUrl = canonicalizePublicUrl(value.logoUrl);
353
+ const key = company?.linkedinUrl ?? name.toLocaleLowerCase("en-US");
354
+ if (seen.has(key))
355
+ continue;
356
+ seen.add(key);
357
+ result.push({
358
+ name,
359
+ ...(company ? { linkedinUrl: company.linkedinUrl } : {}),
360
+ ...(logoUrl ? { logoUrl } : {})
361
+ });
362
+ }
363
+ return result;
364
+ }
365
+ function normalizeCompleteDetail(input, capturedAt) {
366
+ const name = cleanText(input.name);
367
+ if (!name) {
368
+ throw new LinkedInProductCollectorInvariantError("invalid_product_detail", "Product detail name is empty.");
369
+ }
370
+ const vendorLinkedIn = input.vendorLinkedinUrl
371
+ ? canonicalizeLinkedInCompanyUrl(input.vendorLinkedinUrl)
372
+ : undefined;
373
+ const suppliedVendorHandle = normalizeVendorHandle(input.vendorHandle);
374
+ const derivedVendorHandle = vendorLinkedIn
375
+ ? normalizeVendorHandle(decodeURIComponent(vendorLinkedIn.handle))
376
+ : undefined;
377
+ const imageUrl = canonicalizePublicUrl(input.imageUrl, 2_048);
378
+ const websiteUrl = canonicalizePublicUrl(input.websiteUrl);
379
+ if (vendorLinkedIn &&
380
+ suppliedVendorHandle &&
381
+ derivedVendorHandle !== suppliedVendorHandle) {
382
+ throw new LinkedInProductCollectorInvariantError("vendor_identity_drift", "Vendor LinkedIn URL and handle disagree.");
383
+ }
384
+ return {
385
+ status: "complete",
386
+ capturedAt,
387
+ name,
388
+ ...(imageUrl ? { imageUrl } : {}),
389
+ ...(cleanText(input.vendor) ? { vendor: cleanText(input.vendor) } : {}),
390
+ ...(cleanText(input.category) ? { category: cleanText(input.category) } : {}),
391
+ ...(cleanText(input.description) ? { description: cleanText(input.description) } : {}),
392
+ ...(websiteUrl ? { websiteUrl } : {}),
393
+ ...(vendorLinkedIn ? { vendorLinkedinUrl: vendorLinkedIn.linkedinUrl } : {}),
394
+ ...(suppliedVendorHandle || derivedVendorHandle
395
+ ? { vendorHandle: suppliedVendorHandle ?? derivedVendorHandle }
396
+ : {}),
397
+ intendedRoles: uniqueText(input.intendedRoles),
398
+ usedBy: normalizeUsedBy(input.usedBy),
399
+ features: uniqueText(input.features)
400
+ };
401
+ }
402
+ function assertIsoTimestamp(value, field) {
403
+ if (typeof value !== "string") {
404
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", `${field} is not a timestamp.`);
405
+ }
406
+ const parsed = new Date(value);
407
+ if (Number.isNaN(parsed.valueOf()) || parsed.toISOString() !== value) {
408
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", `${field} is not canonical ISO time.`);
409
+ }
410
+ }
411
+ export function validateLinkedInProductRankingArtifact(input) {
412
+ const artifact = rankingArtifactSchema.parse(input);
413
+ const queryUrl = normalizeLinkedInProductSearchUrl(artifact.source.queryUrl);
414
+ if (queryUrl !== artifact.source.queryUrl || sha256(queryUrl) !== artifact.source.queryHash) {
415
+ throw new LinkedInProductCollectorInvariantError("invalid_catalog", "Ranking artifact query URL and query hash disagree.");
416
+ }
417
+ if (artifact.productCount !== artifact.products.length) {
418
+ throw new LinkedInProductCollectorInvariantError("invalid_catalog", "Ranking artifact product count does not match its products.");
419
+ }
420
+ assertIsoTimestamp(artifact.capture.startedAt, "catalog.capture.startedAt");
421
+ assertIsoTimestamp(artifact.capture.completedAt, "catalog.capture.completedAt");
422
+ assertIsoTimestamp(artifact.capture.terminalPage.capturedAt, "catalog.capture.terminalPage.capturedAt");
423
+ if (artifact.capture.completedAt !== artifact.capture.terminalPage.capturedAt ||
424
+ artifact.capture.startedAt > artifact.capture.completedAt ||
425
+ artifact.capture.terminalPage.pageNumber !== artifact.capture.pagesAccepted ||
426
+ artifact.capture.terminalPage.currentUrl !==
427
+ buildLinkedInProductSearchPageUrl(artifact.source.queryUrl, artifact.capture.terminalPage.pageNumber) ||
428
+ artifact.capture.terminalPage.newProductCount > artifact.capture.terminalPage.visibleItemCount) {
429
+ throw new LinkedInProductCollectorInvariantError("invalid_catalog", "Ranking artifact terminal-page evidence is inconsistent.");
430
+ }
431
+ const seenUrls = new Set();
432
+ artifact.products.forEach((product, index) => {
433
+ if (product.rank !== index + 1) {
434
+ throw new LinkedInProductCollectorInvariantError("rank_drift", "Ranking artifact ranks are not contiguous.");
435
+ }
436
+ const canonical = canonicalizeLinkedInProductUrl(product.linkedinUrl);
437
+ if (canonical.linkedinUrl !== product.linkedinUrl || canonical.linkedinSlug !== product.linkedinSlug) {
438
+ throw new LinkedInProductCollectorInvariantError("canonical_url_drift", `Ranking artifact product ${product.rank} is not canonical.`);
439
+ }
440
+ if (seenUrls.has(canonical.linkedinUrl)) {
441
+ throw new LinkedInProductCollectorInvariantError("duplicate_product", "Ranking artifact contains duplicate URLs.");
442
+ }
443
+ seenUrls.add(canonical.linkedinUrl);
444
+ if (product.imageUrl) {
445
+ canonicalizePublicUrl(product.imageUrl, 2_048);
446
+ }
447
+ });
448
+ if (computeLinkedInProductOrderChecksum(artifact.products) !== artifact.orderChecksum) {
449
+ throw new LinkedInProductCollectorInvariantError("checksum_drift", "Ranking artifact order checksum does not match its products.");
450
+ }
451
+ return artifact;
452
+ }
453
+ export async function readLinkedInProductRankingArtifact(catalogPath) {
454
+ let parsed;
455
+ try {
456
+ parsed = JSON.parse(await readFile(path.resolve(catalogPath), "utf8"));
457
+ }
458
+ catch (error) {
459
+ if (error.code === "ENOENT") {
460
+ throw new LinkedInProductCollectorInvariantError("missing_catalog", "Ranking artifact does not exist.");
461
+ }
462
+ throw error;
463
+ }
464
+ return validateLinkedInProductRankingArtifact(parsed);
465
+ }
466
+ export function createLinkedInProductDetailsCheckpoint(catalog, now = new Date()) {
467
+ validateLinkedInProductRankingArtifact(catalog);
468
+ const timestamp = now.toISOString();
469
+ return {
470
+ schemaVersion: CHECKPOINT_SCHEMA_VERSION,
471
+ source: {
472
+ queryHash: catalog.source.queryHash,
473
+ orderChecksum: catalog.orderChecksum,
474
+ productCount: catalog.productCount
475
+ },
476
+ startedAt: timestamp,
477
+ updatedAt: timestamp,
478
+ status: "collecting",
479
+ nextRank: 1,
480
+ tasksIssued: 0,
481
+ details: []
482
+ };
483
+ }
484
+ function assertExpectedProductDetailPage(currentUrl, expectedProduct) {
485
+ let parsed;
486
+ try {
487
+ parsed = new URL(currentUrl);
488
+ }
489
+ catch {
490
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", "Browser reported an invalid product URL.");
491
+ }
492
+ if (parsed.protocol !== "https:" ||
493
+ parsed.port !== "" ||
494
+ parsed.username !== "" ||
495
+ parsed.password !== "" ||
496
+ !["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
497
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", "Browser is not on the expected LinkedIn product page.");
498
+ }
499
+ const canonical = canonicalizeLinkedInProductUrl(currentUrl);
500
+ if (canonical.linkedinUrl !== expectedProduct.linkedinUrl) {
501
+ throw new LinkedInProductCollectorInvariantError("unexpected_page", `Expected ${expectedProduct.linkedinUrl}, received another product page.`);
502
+ }
503
+ return canonical.linkedinUrl;
504
+ }
505
+ export function computeLinkedInProductDetailChecksum(details) {
506
+ const manifest = details
507
+ .map((entry) => `${entry.rank}\t${entry.linkedinUrl}\t${canonicalJson(entry.detail)}`)
508
+ .join("\n");
509
+ return sha256(manifest);
510
+ }
511
+ export function applyLinkedInProductDetailSubmission(catalog, checkpoint, submission, now = new Date()) {
512
+ validateLinkedInProductRankingArtifact(catalog);
513
+ assertValidCheckpoint(checkpoint, catalog);
514
+ if (checkpoint.status !== "collecting") {
515
+ throw new LinkedInProductCollectorInvariantError("checkpoint_closed", "Detail checkpoint is complete.");
516
+ }
517
+ if (submission.rank !== checkpoint.nextRank) {
518
+ throw new LinkedInProductCollectorInvariantError("out_of_order_detail", `Expected product rank ${checkpoint.nextRank}, received ${submission.rank}.`);
519
+ }
520
+ const product = catalog.products[submission.rank - 1];
521
+ if (!product) {
522
+ throw new LinkedInProductCollectorInvariantError("invalid_rank", "Detail rank exceeds catalog size.");
523
+ }
524
+ assertExpectedProductDetailPage(submission.currentUrl, product);
525
+ const capturedAt = now.toISOString();
526
+ const detail = "detail" in submission
527
+ ? normalizeCompleteDetail(submission.detail, capturedAt)
528
+ : {
529
+ status: "unavailable",
530
+ capturedAt,
531
+ proof: submission.unavailable.proof
532
+ };
533
+ const accepted = {
534
+ rank: product.rank,
535
+ linkedinUrl: product.linkedinUrl,
536
+ linkedinSlug: product.linkedinSlug,
537
+ detail
538
+ };
539
+ const details = [...checkpoint.details, accepted];
540
+ const complete = details.length === catalog.productCount;
541
+ const nextCheckpoint = {
542
+ ...checkpoint,
543
+ updatedAt: capturedAt,
544
+ status: complete ? "complete" : "collecting",
545
+ nextRank: product.rank + 1,
546
+ tasksIssued: Math.max(checkpoint.tasksIssued, details.length),
547
+ details,
548
+ lastStop: undefined,
549
+ ...(complete
550
+ ? {
551
+ completedAt: capturedAt,
552
+ detailChecksum: computeLinkedInProductDetailChecksum(details)
553
+ }
554
+ : {
555
+ completedAt: undefined,
556
+ detailChecksum: undefined
557
+ })
558
+ };
559
+ return { checkpoint: nextCheckpoint, accepted };
560
+ }
561
+ export function buildLinkedInProductDetailsArtifact(catalog, checkpoint) {
562
+ validateLinkedInProductRankingArtifact(catalog);
563
+ assertValidCheckpoint(checkpoint, catalog);
564
+ if (checkpoint.status !== "complete" || !checkpoint.completedAt || !checkpoint.detailChecksum) {
565
+ throw new LinkedInProductCollectorInvariantError("incomplete_checkpoint", "A complete detail artifact requires every ranked product to have terminal detail evidence.");
566
+ }
567
+ const byRank = new Map(checkpoint.details.map((entry) => [entry.rank, entry]));
568
+ const products = catalog.products.map((product) => {
569
+ const stored = byRank.get(product.rank);
570
+ if (!stored) {
571
+ throw new LinkedInProductCollectorInvariantError("missing_detail", `Missing detail for rank ${product.rank}.`);
572
+ }
573
+ if (stored.detail.status === "unavailable") {
574
+ return {
575
+ ...product,
576
+ detailStatus: "unavailable",
577
+ detailsScrapedAt: stored.detail.capturedAt,
578
+ intendedRoles: [],
579
+ usedBy: [],
580
+ features: [],
581
+ unavailableProof: stored.detail.proof
582
+ };
583
+ }
584
+ const detail = stored.detail;
585
+ return {
586
+ ...product,
587
+ name: detail.name,
588
+ ...(detail.imageUrl ? { imageUrl: detail.imageUrl } : {}),
589
+ ...(detail.vendor ? { vendor: detail.vendor } : {}),
590
+ ...(detail.category ? { category: detail.category } : {}),
591
+ ...(detail.description ? { description: detail.description } : {}),
592
+ detailStatus: "complete",
593
+ detailsScrapedAt: detail.capturedAt,
594
+ ...(detail.websiteUrl ? { websiteUrl: detail.websiteUrl } : {}),
595
+ ...(detail.vendorLinkedinUrl ? { vendorLinkedinUrl: detail.vendorLinkedinUrl } : {}),
596
+ ...(detail.vendorHandle ? { vendorHandle: detail.vendorHandle } : {}),
597
+ intendedRoles: [...detail.intendedRoles],
598
+ usedBy: detail.usedBy.map((company) => ({ ...company })),
599
+ features: [...detail.features]
600
+ };
601
+ });
602
+ const detailChecksum = computeLinkedInProductDetailChecksum(checkpoint.details);
603
+ if (detailChecksum !== checkpoint.detailChecksum) {
604
+ throw new LinkedInProductCollectorInvariantError("checksum_drift", "Detail checkpoint checksum changed.");
605
+ }
606
+ const productsUnavailable = checkpoint.details.filter((entry) => entry.detail.status === "unavailable").length;
607
+ return {
608
+ schemaVersion: ENRICHED_ARTIFACT_SCHEMA_VERSION,
609
+ kind: catalog.kind,
610
+ complete: true,
611
+ source: { ...catalog.source },
612
+ capture: {
613
+ ...catalog.capture,
614
+ terminalPage: { ...catalog.capture.terminalPage }
615
+ },
616
+ productCount: catalog.productCount,
617
+ orderChecksumAlgorithm: "sha256",
618
+ orderChecksum: catalog.orderChecksum,
619
+ detailCapture: {
620
+ startedAt: checkpoint.startedAt,
621
+ completedAt: checkpoint.completedAt,
622
+ productsAttempted: checkpoint.details.length,
623
+ productsComplete: checkpoint.details.length - productsUnavailable,
624
+ productsUnavailable
625
+ },
626
+ detailChecksumAlgorithm: "sha256",
627
+ detailChecksum,
628
+ products
629
+ };
630
+ }
631
+ function assertValidCheckpoint(checkpoint, catalog) {
632
+ checkpointSchema.parse(checkpoint);
633
+ if (checkpoint.source.queryHash !== catalog.source.queryHash ||
634
+ checkpoint.source.orderChecksum !== catalog.orderChecksum ||
635
+ checkpoint.source.productCount !== catalog.productCount) {
636
+ throw new LinkedInProductCollectorInvariantError("checkpoint_catalog_mismatch", "Detail checkpoint belongs to a different ranking artifact.");
637
+ }
638
+ assertIsoTimestamp(checkpoint.startedAt, "checkpoint.startedAt");
639
+ assertIsoTimestamp(checkpoint.updatedAt, "checkpoint.updatedAt");
640
+ if (checkpoint.updatedAt < checkpoint.startedAt) {
641
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint timestamps are out of order.");
642
+ }
643
+ if (checkpoint.tasksIssued < checkpoint.details.length) {
644
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint has more details than issued tasks.");
645
+ }
646
+ checkpoint.details.forEach((entry, index) => {
647
+ const product = catalog.products[index];
648
+ if (!product ||
649
+ entry.rank !== index + 1 ||
650
+ entry.rank !== product.rank ||
651
+ entry.linkedinUrl !== product.linkedinUrl ||
652
+ entry.linkedinSlug !== product.linkedinSlug) {
653
+ throw new LinkedInProductCollectorInvariantError("detail_order_drift", "Checkpoint details no longer match the ranked catalog order.");
654
+ }
655
+ assertIsoTimestamp(entry.detail.capturedAt, `checkpoint.details[${index}].capturedAt`);
656
+ });
657
+ if (checkpoint.nextRank !== checkpoint.details.length + 1) {
658
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint next rank does not follow accepted details.");
659
+ }
660
+ if (checkpoint.status === "complete") {
661
+ if (checkpoint.details.length !== catalog.productCount ||
662
+ checkpoint.nextRank !== catalog.productCount + 1 ||
663
+ !checkpoint.completedAt ||
664
+ !checkpoint.detailChecksum ||
665
+ checkpoint.lastStop !== undefined) {
666
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Complete detail checkpoint is missing terminal evidence.");
667
+ }
668
+ assertIsoTimestamp(checkpoint.completedAt, "checkpoint.completedAt");
669
+ if (checkpoint.completedAt !== checkpoint.updatedAt) {
670
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Complete checkpoint timestamps disagree.");
671
+ }
672
+ if (computeLinkedInProductDetailChecksum(checkpoint.details) !== checkpoint.detailChecksum) {
673
+ throw new LinkedInProductCollectorInvariantError("checksum_drift", "Checkpoint detail checksum changed.");
674
+ }
675
+ }
676
+ else if (checkpoint.completedAt !== undefined || checkpoint.detailChecksum !== undefined) {
677
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Incomplete checkpoint cannot contain completion evidence.");
678
+ }
679
+ if (checkpoint.lastStop) {
680
+ assertIsoTimestamp(checkpoint.lastStop.at, "checkpoint.lastStop.at");
681
+ if (checkpoint.lastStop.rank !== checkpoint.nextRank) {
682
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint stop evidence does not match the next rank.");
683
+ }
684
+ }
685
+ }
686
+ async function ensureParent(filePath) {
687
+ await mkdir(path.dirname(path.resolve(filePath)), { recursive: true });
688
+ }
689
+ async function writePrivateJsonAtomic(filePath, value) {
690
+ const resolved = path.resolve(filePath);
691
+ await ensureParent(resolved);
692
+ const temporary = `${resolved}.${process.pid}.${randomUUID()}.tmp`;
693
+ await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
694
+ await chmod(temporary, 0o600);
695
+ await rename(temporary, resolved);
696
+ await chmod(resolved, 0o600);
697
+ }
698
+ async function appendPrivateJsonLine(filePath, value) {
699
+ const resolved = path.resolve(filePath);
700
+ await ensureParent(resolved);
701
+ await appendFile(resolved, `${JSON.stringify(value)}\n`, { encoding: "utf8", mode: 0o600 });
702
+ await chmod(resolved, 0o600);
703
+ }
704
+ async function fileExists(filePath) {
705
+ try {
706
+ await access(path.resolve(filePath));
707
+ return true;
708
+ }
709
+ catch {
710
+ return false;
711
+ }
712
+ }
713
+ async function readCheckpoint(checkpointPath, catalog) {
714
+ try {
715
+ const parsed = checkpointSchema.parse(JSON.parse(await readFile(path.resolve(checkpointPath), "utf8")));
716
+ assertValidCheckpoint(parsed, catalog);
717
+ return parsed;
718
+ }
719
+ catch (error) {
720
+ if (error.code === "ENOENT")
721
+ return null;
722
+ throw error;
723
+ }
724
+ }
725
+ function rawResultFromStoredDetail(detail) {
726
+ if (detail.status === "unavailable") {
727
+ return { status: "unavailable", proof: detail.proof };
728
+ }
729
+ const { status: _status, capturedAt: _capturedAt, ...publicDetail } = detail;
730
+ return { status: "complete", detail: publicDetail };
731
+ }
732
+ function submissionFromRawDetail(record) {
733
+ if (record.result.status === "unavailable") {
734
+ return {
735
+ rank: record.rank,
736
+ currentUrl: record.currentUrl,
737
+ unavailable: {
738
+ code: "product_unavailable",
739
+ proof: record.result.proof
740
+ }
741
+ };
742
+ }
743
+ return {
744
+ rank: record.rank,
745
+ currentUrl: record.currentUrl,
746
+ detail: record.result.detail
747
+ };
748
+ }
749
+ async function assertRawEvidenceMatchesCheckpoint(rawJsonlPath, catalog, checkpoint) {
750
+ const raw = await readFile(path.resolve(rawJsonlPath), "utf8");
751
+ const lines = raw.split(/\r?\n/).filter((line) => line.trim() !== "");
752
+ let replay = createLinkedInProductDetailsCheckpoint(catalog, new Date(checkpoint.startedAt));
753
+ let acceptedDetails = 0;
754
+ for (const [index, line] of lines.entries()) {
755
+ let parsedJson;
756
+ try {
757
+ parsedJson = JSON.parse(line);
758
+ }
759
+ catch {
760
+ throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw detail JSONL line ${index + 1} is not valid JSON.`);
761
+ }
762
+ const record = rawEvidenceRecordSchema.parse(parsedJson);
763
+ if (record.orderChecksum !== catalog.orderChecksum) {
764
+ throw new LinkedInProductCollectorInvariantError("raw_catalog_mismatch", `Raw detail JSONL line ${index + 1} belongs to another ranking artifact.`);
765
+ }
766
+ assertIsoTimestamp(record.receivedAt, `raw detail line ${index + 1} receivedAt`);
767
+ if (record.type === "terminal_error") {
768
+ if (record.rank !== replay.nextRank) {
769
+ throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw terminal detail error on line ${index + 1} is out of sequence.`);
770
+ }
771
+ continue;
772
+ }
773
+ if (record.rank !== replay.nextRank) {
774
+ throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw detail on line ${index + 1} is duplicate or out of sequence.`);
775
+ }
776
+ const applied = applyLinkedInProductDetailSubmission(catalog, replay, submissionFromRawDetail(record), new Date(record.receivedAt));
777
+ replay = applied.checkpoint;
778
+ acceptedDetails += 1;
779
+ }
780
+ if (acceptedDetails !== checkpoint.details.length) {
781
+ throw new LinkedInProductCollectorInvariantError("missing_raw_evidence", `Raw detail JSONL proves ${acceptedDetails} products, but checkpoint claims ${checkpoint.details.length}.`);
782
+ }
783
+ if (canonicalJson(replay.details) !== canonicalJson(checkpoint.details)) {
784
+ throw new LinkedInProductCollectorInvariantError("raw_detail_mismatch", "Raw detail JSONL does not reproduce the checkpoint.");
785
+ }
786
+ if ((checkpoint.status === "complete") !== (replay.status === "complete")) {
787
+ throw new LinkedInProductCollectorInvariantError("raw_state_mismatch", "Raw detail JSONL completion state does not reproduce the checkpoint.");
788
+ }
789
+ }
790
+ function sanitizeTerminalErrorUrl(currentUrl) {
791
+ let parsed;
792
+ try {
793
+ parsed = new URL(currentUrl);
794
+ }
795
+ catch {
796
+ return "invalid-url-redacted";
797
+ }
798
+ if (parsed.protocol !== "https:" ||
799
+ parsed.port !== "" ||
800
+ parsed.username !== "" ||
801
+ parsed.password !== "" ||
802
+ !["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
803
+ return "external-url-redacted";
804
+ }
805
+ return `https://www.linkedin.com${parsed.pathname}`;
806
+ }
807
+ function terminalStopMessage(code) {
808
+ switch (code) {
809
+ case "auth_required":
810
+ return "The signed-in LinkedIn session is no longer available.";
811
+ case "challenge":
812
+ return "LinkedIn displayed a checkpoint or challenge.";
813
+ case "rate_limited":
814
+ return "LinkedIn rate-limited product detail collection.";
815
+ case "interrupted":
816
+ return "The browser worker was interrupted.";
817
+ case "unexpected_page":
818
+ return "The browser reached an unexpected LinkedIn page.";
819
+ }
820
+ }
821
+ async function readRelayJson(req) {
822
+ const chunks = [];
823
+ let size = 0;
824
+ for await (const chunk of req) {
825
+ const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
826
+ size += buffer.length;
827
+ if (size > MAX_RELAY_BODY_BYTES) {
828
+ throw new LinkedInProductCollectorInvariantError("body_too_large", "Relay request body is too large.");
829
+ }
830
+ chunks.push(buffer);
831
+ }
832
+ if (chunks.length === 0)
833
+ throw new Error("Relay request body is empty.");
834
+ return JSON.parse(Buffer.concat(chunks).toString("utf8"));
835
+ }
836
+ function writeRelayJson(res, statusCode, value, origin) {
837
+ const body = `${JSON.stringify(value)}\n`;
838
+ res.writeHead(statusCode, {
839
+ "Content-Type": "application/json; charset=utf-8",
840
+ "Content-Length": Buffer.byteLength(body),
841
+ "Cache-Control": "no-store",
842
+ "Access-Control-Allow-Origin": origin === "https://www.linkedin.com" ? origin : "https://www.linkedin.com",
843
+ "Access-Control-Allow-Private-Network": "true",
844
+ Vary: "Origin"
845
+ });
846
+ res.end(body);
847
+ }
848
+ function isAllowedRelayOrigin(req) {
849
+ const origin = req.headers.origin;
850
+ return origin === undefined || origin === "https://www.linkedin.com";
851
+ }
852
+ function closeServer(server) {
853
+ return new Promise((resolve) => server.close(() => resolve()));
854
+ }
855
+ function detailCoverage(checkpoint) {
856
+ const unavailable = checkpoint.details.filter((entry) => entry.detail.status === "unavailable").length;
857
+ return { complete: checkpoint.details.length - unavailable, unavailable };
858
+ }
859
+ export async function collectLinkedInProductDetailsViaBrowserRelay(options) {
860
+ const catalog = await readLinkedInProductRankingArtifact(options.catalogPath);
861
+ const resolvedPaths = [
862
+ options.catalogPath,
863
+ options.checkpointPath,
864
+ options.rawJsonlPath,
865
+ options.outPath
866
+ ].map((value) => path.resolve(value));
867
+ if (new Set(resolvedPaths).size !== resolvedPaths.length) {
868
+ throw new LinkedInProductCollectorInvariantError("artifact_path_conflict", "Catalog, checkpoint, raw detail JSONL, and enriched output paths must be distinct.");
869
+ }
870
+ const existingCheckpoint = await readCheckpoint(options.checkpointPath, catalog);
871
+ const [rawExists, outExists] = await Promise.all([
872
+ fileExists(options.rawJsonlPath),
873
+ fileExists(options.outPath)
874
+ ]);
875
+ if (!existingCheckpoint && (rawExists || outExists)) {
876
+ throw new LinkedInProductCollectorInvariantError("orphaned_artifact", "A new detail collection requires unused raw and output paths.");
877
+ }
878
+ if (existingCheckpoint && existingCheckpoint.details.length > 0 && !rawExists) {
879
+ throw new LinkedInProductCollectorInvariantError("missing_raw_evidence", "Detail checkpoint has accepted products but raw evidence is missing.");
880
+ }
881
+ if (existingCheckpoint && rawExists) {
882
+ await assertRawEvidenceMatchesCheckpoint(options.rawJsonlPath, catalog, existingCheckpoint);
883
+ }
884
+ if (existingCheckpoint && existingCheckpoint.status !== "complete" && outExists) {
885
+ throw new LinkedInProductCollectorInvariantError("stale_complete_artifact", "Incomplete detail checkpoint cannot share an existing enriched output path.");
886
+ }
887
+ let checkpoint = existingCheckpoint ?? createLinkedInProductDetailsCheckpoint(catalog);
888
+ await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
889
+ const makeResult = (status, reason) => {
890
+ const coverage = detailCoverage(checkpoint);
891
+ return {
892
+ status,
893
+ complete: status === "complete",
894
+ ...(reason ? { reason } : {}),
895
+ catalogPath: path.resolve(options.catalogPath),
896
+ checkpointPath: path.resolve(options.checkpointPath),
897
+ rawJsonlPath: path.resolve(options.rawJsonlPath),
898
+ outPath: status === "complete" ? path.resolve(options.outPath) : null,
899
+ orderChecksum: catalog.orderChecksum,
900
+ ...(checkpoint.detailChecksum ? { detailChecksum: checkpoint.detailChecksum } : {}),
901
+ productsTotal: catalog.productCount,
902
+ detailsCollected: checkpoint.details.length,
903
+ detailsComplete: coverage.complete,
904
+ productsUnavailable: coverage.unavailable,
905
+ nextRank: checkpoint.nextRank
906
+ };
907
+ };
908
+ if (checkpoint.status === "complete") {
909
+ await writePrivateJsonAtomic(options.outPath, buildLinkedInProductDetailsArtifact(catalog, checkpoint));
910
+ return makeResult("complete");
911
+ }
912
+ let uniqueTasksIssuedThisRun = 0;
913
+ let lastIssuedRank = null;
914
+ let settled = false;
915
+ let acceptingRequests = true;
916
+ let idleTimer;
917
+ let incompleteStopPromise;
918
+ let resolveResult;
919
+ let rejectResult;
920
+ const resultPromise = new Promise((resolve, reject) => {
921
+ resolveResult = resolve;
922
+ rejectResult = reject;
923
+ });
924
+ let requestQueue = Promise.resolve();
925
+ const enqueueExclusive = (operation) => {
926
+ const queued = requestQueue.then(operation, operation);
927
+ requestQueue = queued.catch(() => undefined);
928
+ return queued;
929
+ };
930
+ const server = createServer((req, res) => {
931
+ if (!acceptingRequests) {
932
+ writeRelayJson(res, 410, { status: "closed" }, req.headers.origin);
933
+ return;
934
+ }
935
+ const execute = async () => {
936
+ try {
937
+ await handleRequest(req, res);
938
+ }
939
+ catch (error) {
940
+ const browserInputError = error instanceof LinkedInProductCollectorInvariantError ||
941
+ error instanceof z.ZodError ||
942
+ error instanceof SyntaxError;
943
+ if (!res.headersSent) {
944
+ writeRelayJson(res, browserInputError ? 409 : 500, { status: "rejected", error: error instanceof Error ? error.message : String(error) }, req.headers.origin);
945
+ }
946
+ if (!browserInputError)
947
+ await abortWithError(error);
948
+ }
949
+ };
950
+ void enqueueExclusive(execute);
951
+ });
952
+ const finish = async (status, reason) => {
953
+ if (settled)
954
+ return;
955
+ settled = true;
956
+ acceptingRequests = false;
957
+ if (idleTimer)
958
+ clearTimeout(idleTimer);
959
+ process.off("SIGINT", handleSigint);
960
+ process.off("SIGTERM", handleSigterm);
961
+ await closeServer(server);
962
+ resolveResult?.(makeResult(status, reason));
963
+ };
964
+ const abortWithError = async (error) => {
965
+ if (settled)
966
+ return;
967
+ settled = true;
968
+ acceptingRequests = false;
969
+ if (idleTimer)
970
+ clearTimeout(idleTimer);
971
+ process.off("SIGINT", handleSigint);
972
+ process.off("SIGTERM", handleSigterm);
973
+ await closeServer(server);
974
+ rejectResult?.(error);
975
+ };
976
+ const stopIncomplete = (reason, message) => {
977
+ acceptingRequests = false;
978
+ if (incompleteStopPromise)
979
+ return incompleteStopPromise;
980
+ incompleteStopPromise = (async () => {
981
+ if (settled)
982
+ return;
983
+ const now = new Date().toISOString();
984
+ checkpoint = {
985
+ ...checkpoint,
986
+ updatedAt: now,
987
+ lastStop: {
988
+ reason,
989
+ at: now,
990
+ rank: checkpoint.nextRank,
991
+ ...(message ? { message } : {})
992
+ }
993
+ };
994
+ await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
995
+ await finish("incomplete", reason);
996
+ })();
997
+ return incompleteStopPromise;
998
+ };
999
+ const requestIncompleteStop = (reason, message) => {
1000
+ acceptingRequests = false;
1001
+ return enqueueExclusive(async () => {
1002
+ if (settled)
1003
+ return;
1004
+ if (checkpoint.status === "complete") {
1005
+ await finish("complete");
1006
+ return;
1007
+ }
1008
+ await stopIncomplete(reason, message);
1009
+ });
1010
+ };
1011
+ function resetIdleTimer() {
1012
+ if (idleTimer)
1013
+ clearTimeout(idleTimer);
1014
+ idleTimer = setTimeout(() => {
1015
+ void requestIncompleteStop("idle_timeout", "No browser detail relay activity arrived before the local timeout.").catch((error) => rejectResult?.(error));
1016
+ }, options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS);
1017
+ idleTimer.unref();
1018
+ }
1019
+ async function handleUnexpectedPage(submission, error, res, origin) {
1020
+ acceptingRequests = false;
1021
+ const receivedAt = new Date().toISOString();
1022
+ await appendPrivateJsonLine(options.rawJsonlPath, {
1023
+ type: "terminal_error",
1024
+ orderChecksum: catalog.orderChecksum,
1025
+ rank: submission.rank,
1026
+ currentUrl: sanitizeTerminalErrorUrl(submission.currentUrl),
1027
+ error: { code: "unexpected_page" },
1028
+ receivedAt
1029
+ });
1030
+ writeRelayJson(res, 202, { status: "checkpointed", complete: false }, origin);
1031
+ res.once("finish", () => {
1032
+ void requestIncompleteStop("unexpected_page", error.message).catch((failure) => rejectResult?.(failure));
1033
+ });
1034
+ }
1035
+ async function handleRequest(req, res) {
1036
+ if (!acceptingRequests) {
1037
+ writeRelayJson(res, 410, { status: "closed" }, req.headers.origin);
1038
+ return;
1039
+ }
1040
+ if (!isAllowedRelayOrigin(req)) {
1041
+ writeRelayJson(res, 403, { status: "rejected", error: "Relay origin is not allowed." });
1042
+ return;
1043
+ }
1044
+ resetIdleTimer();
1045
+ const requestUrl = new URL(req.url ?? "/", "http://127.0.0.1");
1046
+ if (req.method === "OPTIONS") {
1047
+ res.writeHead(204, {
1048
+ "Access-Control-Allow-Origin": "https://www.linkedin.com",
1049
+ "Access-Control-Allow-Methods": "GET, POST, OPTIONS",
1050
+ "Access-Control-Allow-Headers": "Content-Type",
1051
+ "Access-Control-Allow-Private-Network": "true",
1052
+ "Access-Control-Max-Age": "600",
1053
+ Vary: "Origin"
1054
+ });
1055
+ res.end();
1056
+ return;
1057
+ }
1058
+ if (req.method === "GET" && requestUrl.pathname === "/health") {
1059
+ writeRelayJson(res, 200, { status: "ok", orderChecksum: catalog.orderChecksum }, req.headers.origin);
1060
+ return;
1061
+ }
1062
+ if (req.method === "GET" && requestUrl.pathname === "/task") {
1063
+ if (options.maxRequests !== undefined &&
1064
+ uniqueTasksIssuedThisRun >= options.maxRequests &&
1065
+ lastIssuedRank !== checkpoint.nextRank) {
1066
+ acceptingRequests = false;
1067
+ writeRelayJson(res, 200, { status: "incomplete", reason: "request_limit" }, req.headers.origin);
1068
+ res.once("finish", () => void requestIncompleteStop("request_limit"));
1069
+ return;
1070
+ }
1071
+ const product = catalog.products[checkpoint.nextRank - 1];
1072
+ if (!product) {
1073
+ throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Next detail product is missing.");
1074
+ }
1075
+ if (lastIssuedRank !== checkpoint.nextRank) {
1076
+ uniqueTasksIssuedThisRun += 1;
1077
+ lastIssuedRank = checkpoint.nextRank;
1078
+ checkpoint = {
1079
+ ...checkpoint,
1080
+ tasksIssued: checkpoint.tasksIssued + 1,
1081
+ updatedAt: new Date().toISOString()
1082
+ };
1083
+ await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
1084
+ }
1085
+ writeRelayJson(res, 200, {
1086
+ status: "task",
1087
+ taskType: "linkedin_product_detail",
1088
+ orderChecksum: catalog.orderChecksum,
1089
+ rank: product.rank,
1090
+ productCount: catalog.productCount,
1091
+ currentUrl: product.linkedinUrl,
1092
+ productName: product.name,
1093
+ detailsAccepted: checkpoint.details.length
1094
+ }, req.headers.origin);
1095
+ return;
1096
+ }
1097
+ if (req.method === "POST" && requestUrl.pathname === "/detail") {
1098
+ const submission = linkedInProductDetailSubmissionSchema.parse(await readRelayJson(req));
1099
+ if (submission.rank !== checkpoint.nextRank) {
1100
+ throw new LinkedInProductCollectorInvariantError("out_of_order_detail", `Expected product rank ${checkpoint.nextRank}, received ${submission.rank}.`);
1101
+ }
1102
+ if (lastIssuedRank !== submission.rank) {
1103
+ throw new LinkedInProductCollectorInvariantError("unleased_detail", "Poll GET /task before posting the expected product detail.");
1104
+ }
1105
+ if ("error" in submission) {
1106
+ acceptingRequests = false;
1107
+ const receivedAt = new Date().toISOString();
1108
+ await appendPrivateJsonLine(options.rawJsonlPath, {
1109
+ type: "terminal_error",
1110
+ orderChecksum: catalog.orderChecksum,
1111
+ rank: submission.rank,
1112
+ currentUrl: sanitizeTerminalErrorUrl(submission.currentUrl),
1113
+ error: { code: submission.error.code },
1114
+ receivedAt
1115
+ });
1116
+ writeRelayJson(res, 202, { status: "checkpointed", complete: false }, req.headers.origin);
1117
+ res.once("finish", () => {
1118
+ void requestIncompleteStop(submission.error.code, submission.error.message ?? terminalStopMessage(submission.error.code)).catch((error) => rejectResult?.(error));
1119
+ });
1120
+ return;
1121
+ }
1122
+ let applied;
1123
+ try {
1124
+ applied = applyLinkedInProductDetailSubmission(catalog, checkpoint, submission);
1125
+ }
1126
+ catch (error) {
1127
+ if (error instanceof LinkedInProductCollectorInvariantError && error.code === "unexpected_page") {
1128
+ await handleUnexpectedPage(submission, error, res, req.headers.origin);
1129
+ return;
1130
+ }
1131
+ throw error;
1132
+ }
1133
+ await appendPrivateJsonLine(options.rawJsonlPath, {
1134
+ type: "detail",
1135
+ orderChecksum: catalog.orderChecksum,
1136
+ rank: submission.rank,
1137
+ currentUrl: applied.accepted.linkedinUrl,
1138
+ result: rawResultFromStoredDetail(applied.accepted.detail),
1139
+ receivedAt: applied.accepted.detail.capturedAt
1140
+ });
1141
+ checkpoint = applied.checkpoint;
1142
+ await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
1143
+ if (checkpoint.status === "complete") {
1144
+ await writePrivateJsonAtomic(options.outPath, buildLinkedInProductDetailsArtifact(catalog, checkpoint));
1145
+ acceptingRequests = false;
1146
+ }
1147
+ lastIssuedRank = null;
1148
+ writeRelayJson(res, 202, {
1149
+ status: checkpoint.status === "complete" ? "complete" : "accepted",
1150
+ complete: checkpoint.status === "complete",
1151
+ rank: submission.rank,
1152
+ detailStatus: applied.accepted.detail.status,
1153
+ detailsAccepted: checkpoint.details.length,
1154
+ nextRank: checkpoint.nextRank
1155
+ }, req.headers.origin);
1156
+ if (checkpoint.status === "complete") {
1157
+ res.once("finish", () => void finish("complete"));
1158
+ }
1159
+ else if (options.maxRequests !== undefined &&
1160
+ uniqueTasksIssuedThisRun >= options.maxRequests) {
1161
+ acceptingRequests = false;
1162
+ res.once("finish", () => void requestIncompleteStop("request_limit"));
1163
+ }
1164
+ return;
1165
+ }
1166
+ writeRelayJson(res, 404, { status: "not_found" }, req.headers.origin);
1167
+ }
1168
+ const handleSigint = () => {
1169
+ void requestIncompleteStop("interrupted", "Detail collector received SIGINT.").catch((error) => rejectResult?.(error));
1170
+ };
1171
+ const handleSigterm = () => {
1172
+ void requestIncompleteStop("interrupted", "Detail collector received SIGTERM.").catch((error) => rejectResult?.(error));
1173
+ };
1174
+ process.once("SIGINT", handleSigint);
1175
+ process.once("SIGTERM", handleSigterm);
1176
+ try {
1177
+ await new Promise((resolve, reject) => {
1178
+ server.once("error", reject);
1179
+ server.listen(options.relayPort, "127.0.0.1", () => resolve());
1180
+ });
1181
+ resetIdleTimer();
1182
+ const address = server.address();
1183
+ if (!address || typeof address === "string") {
1184
+ throw new Error("LinkedIn product detail relay did not bind to TCP.");
1185
+ }
1186
+ options.onListening?.({
1187
+ host: "127.0.0.1",
1188
+ port: address.port,
1189
+ taskUrl: `http://127.0.0.1:${address.port}/task`,
1190
+ detailUrl: `http://127.0.0.1:${address.port}/detail`
1191
+ });
1192
+ return await resultPromise;
1193
+ }
1194
+ catch (error) {
1195
+ if (idleTimer)
1196
+ clearTimeout(idleTimer);
1197
+ process.off("SIGINT", handleSigint);
1198
+ process.off("SIGTERM", handleSigterm);
1199
+ if (server.listening)
1200
+ await closeServer(server);
1201
+ throw error;
1202
+ }
1203
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "salesprompter-cli",
3
- "version": "0.1.68",
3
+ "version": "0.1.69",
4
4
  "description": "Sales workflow CLI for guided lead generation, enrichment, scoring, and sync.",
5
5
  "author": "Daniel Sinewe <hello@danielsinewe.com>",
6
6
  "type": "module",