salesprompter-cli 0.1.66 → 0.1.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/cli.js +41 -1
- package/dist/linkedin-product-search.js +1081 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -125,6 +125,19 @@ salesprompter packs:list
|
|
|
125
125
|
salesprompter --help
|
|
126
126
|
```
|
|
127
127
|
|
|
128
|
+
Collect an exact signed-in LinkedIn product-search order locally, without uploading it:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
salesprompter products:collect \
|
|
132
|
+
--query-url "$LINKEDIN_PRODUCTS_URL" \
|
|
133
|
+
--checkpoint ./data/products.checkpoint.json \
|
|
134
|
+
--raw-jsonl ./data/products.raw.jsonl \
|
|
135
|
+
--out ./data/products.complete.json \
|
|
136
|
+
--relay-port 43117
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Use a signed-in in-app-browser worker with the loopback `GET /task` and `POST /page` protocol. Interrupted, bounded, challenged, or rate-limited runs remain checkpointed and do not create the complete artifact.
|
|
140
|
+
|
|
128
141
|
Download stored workspace leads without starting a new Sales Navigator scrape:
|
|
129
142
|
|
|
130
143
|
```bash
|
package/dist/cli.js
CHANGED
|
@@ -28,6 +28,7 @@ import { buildHistoricalVendorIcp, buildVendorIcp } from "./icp-templates.js";
|
|
|
28
28
|
import { InstantlySyncProvider } from "./instantly.js";
|
|
29
29
|
import { backfillLinkedInCompanies } from "./linkedin-companies.js";
|
|
30
30
|
import { parseLinkedInCompanyPage } from "./linkedin-companies.js";
|
|
31
|
+
import { collectLinkedInProductsViaBrowserRelay } from "./linkedin-product-search.js";
|
|
31
32
|
import { crawlLinkedInProductCategory } from "./linkedin-products.js";
|
|
32
33
|
import { claimLinkedInSessionCookieForCli, claimValidatedSalesNavigatorSessionCookieForCli, createLinkedInSessionSupabaseClient, recordLinkedInSessionCookieAudit, resolveConfiguredEnvValue } from "./linkedin-session.js";
|
|
33
34
|
import { buildLeadlistsFunnelQueries } from "./leadlists-funnel.js";
|
|
@@ -598,7 +599,7 @@ const cliPacks = [
|
|
|
598
599
|
slug: "research",
|
|
599
600
|
title: "Research",
|
|
600
601
|
summary: "Scrape markets and enrich companies before outreach.",
|
|
601
|
-
commands: ["market:scrape", "companies:enrich"],
|
|
602
|
+
commands: ["products:collect", "market:scrape", "companies:enrich"],
|
|
602
603
|
installStatus: "included"
|
|
603
604
|
},
|
|
604
605
|
{
|
|
@@ -648,6 +649,7 @@ const helpAliasByCommandName = new Map([
|
|
|
648
649
|
["linkedin-companies:backfill", "companies:enrich"],
|
|
649
650
|
["linkedin-companies:scrape-local", "companies:scrape-linkedin"],
|
|
650
651
|
["dealroom-companies:scrape-local", "companies:scrape-dealroom"],
|
|
652
|
+
["linkedin-products:collect", "products:collect"],
|
|
651
653
|
["linkedin-products:scrape", "market:scrape"],
|
|
652
654
|
["salesnav:from-product-category", "leads:discover"],
|
|
653
655
|
["salesnav:people:collect", "leads:collect"],
|
|
@@ -698,6 +700,7 @@ const helpVisibleCommandNames = new Set([
|
|
|
698
700
|
"linkedin-companies:backfill",
|
|
699
701
|
"linkedin-companies:scrape-local",
|
|
700
702
|
"dealroom-companies:scrape-local",
|
|
703
|
+
"linkedin-products:collect",
|
|
701
704
|
"linkedin-products:scrape",
|
|
702
705
|
"salesnav:from-product-category",
|
|
703
706
|
"salesnav:people:collect",
|
|
@@ -15183,6 +15186,43 @@ program
|
|
|
15183
15186
|
profile,
|
|
15184
15187
|
});
|
|
15185
15188
|
});
|
|
15189
|
+
program
|
|
15190
|
+
.command("linkedin-products:collect")
|
|
15191
|
+
.alias("products:collect")
|
|
15192
|
+
.description("Collect an exact LinkedIn product-search order locally through a loopback browser relay.")
|
|
15193
|
+
.requiredOption("--query-url <url>", "Exact LinkedIn /search/results/products/ query URL")
|
|
15194
|
+
.requiredOption("--checkpoint <path>", "Private resumable checkpoint JSON path")
|
|
15195
|
+
.requiredOption("--raw-jsonl <path>", "Private append-only normalized page evidence path")
|
|
15196
|
+
.requiredOption("--out <path>", "Complete public-fields-only ranked artifact path")
|
|
15197
|
+
.requiredOption("--relay-port <number>", "Loopback relay port for GET /task and POST /page")
|
|
15198
|
+
.option("--max-pages <number>", "Optional highest page number to accept in this run")
|
|
15199
|
+
.option("--max-requests <number>", "Optional number of unique browser tasks to issue in this run")
|
|
15200
|
+
.action(async (options) => {
|
|
15201
|
+
const relayPort = z.coerce.number().int().min(1).max(65_535).parse(options.relayPort);
|
|
15202
|
+
const maxPages = options.maxPages === undefined
|
|
15203
|
+
? undefined
|
|
15204
|
+
: z.coerce.number().int().min(1).max(100_000).parse(options.maxPages);
|
|
15205
|
+
const maxRequests = options.maxRequests === undefined
|
|
15206
|
+
? undefined
|
|
15207
|
+
: z.coerce.number().int().min(1).max(100_000).parse(options.maxRequests);
|
|
15208
|
+
const idleTimeoutMs = process.env.SALESPROMPTER_LINKEDIN_PRODUCTS_RELAY_IDLE_TIMEOUT_MS
|
|
15209
|
+
? z.coerce.number().int().min(100).max(86_400_000).parse(process.env.SALESPROMPTER_LINKEDIN_PRODUCTS_RELAY_IDLE_TIMEOUT_MS)
|
|
15210
|
+
: undefined;
|
|
15211
|
+
const result = await collectLinkedInProductsViaBrowserRelay({
|
|
15212
|
+
queryUrl: String(options.queryUrl),
|
|
15213
|
+
checkpointPath: path.resolve(String(options.checkpoint)),
|
|
15214
|
+
rawJsonlPath: path.resolve(String(options.rawJsonl)),
|
|
15215
|
+
outPath: path.resolve(String(options.out)),
|
|
15216
|
+
relayPort,
|
|
15217
|
+
maxPages,
|
|
15218
|
+
maxRequests,
|
|
15219
|
+
idleTimeoutMs,
|
|
15220
|
+
onListening: ({ taskUrl, pageUrl }) => {
|
|
15221
|
+
writeProgress(`LinkedIn product relay ready: GET ${taskUrl} and POST ${pageUrl}`);
|
|
15222
|
+
}
|
|
15223
|
+
});
|
|
15224
|
+
printOutput(result);
|
|
15225
|
+
});
|
|
15186
15226
|
program
|
|
15187
15227
|
.command("linkedin-products:scrape")
|
|
15188
15228
|
.alias("market:scrape")
|
|
@@ -0,0 +1,1081 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { access, appendFile, chmod, mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
const CHECKPOINT_SCHEMA_VERSION = 1;
|
|
7
|
+
const ARTIFACT_SCHEMA_VERSION = 1;
|
|
8
|
+
const MAX_RELAY_BODY_BYTES = 5 * 1024 * 1024;
|
|
9
|
+
const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
|
|
10
|
+
const nonEmptyText = (max) => z.string().trim().min(1).max(max);
|
|
11
|
+
const optionalPublicUrl = z.string().trim().url().max(4_096).optional();
|
|
12
|
+
export const visibleLinkedInProductCardSchema = z
|
|
13
|
+
.object({
|
|
14
|
+
identity: z.string().trim().min(1).max(256).optional(),
|
|
15
|
+
name: nonEmptyText(500),
|
|
16
|
+
linkedinUrl: nonEmptyText(4_096),
|
|
17
|
+
imageUrl: optionalPublicUrl,
|
|
18
|
+
vendor: nonEmptyText(500).optional(),
|
|
19
|
+
category: nonEmptyText(500).optional(),
|
|
20
|
+
description: nonEmptyText(10_000).optional()
|
|
21
|
+
})
|
|
22
|
+
.strict();
|
|
23
|
+
const pageSuccessSchema = z
|
|
24
|
+
.object({
|
|
25
|
+
pageNumber: z.number().int().min(1),
|
|
26
|
+
currentUrl: nonEmptyText(8_192),
|
|
27
|
+
reportedTotal: z.number().int().min(0).nullable().optional(),
|
|
28
|
+
hasNext: z.boolean(),
|
|
29
|
+
items: z.array(visibleLinkedInProductCardSchema).max(1_000)
|
|
30
|
+
})
|
|
31
|
+
.strict();
|
|
32
|
+
const terminalErrorSchema = z
|
|
33
|
+
.object({
|
|
34
|
+
pageNumber: z.number().int().min(1),
|
|
35
|
+
currentUrl: nonEmptyText(8_192),
|
|
36
|
+
error: z
|
|
37
|
+
.object({
|
|
38
|
+
code: z.enum([
|
|
39
|
+
"auth_required",
|
|
40
|
+
"challenge",
|
|
41
|
+
"rate_limited",
|
|
42
|
+
"interrupted",
|
|
43
|
+
"unexpected_page"
|
|
44
|
+
]),
|
|
45
|
+
message: z.string().trim().max(1_000).optional()
|
|
46
|
+
})
|
|
47
|
+
.strict()
|
|
48
|
+
})
|
|
49
|
+
.strict();
|
|
50
|
+
export const linkedInProductPageSubmissionSchema = z.union([pageSuccessSchema, terminalErrorSchema]);
|
|
51
|
+
const rawVisibleProductCardSchema = visibleLinkedInProductCardSchema
|
|
52
|
+
.extend({ linkedinSlug: nonEmptyText(500) })
|
|
53
|
+
.strict();
|
|
54
|
+
const rawPageEvidenceSchema = z
|
|
55
|
+
.object({
|
|
56
|
+
type: z.literal("page"),
|
|
57
|
+
queryHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
58
|
+
pageNumber: z.number().int().min(1),
|
|
59
|
+
currentUrl: nonEmptyText(8_192),
|
|
60
|
+
reportedTotal: z.number().int().min(0).nullable(),
|
|
61
|
+
hasNext: z.boolean(),
|
|
62
|
+
items: z.array(rawVisibleProductCardSchema).max(1_000),
|
|
63
|
+
visibleItemCount: z.number().int().min(0),
|
|
64
|
+
newProductCount: z.number().int().min(0),
|
|
65
|
+
duplicateCount: z.number().int().min(0),
|
|
66
|
+
receivedAt: nonEmptyText(100)
|
|
67
|
+
})
|
|
68
|
+
.strict();
|
|
69
|
+
const rawTerminalEvidenceSchema = z
|
|
70
|
+
.object({
|
|
71
|
+
type: z.literal("terminal_error"),
|
|
72
|
+
queryHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
73
|
+
pageNumber: z.number().int().min(1),
|
|
74
|
+
currentUrl: nonEmptyText(8_192),
|
|
75
|
+
error: z.object({ code: terminalErrorSchema.shape.error.shape.code }).strict(),
|
|
76
|
+
receivedAt: nonEmptyText(100)
|
|
77
|
+
})
|
|
78
|
+
.strict();
|
|
79
|
+
const rawEvidenceRecordSchema = z.union([rawPageEvidenceSchema, rawTerminalEvidenceSchema]);
|
|
80
|
+
export class LinkedInProductCollectorInvariantError extends Error {
|
|
81
|
+
code;
|
|
82
|
+
constructor(code, message) {
|
|
83
|
+
super(message);
|
|
84
|
+
this.code = code;
|
|
85
|
+
this.name = "LinkedInProductCollectorInvariantError";
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
function sha256(value) {
|
|
89
|
+
return createHash("sha256").update(value, "utf8").digest("hex");
|
|
90
|
+
}
|
|
91
|
+
function cleanText(value) {
|
|
92
|
+
const cleaned = value?.replace(/\s+/g, " ").trim();
|
|
93
|
+
return cleaned || undefined;
|
|
94
|
+
}
|
|
95
|
+
function canonicalizePublicUrl(value) {
|
|
96
|
+
if (!value)
|
|
97
|
+
return undefined;
|
|
98
|
+
const parsed = new URL(value);
|
|
99
|
+
if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
|
|
100
|
+
throw new LinkedInProductCollectorInvariantError("invalid_public_url", `Unsupported public URL: ${value}`);
|
|
101
|
+
}
|
|
102
|
+
parsed.username = "";
|
|
103
|
+
parsed.password = "";
|
|
104
|
+
parsed.hash = "";
|
|
105
|
+
return parsed.toString();
|
|
106
|
+
}
|
|
107
|
+
export function normalizeLinkedInProductSearchUrl(input) {
|
|
108
|
+
let parsed;
|
|
109
|
+
try {
|
|
110
|
+
parsed = new URL(input);
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
throw new LinkedInProductCollectorInvariantError("invalid_query_url", "The query URL is not a valid URL.");
|
|
114
|
+
}
|
|
115
|
+
if (parsed.protocol !== "https:" ||
|
|
116
|
+
parsed.port !== "" ||
|
|
117
|
+
parsed.username !== "" ||
|
|
118
|
+
parsed.password !== "" ||
|
|
119
|
+
!["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
|
|
120
|
+
throw new LinkedInProductCollectorInvariantError("invalid_query_url", "The query URL must use https://www.linkedin.com.");
|
|
121
|
+
}
|
|
122
|
+
if (parsed.pathname !== "/search/results/products" && parsed.pathname !== "/search/results/products/") {
|
|
123
|
+
throw new LinkedInProductCollectorInvariantError("invalid_query_url", "The query URL must point to /search/results/products/.");
|
|
124
|
+
}
|
|
125
|
+
const suppliedPages = parsed.searchParams.getAll("page");
|
|
126
|
+
const suppliedPage = suppliedPages[0];
|
|
127
|
+
if (suppliedPages.length > 1 || (suppliedPage && suppliedPage !== "1")) {
|
|
128
|
+
throw new LinkedInProductCollectorInvariantError("invalid_query_url", "An exhaustive collection must begin on page 1; remove the page parameter.");
|
|
129
|
+
}
|
|
130
|
+
parsed.protocol = "https:";
|
|
131
|
+
parsed.hostname = "www.linkedin.com";
|
|
132
|
+
parsed.port = "";
|
|
133
|
+
parsed.username = "";
|
|
134
|
+
parsed.password = "";
|
|
135
|
+
parsed.pathname = "/search/results/products/";
|
|
136
|
+
parsed.hash = "";
|
|
137
|
+
parsed.searchParams.delete("page");
|
|
138
|
+
return parsed.toString();
|
|
139
|
+
}
|
|
140
|
+
export function buildLinkedInProductSearchPageUrl(queryUrl, pageNumber) {
|
|
141
|
+
if (!Number.isInteger(pageNumber) || pageNumber < 1) {
|
|
142
|
+
throw new LinkedInProductCollectorInvariantError("invalid_page", "Page numbers must be positive integers.");
|
|
143
|
+
}
|
|
144
|
+
const parsed = new URL(normalizeLinkedInProductSearchUrl(queryUrl));
|
|
145
|
+
if (pageNumber > 1)
|
|
146
|
+
parsed.searchParams.set("page", String(pageNumber));
|
|
147
|
+
return parsed.toString();
|
|
148
|
+
}
|
|
149
|
+
function assertExpectedLinkedInSearchPage(currentUrl, queryUrl, pageNumber) {
|
|
150
|
+
let actual;
|
|
151
|
+
try {
|
|
152
|
+
actual = new URL(currentUrl);
|
|
153
|
+
}
|
|
154
|
+
catch {
|
|
155
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported an invalid page URL.");
|
|
156
|
+
}
|
|
157
|
+
const source = new URL(normalizeLinkedInProductSearchUrl(queryUrl));
|
|
158
|
+
if (actual.protocol !== "https:" ||
|
|
159
|
+
actual.port !== "" ||
|
|
160
|
+
actual.username !== "" ||
|
|
161
|
+
actual.password !== "" ||
|
|
162
|
+
!["linkedin.com", "www.linkedin.com"].includes(actual.hostname.toLowerCase()) ||
|
|
163
|
+
(actual.pathname !== "/search/results/products" && actual.pathname !== "/search/results/products/")) {
|
|
164
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser is not on the expected LinkedIn product-search page.");
|
|
165
|
+
}
|
|
166
|
+
for (const key of new Set(source.searchParams.keys())) {
|
|
167
|
+
const expectedValues = source.searchParams.getAll(key);
|
|
168
|
+
const actualValues = actual.searchParams.getAll(key);
|
|
169
|
+
if (expectedValues.length !== actualValues.length || expectedValues.some((value, index) => value !== actualValues[index])) {
|
|
170
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", `The browser page changed the original ${key} query filter.`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
const sourceKeys = new Set(source.searchParams.keys());
|
|
174
|
+
for (const key of new Set(actual.searchParams.keys())) {
|
|
175
|
+
if (!sourceKeys.has(key) && key !== "page" && key !== "spellCorrectionEnabled") {
|
|
176
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", `The browser page added the unapproved ${key} query parameter.`);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
const spellCorrectionValues = actual.searchParams.getAll("spellCorrectionEnabled");
|
|
180
|
+
if (spellCorrectionValues.length > 1 ||
|
|
181
|
+
spellCorrectionValues.some((value) => value !== "true" && value !== "false")) {
|
|
182
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported an invalid spellCorrectionEnabled parameter.");
|
|
183
|
+
}
|
|
184
|
+
const actualPages = actual.searchParams.getAll("page");
|
|
185
|
+
const actualPage = actualPages[0] ?? null;
|
|
186
|
+
if (actualPages.length > 1) {
|
|
187
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported multiple page numbers.");
|
|
188
|
+
}
|
|
189
|
+
if (pageNumber === 1) {
|
|
190
|
+
if (actualPage && actualPage !== "1") {
|
|
191
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported the wrong page number.");
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
else if (actualPage !== String(pageNumber)) {
|
|
195
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported the wrong page number.");
|
|
196
|
+
}
|
|
197
|
+
return buildLinkedInProductSearchPageUrl(queryUrl, pageNumber);
|
|
198
|
+
}
|
|
199
|
+
export function canonicalizeLinkedInProductUrl(value) {
|
|
200
|
+
let parsed;
|
|
201
|
+
try {
|
|
202
|
+
parsed = new URL(value, "https://www.linkedin.com");
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid LinkedIn product URL: ${value}`);
|
|
206
|
+
}
|
|
207
|
+
if (parsed.protocol !== "https:" ||
|
|
208
|
+
parsed.port !== "" ||
|
|
209
|
+
parsed.username !== "" ||
|
|
210
|
+
parsed.password !== "" ||
|
|
211
|
+
!["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
|
|
212
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Product URL is outside LinkedIn: ${value}`);
|
|
213
|
+
}
|
|
214
|
+
const match = parsed.pathname.match(/^\/products\/([^/]+)\/?$/i);
|
|
215
|
+
if (!match?.[1]) {
|
|
216
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Product URL must match /products/<slug>/: ${value}`);
|
|
217
|
+
}
|
|
218
|
+
let decodedSlug;
|
|
219
|
+
try {
|
|
220
|
+
decodedSlug = decodeURIComponent(match[1]).trim().normalize("NFC").toLowerCase();
|
|
221
|
+
}
|
|
222
|
+
catch {
|
|
223
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid encoded product slug: ${value}`);
|
|
224
|
+
}
|
|
225
|
+
if (!decodedSlug || /[\u0000-\u001f\u007f/\\]/.test(decodedSlug)) {
|
|
226
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid LinkedIn product slug: ${match[1]}`);
|
|
227
|
+
}
|
|
228
|
+
const slug = encodeURIComponent(decodedSlug)
|
|
229
|
+
.replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).padStart(2, "0")}`)
|
|
230
|
+
.toLowerCase();
|
|
231
|
+
if (slug.length > 720 || !/^(?:[a-z0-9._~-]|%[0-9a-f]{2})+$/.test(slug)) {
|
|
232
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid LinkedIn product slug: ${slug}`);
|
|
233
|
+
}
|
|
234
|
+
return {
|
|
235
|
+
linkedinUrl: `https://www.linkedin.com/products/${slug}`,
|
|
236
|
+
linkedinSlug: slug
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
function normalizeIdentity(value) {
|
|
240
|
+
const identity = cleanText(value);
|
|
241
|
+
if (!identity)
|
|
242
|
+
return undefined;
|
|
243
|
+
if (!/^[A-Za-z0-9:._-]+$/.test(identity)) {
|
|
244
|
+
throw new LinkedInProductCollectorInvariantError("invalid_identity", "Product identity contains unsupported characters.");
|
|
245
|
+
}
|
|
246
|
+
return identity;
|
|
247
|
+
}
|
|
248
|
+
function normalizeVisibleProductCard(card, rank) {
|
|
249
|
+
const { linkedinUrl, linkedinSlug } = canonicalizeLinkedInProductUrl(card.linkedinUrl);
|
|
250
|
+
const name = cleanText(card.name);
|
|
251
|
+
if (!name) {
|
|
252
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product", "A visible product card is missing its name.");
|
|
253
|
+
}
|
|
254
|
+
const imageUrl = canonicalizePublicUrl(card.imageUrl);
|
|
255
|
+
const vendor = cleanText(card.vendor);
|
|
256
|
+
const category = cleanText(card.category);
|
|
257
|
+
const description = cleanText(card.description);
|
|
258
|
+
const identity = normalizeIdentity(card.identity);
|
|
259
|
+
return {
|
|
260
|
+
rank,
|
|
261
|
+
name,
|
|
262
|
+
linkedinUrl,
|
|
263
|
+
linkedinSlug,
|
|
264
|
+
...(imageUrl ? { imageUrl } : {}),
|
|
265
|
+
...(vendor ? { vendor } : {}),
|
|
266
|
+
...(category ? { category } : {}),
|
|
267
|
+
...(description ? { description } : {}),
|
|
268
|
+
...(identity ? { identity } : {})
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
export function createLinkedInProductCollectorCheckpoint(queryUrl, now = new Date()) {
|
|
272
|
+
const normalizedQueryUrl = normalizeLinkedInProductSearchUrl(queryUrl);
|
|
273
|
+
const timestamp = now.toISOString();
|
|
274
|
+
return {
|
|
275
|
+
schemaVersion: CHECKPOINT_SCHEMA_VERSION,
|
|
276
|
+
queryUrl: normalizedQueryUrl,
|
|
277
|
+
queryHash: sha256(normalizedQueryUrl),
|
|
278
|
+
startedAt: timestamp,
|
|
279
|
+
updatedAt: timestamp,
|
|
280
|
+
status: "collecting",
|
|
281
|
+
nextPageNumber: 1,
|
|
282
|
+
pagesAccepted: 0,
|
|
283
|
+
tasksIssued: 0,
|
|
284
|
+
approximateReportedTotal: null,
|
|
285
|
+
products: []
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
export function applyLinkedInProductSearchPage(checkpoint, submission, now = new Date()) {
|
|
289
|
+
if (checkpoint.status !== "collecting") {
|
|
290
|
+
throw new LinkedInProductCollectorInvariantError("checkpoint_closed", "The checkpoint is not collecting.");
|
|
291
|
+
}
|
|
292
|
+
if (submission.pageNumber !== checkpoint.nextPageNumber) {
|
|
293
|
+
throw new LinkedInProductCollectorInvariantError("out_of_order_page", `Expected page ${checkpoint.nextPageNumber}, received page ${submission.pageNumber}.`);
|
|
294
|
+
}
|
|
295
|
+
const currentUrl = assertExpectedLinkedInSearchPage(submission.currentUrl, checkpoint.queryUrl, submission.pageNumber);
|
|
296
|
+
if (submission.hasNext && submission.items.length === 0) {
|
|
297
|
+
throw new LinkedInProductCollectorInvariantError("empty_page_with_next", "A page that claims to have a next page must contain at least one visible product card.");
|
|
298
|
+
}
|
|
299
|
+
const products = checkpoint.products.map((product) => ({ ...product }));
|
|
300
|
+
const byUrl = new Map(products.map((product) => [product.linkedinUrl, product]));
|
|
301
|
+
const byIdentity = new Map();
|
|
302
|
+
for (const product of products) {
|
|
303
|
+
if (product.identity)
|
|
304
|
+
byIdentity.set(product.identity, product);
|
|
305
|
+
}
|
|
306
|
+
let duplicateCount = 0;
|
|
307
|
+
let newProductCount = 0;
|
|
308
|
+
for (const input of submission.items) {
|
|
309
|
+
const candidate = normalizeVisibleProductCard(input, products.length + 1);
|
|
310
|
+
const existingByUrl = byUrl.get(candidate.linkedinUrl);
|
|
311
|
+
const existingByIdentity = candidate.identity ? byIdentity.get(candidate.identity) : undefined;
|
|
312
|
+
if (existingByIdentity && existingByIdentity.linkedinUrl !== candidate.linkedinUrl) {
|
|
313
|
+
throw new LinkedInProductCollectorInvariantError("identity_drift", `Product identity ${candidate.identity} moved to a different canonical URL.`);
|
|
314
|
+
}
|
|
315
|
+
if (existingByUrl) {
|
|
316
|
+
if (candidate.identity && existingByUrl.identity && candidate.identity !== existingByUrl.identity) {
|
|
317
|
+
throw new LinkedInProductCollectorInvariantError("identity_drift", `Canonical URL ${candidate.linkedinUrl} changed product identity.`);
|
|
318
|
+
}
|
|
319
|
+
if (existingByUrl.name !== candidate.name) {
|
|
320
|
+
throw new LinkedInProductCollectorInvariantError("identity_drift", `Canonical URL ${candidate.linkedinUrl} changed product name.`);
|
|
321
|
+
}
|
|
322
|
+
if (!existingByUrl.identity && candidate.identity) {
|
|
323
|
+
existingByUrl.identity = candidate.identity;
|
|
324
|
+
byIdentity.set(candidate.identity, existingByUrl);
|
|
325
|
+
}
|
|
326
|
+
duplicateCount += 1;
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
products.push(candidate);
|
|
330
|
+
byUrl.set(candidate.linkedinUrl, candidate);
|
|
331
|
+
if (candidate.identity)
|
|
332
|
+
byIdentity.set(candidate.identity, candidate);
|
|
333
|
+
newProductCount += 1;
|
|
334
|
+
}
|
|
335
|
+
products.forEach((product, index) => {
|
|
336
|
+
if (product.rank !== index + 1) {
|
|
337
|
+
throw new LinkedInProductCollectorInvariantError("rank_drift", "Checkpoint ranks are not contiguous.");
|
|
338
|
+
}
|
|
339
|
+
});
|
|
340
|
+
const capturedAt = now.toISOString();
|
|
341
|
+
const reportedTotal = submission.reportedTotal ?? checkpoint.approximateReportedTotal;
|
|
342
|
+
const terminalPage = submission.hasNext
|
|
343
|
+
? undefined
|
|
344
|
+
: {
|
|
345
|
+
pageNumber: submission.pageNumber,
|
|
346
|
+
currentUrl,
|
|
347
|
+
reportedTotal: submission.reportedTotal ?? null,
|
|
348
|
+
hasNext: false,
|
|
349
|
+
visibleItemCount: submission.items.length,
|
|
350
|
+
newProductCount,
|
|
351
|
+
capturedAt
|
|
352
|
+
};
|
|
353
|
+
const nextCheckpoint = {
|
|
354
|
+
...checkpoint,
|
|
355
|
+
updatedAt: capturedAt,
|
|
356
|
+
status: submission.hasNext ? "collecting" : "complete",
|
|
357
|
+
nextPageNumber: submission.hasNext ? submission.pageNumber + 1 : submission.pageNumber,
|
|
358
|
+
pagesAccepted: checkpoint.pagesAccepted + 1,
|
|
359
|
+
tasksIssued: Math.max(checkpoint.tasksIssued, checkpoint.pagesAccepted + 1),
|
|
360
|
+
approximateReportedTotal: reportedTotal ?? null,
|
|
361
|
+
products,
|
|
362
|
+
...(terminalPage ? { terminalPage, completedAt: capturedAt } : {}),
|
|
363
|
+
lastStop: undefined
|
|
364
|
+
};
|
|
365
|
+
if (terminalPage)
|
|
366
|
+
nextCheckpoint.orderChecksum = computeLinkedInProductOrderChecksum(products);
|
|
367
|
+
return { checkpoint: nextCheckpoint, newProductCount, duplicateCount };
|
|
368
|
+
}
|
|
369
|
+
export function computeLinkedInProductOrderChecksum(products) {
|
|
370
|
+
const manifest = products.map((product) => `${product.rank}\t${product.linkedinUrl}`).join("\n");
|
|
371
|
+
return sha256(manifest);
|
|
372
|
+
}
|
|
373
|
+
export function buildLinkedInProductCollectionArtifact(checkpoint) {
|
|
374
|
+
assertValidCheckpoint(checkpoint);
|
|
375
|
+
if (checkpoint.status !== "complete" || !checkpoint.terminalPage || !checkpoint.completedAt) {
|
|
376
|
+
throw new LinkedInProductCollectorInvariantError("incomplete_checkpoint", "A complete artifact requires natural terminal-page evidence.");
|
|
377
|
+
}
|
|
378
|
+
const products = checkpoint.products.map((stored) => {
|
|
379
|
+
const canonical = canonicalizeLinkedInProductUrl(stored.linkedinUrl);
|
|
380
|
+
if (canonical.linkedinSlug !== stored.linkedinSlug) {
|
|
381
|
+
throw new LinkedInProductCollectorInvariantError("canonical_url_drift", "Checkpoint product URL and slug disagree.");
|
|
382
|
+
}
|
|
383
|
+
const name = cleanText(stored.name);
|
|
384
|
+
if (!name)
|
|
385
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product", "Checkpoint product name is empty.");
|
|
386
|
+
const imageUrl = canonicalizePublicUrl(stored.imageUrl);
|
|
387
|
+
const vendor = cleanText(stored.vendor);
|
|
388
|
+
const category = cleanText(stored.category);
|
|
389
|
+
const description = cleanText(stored.description);
|
|
390
|
+
return {
|
|
391
|
+
rank: stored.rank,
|
|
392
|
+
name,
|
|
393
|
+
linkedinUrl: canonical.linkedinUrl,
|
|
394
|
+
linkedinSlug: canonical.linkedinSlug,
|
|
395
|
+
...(imageUrl ? { imageUrl } : {}),
|
|
396
|
+
...(vendor ? { vendor } : {}),
|
|
397
|
+
...(category ? { category } : {}),
|
|
398
|
+
...(description ? { description } : {})
|
|
399
|
+
};
|
|
400
|
+
});
|
|
401
|
+
products.forEach((product, index) => {
|
|
402
|
+
if (product.rank !== index + 1) {
|
|
403
|
+
throw new LinkedInProductCollectorInvariantError("rank_drift", "Artifact ranks are not contiguous.");
|
|
404
|
+
}
|
|
405
|
+
});
|
|
406
|
+
const orderChecksum = computeLinkedInProductOrderChecksum(products);
|
|
407
|
+
if (checkpoint.orderChecksum && checkpoint.orderChecksum !== orderChecksum) {
|
|
408
|
+
throw new LinkedInProductCollectorInvariantError("checksum_drift", "Checkpoint order checksum changed.");
|
|
409
|
+
}
|
|
410
|
+
const terminalPage = {
|
|
411
|
+
pageNumber: checkpoint.terminalPage.pageNumber,
|
|
412
|
+
currentUrl: checkpoint.terminalPage.currentUrl,
|
|
413
|
+
reportedTotal: checkpoint.terminalPage.reportedTotal,
|
|
414
|
+
hasNext: false,
|
|
415
|
+
visibleItemCount: checkpoint.terminalPage.visibleItemCount,
|
|
416
|
+
newProductCount: checkpoint.terminalPage.newProductCount,
|
|
417
|
+
capturedAt: checkpoint.terminalPage.capturedAt
|
|
418
|
+
};
|
|
419
|
+
return {
|
|
420
|
+
schemaVersion: ARTIFACT_SCHEMA_VERSION,
|
|
421
|
+
kind: "linkedin_product_search_snapshot",
|
|
422
|
+
complete: true,
|
|
423
|
+
source: {
|
|
424
|
+
queryUrl: checkpoint.queryUrl,
|
|
425
|
+
queryHash: checkpoint.queryHash
|
|
426
|
+
},
|
|
427
|
+
capture: {
|
|
428
|
+
startedAt: checkpoint.startedAt,
|
|
429
|
+
completedAt: checkpoint.completedAt,
|
|
430
|
+
approximateReportedTotal: checkpoint.approximateReportedTotal,
|
|
431
|
+
pagesAccepted: checkpoint.pagesAccepted,
|
|
432
|
+
terminalPage
|
|
433
|
+
},
|
|
434
|
+
productCount: products.length,
|
|
435
|
+
orderChecksumAlgorithm: "sha256",
|
|
436
|
+
orderChecksum,
|
|
437
|
+
products
|
|
438
|
+
};
|
|
439
|
+
}
|
|
440
|
+
function rawVisibleProductCard(card) {
|
|
441
|
+
const { rank: _rank, ...normalized } = normalizeVisibleProductCard(card, 1);
|
|
442
|
+
return normalized;
|
|
443
|
+
}
|
|
444
|
+
function sanitizeTerminalErrorUrl(currentUrl) {
|
|
445
|
+
let parsed;
|
|
446
|
+
try {
|
|
447
|
+
parsed = new URL(currentUrl);
|
|
448
|
+
}
|
|
449
|
+
catch {
|
|
450
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported an invalid terminal URL.");
|
|
451
|
+
}
|
|
452
|
+
if (parsed.protocol !== "https:" ||
|
|
453
|
+
parsed.port !== "" ||
|
|
454
|
+
parsed.username !== "" ||
|
|
455
|
+
parsed.password !== "" ||
|
|
456
|
+
!["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
|
|
457
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "A terminal browser error must come from a LinkedIn page.");
|
|
458
|
+
}
|
|
459
|
+
return `https://www.linkedin.com${parsed.pathname}`;
|
|
460
|
+
}
|
|
461
|
+
function terminalStopMessage(code) {
|
|
462
|
+
switch (code) {
|
|
463
|
+
case "auth_required":
|
|
464
|
+
return "The signed-in LinkedIn session is no longer available.";
|
|
465
|
+
case "challenge":
|
|
466
|
+
return "LinkedIn displayed a checkpoint or challenge.";
|
|
467
|
+
case "rate_limited":
|
|
468
|
+
return "LinkedIn rate-limited the product search.";
|
|
469
|
+
case "interrupted":
|
|
470
|
+
return "The browser worker was interrupted.";
|
|
471
|
+
case "unexpected_page":
|
|
472
|
+
return "The browser reached an unexpected LinkedIn page.";
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
async function ensureParent(filePath) {
|
|
476
|
+
await mkdir(path.dirname(path.resolve(filePath)), { recursive: true });
|
|
477
|
+
}
|
|
478
|
+
async function writePrivateJsonAtomic(filePath, value) {
|
|
479
|
+
const resolved = path.resolve(filePath);
|
|
480
|
+
await ensureParent(resolved);
|
|
481
|
+
const temporary = `${resolved}.${process.pid}.${randomUUID()}.tmp`;
|
|
482
|
+
await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
483
|
+
await chmod(temporary, 0o600);
|
|
484
|
+
await rename(temporary, resolved);
|
|
485
|
+
await chmod(resolved, 0o600);
|
|
486
|
+
}
|
|
487
|
+
async function appendPrivateJsonLine(filePath, value) {
|
|
488
|
+
const resolved = path.resolve(filePath);
|
|
489
|
+
await ensureParent(resolved);
|
|
490
|
+
await appendFile(resolved, `${JSON.stringify(value)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
491
|
+
await chmod(resolved, 0o600);
|
|
492
|
+
}
|
|
493
|
+
async function readCheckpoint(filePath) {
|
|
494
|
+
try {
|
|
495
|
+
const value = JSON.parse(await readFile(path.resolve(filePath), "utf8"));
|
|
496
|
+
if (value.schemaVersion !== CHECKPOINT_SCHEMA_VERSION || !Array.isArray(value.products)) {
|
|
497
|
+
throw new Error("Unsupported LinkedIn product collector checkpoint.");
|
|
498
|
+
}
|
|
499
|
+
assertValidCheckpoint(value);
|
|
500
|
+
return value;
|
|
501
|
+
}
|
|
502
|
+
catch (error) {
|
|
503
|
+
if (error.code === "ENOENT")
|
|
504
|
+
return null;
|
|
505
|
+
throw error;
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
function assertValidCheckpoint(checkpoint) {
|
|
509
|
+
if (normalizeLinkedInProductSearchUrl(checkpoint.queryUrl) !== checkpoint.queryUrl) {
|
|
510
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint query URL is invalid.");
|
|
511
|
+
}
|
|
512
|
+
if (!/^[a-f0-9]{64}$/.test(checkpoint.queryHash)) {
|
|
513
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint query hash is invalid.");
|
|
514
|
+
}
|
|
515
|
+
if (!["collecting", "complete", "failed"].includes(checkpoint.status)) {
|
|
516
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint status is invalid.");
|
|
517
|
+
}
|
|
518
|
+
if (!Number.isInteger(checkpoint.nextPageNumber) ||
|
|
519
|
+
checkpoint.nextPageNumber < 1 ||
|
|
520
|
+
!Number.isInteger(checkpoint.pagesAccepted) ||
|
|
521
|
+
checkpoint.pagesAccepted < 0 ||
|
|
522
|
+
!Number.isInteger(checkpoint.tasksIssued) ||
|
|
523
|
+
checkpoint.tasksIssued < 0) {
|
|
524
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint counters are invalid.");
|
|
525
|
+
}
|
|
526
|
+
if (checkpoint.tasksIssued < checkpoint.pagesAccepted) {
|
|
527
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint has more accepted pages than issued browser tasks.");
|
|
528
|
+
}
|
|
529
|
+
assertIsoTimestamp(checkpoint.startedAt, "startedAt");
|
|
530
|
+
assertIsoTimestamp(checkpoint.updatedAt, "updatedAt");
|
|
531
|
+
if (checkpoint.updatedAt < checkpoint.startedAt) {
|
|
532
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint timestamps are out of order.");
|
|
533
|
+
}
|
|
534
|
+
if (checkpoint.approximateReportedTotal !== null &&
|
|
535
|
+
(!Number.isInteger(checkpoint.approximateReportedTotal) || checkpoint.approximateReportedTotal < 0)) {
|
|
536
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint approximate reported total is invalid.");
|
|
537
|
+
}
|
|
538
|
+
const seenUrls = new Set();
|
|
539
|
+
const seenIdentities = new Set();
|
|
540
|
+
checkpoint.products.forEach((product, index) => {
|
|
541
|
+
if (product.rank !== index + 1 || !Number.isInteger(product.rank)) {
|
|
542
|
+
throw new LinkedInProductCollectorInvariantError("rank_drift", "Checkpoint ranks are not contiguous.");
|
|
543
|
+
}
|
|
544
|
+
const canonical = canonicalizeLinkedInProductUrl(product.linkedinUrl);
|
|
545
|
+
if (canonical.linkedinUrl !== product.linkedinUrl || canonical.linkedinSlug !== product.linkedinSlug) {
|
|
546
|
+
throw new LinkedInProductCollectorInvariantError("canonical_url_drift", "Checkpoint contains a non-canonical product URL.");
|
|
547
|
+
}
|
|
548
|
+
if (seenUrls.has(product.linkedinUrl)) {
|
|
549
|
+
throw new LinkedInProductCollectorInvariantError("duplicate_product", "Checkpoint contains a duplicate product URL.");
|
|
550
|
+
}
|
|
551
|
+
seenUrls.add(product.linkedinUrl);
|
|
552
|
+
if (product.identity) {
|
|
553
|
+
const identity = normalizeIdentity(product.identity);
|
|
554
|
+
if (!identity || seenIdentities.has(identity)) {
|
|
555
|
+
throw new LinkedInProductCollectorInvariantError("identity_drift", "Checkpoint contains a duplicate or invalid product identity.");
|
|
556
|
+
}
|
|
557
|
+
seenIdentities.add(identity);
|
|
558
|
+
}
|
|
559
|
+
});
|
|
560
|
+
if (checkpoint.status === "complete") {
|
|
561
|
+
if (checkpoint.pagesAccepted < 1 ||
|
|
562
|
+
checkpoint.nextPageNumber !== checkpoint.pagesAccepted ||
|
|
563
|
+
!checkpoint.terminalPage ||
|
|
564
|
+
!checkpoint.completedAt ||
|
|
565
|
+
checkpoint.lastStop !== undefined ||
|
|
566
|
+
!/^[a-f0-9]{64}$/.test(checkpoint.orderChecksum ?? "")) {
|
|
567
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "A complete checkpoint is missing terminal evidence or its checksum.");
|
|
568
|
+
}
|
|
569
|
+
const terminal = checkpoint.terminalPage;
|
|
570
|
+
if (terminal.hasNext !== false ||
|
|
571
|
+
!Number.isInteger(terminal.pageNumber) ||
|
|
572
|
+
terminal.pageNumber !== checkpoint.pagesAccepted ||
|
|
573
|
+
terminal.currentUrl !== buildLinkedInProductSearchPageUrl(checkpoint.queryUrl, terminal.pageNumber) ||
|
|
574
|
+
!Number.isInteger(terminal.visibleItemCount) ||
|
|
575
|
+
terminal.visibleItemCount < 0 ||
|
|
576
|
+
!Number.isInteger(terminal.newProductCount) ||
|
|
577
|
+
terminal.newProductCount < 0 ||
|
|
578
|
+
terminal.newProductCount > terminal.visibleItemCount ||
|
|
579
|
+
(terminal.reportedTotal !== null &&
|
|
580
|
+
(!Number.isInteger(terminal.reportedTotal) || terminal.reportedTotal < 0))) {
|
|
581
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Complete checkpoint terminal-page evidence is invalid.");
|
|
582
|
+
}
|
|
583
|
+
assertIsoTimestamp(terminal.capturedAt, "terminalPage.capturedAt");
|
|
584
|
+
assertIsoTimestamp(checkpoint.completedAt, "completedAt");
|
|
585
|
+
if (terminal.capturedAt !== checkpoint.completedAt || checkpoint.completedAt !== checkpoint.updatedAt) {
|
|
586
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Complete checkpoint capture timestamps disagree.");
|
|
587
|
+
}
|
|
588
|
+
if (computeLinkedInProductOrderChecksum(checkpoint.products) !== checkpoint.orderChecksum) {
|
|
589
|
+
throw new LinkedInProductCollectorInvariantError("checksum_drift", "Checkpoint order checksum changed.");
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
else {
|
|
593
|
+
if (checkpoint.nextPageNumber !== checkpoint.pagesAccepted + 1 ||
|
|
594
|
+
checkpoint.terminalPage !== undefined ||
|
|
595
|
+
checkpoint.completedAt !== undefined ||
|
|
596
|
+
checkpoint.orderChecksum !== undefined) {
|
|
597
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Incomplete checkpoint state is internally inconsistent.");
|
|
598
|
+
}
|
|
599
|
+
if (checkpoint.lastStop) {
|
|
600
|
+
if (typeof checkpoint.lastStop.reason !== "string" ||
|
|
601
|
+
checkpoint.lastStop.reason.trim() === "" ||
|
|
602
|
+
!Number.isInteger(checkpoint.lastStop.pageNumber) ||
|
|
603
|
+
checkpoint.lastStop.pageNumber !== checkpoint.nextPageNumber) {
|
|
604
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint stop evidence is invalid.");
|
|
605
|
+
}
|
|
606
|
+
assertIsoTimestamp(checkpoint.lastStop.at, "lastStop.at");
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
function assertIsoTimestamp(value, field) {
|
|
611
|
+
if (typeof value !== "string") {
|
|
612
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", `Checkpoint ${field} is not a timestamp.`);
|
|
613
|
+
}
|
|
614
|
+
const parsed = new Date(value);
|
|
615
|
+
if (Number.isNaN(parsed.valueOf()) || parsed.toISOString() !== value) {
|
|
616
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", `Checkpoint ${field} is not canonical ISO time.`);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
async function fileExists(filePath) {
|
|
620
|
+
try {
|
|
621
|
+
await access(path.resolve(filePath));
|
|
622
|
+
return true;
|
|
623
|
+
}
|
|
624
|
+
catch {
|
|
625
|
+
return false;
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
async function assertRawEvidenceMatchesCheckpoint(filePath, checkpoint) {
|
|
629
|
+
const raw = await readFile(path.resolve(filePath), "utf8");
|
|
630
|
+
const lines = raw.split(/\r?\n/).filter((line) => line.trim() !== "");
|
|
631
|
+
let replay = createLinkedInProductCollectorCheckpoint(checkpoint.queryUrl, new Date(checkpoint.startedAt));
|
|
632
|
+
let acceptedPageRecords = 0;
|
|
633
|
+
for (const [index, line] of lines.entries()) {
|
|
634
|
+
let parsedJson;
|
|
635
|
+
try {
|
|
636
|
+
parsedJson = JSON.parse(line);
|
|
637
|
+
}
|
|
638
|
+
catch {
|
|
639
|
+
throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw JSONL line ${index + 1} is not valid JSON.`);
|
|
640
|
+
}
|
|
641
|
+
const record = rawEvidenceRecordSchema.parse(parsedJson);
|
|
642
|
+
if (record.queryHash !== checkpoint.queryHash) {
|
|
643
|
+
throw new LinkedInProductCollectorInvariantError("raw_query_mismatch", `Raw JSONL line ${index + 1} belongs to another query.`);
|
|
644
|
+
}
|
|
645
|
+
assertIsoTimestamp(record.receivedAt, `raw line ${index + 1} receivedAt`);
|
|
646
|
+
if (record.type === "terminal_error") {
|
|
647
|
+
if (replay.status !== "collecting" || record.pageNumber !== replay.nextPageNumber) {
|
|
648
|
+
throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw terminal error on line ${index + 1} is out of sequence.`);
|
|
649
|
+
}
|
|
650
|
+
continue;
|
|
651
|
+
}
|
|
652
|
+
if (record.pageNumber !== replay.nextPageNumber) {
|
|
653
|
+
throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw page on line ${index + 1} is duplicate or out of sequence.`);
|
|
654
|
+
}
|
|
655
|
+
const items = record.items.map(({ linkedinSlug, ...item }) => {
|
|
656
|
+
const canonical = canonicalizeLinkedInProductUrl(item.linkedinUrl);
|
|
657
|
+
if (canonical.linkedinSlug !== linkedinSlug) {
|
|
658
|
+
throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw product slug on line ${index + 1} disagrees with its URL.`);
|
|
659
|
+
}
|
|
660
|
+
return item;
|
|
661
|
+
});
|
|
662
|
+
const applied = applyLinkedInProductSearchPage(replay, {
|
|
663
|
+
pageNumber: record.pageNumber,
|
|
664
|
+
currentUrl: record.currentUrl,
|
|
665
|
+
reportedTotal: record.reportedTotal,
|
|
666
|
+
hasNext: record.hasNext,
|
|
667
|
+
items
|
|
668
|
+
}, new Date(record.receivedAt));
|
|
669
|
+
if (record.visibleItemCount !== record.items.length ||
|
|
670
|
+
record.newProductCount !== applied.newProductCount ||
|
|
671
|
+
record.duplicateCount !== applied.duplicateCount) {
|
|
672
|
+
throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw page counts on line ${index + 1} do not match its normalized items.`);
|
|
673
|
+
}
|
|
674
|
+
replay = applied.checkpoint;
|
|
675
|
+
acceptedPageRecords += 1;
|
|
676
|
+
}
|
|
677
|
+
if (acceptedPageRecords !== checkpoint.pagesAccepted) {
|
|
678
|
+
throw new LinkedInProductCollectorInvariantError("missing_raw_evidence", `Raw JSONL proves ${acceptedPageRecords} pages, but the checkpoint claims ${checkpoint.pagesAccepted}.`);
|
|
679
|
+
}
|
|
680
|
+
if (JSON.stringify(replay.products) !== JSON.stringify(checkpoint.products)) {
|
|
681
|
+
throw new LinkedInProductCollectorInvariantError("raw_order_mismatch", "Raw JSONL product order does not reproduce the checkpoint.");
|
|
682
|
+
}
|
|
683
|
+
if (replay.approximateReportedTotal !== checkpoint.approximateReportedTotal ||
|
|
684
|
+
(checkpoint.status === "complete") !== (replay.status === "complete")) {
|
|
685
|
+
throw new LinkedInProductCollectorInvariantError("raw_state_mismatch", "Raw JSONL completion metadata does not reproduce the checkpoint.");
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
async function readRelayJson(req) {
|
|
689
|
+
const chunks = [];
|
|
690
|
+
let size = 0;
|
|
691
|
+
for await (const chunk of req) {
|
|
692
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
693
|
+
size += buffer.length;
|
|
694
|
+
if (size > MAX_RELAY_BODY_BYTES) {
|
|
695
|
+
throw new LinkedInProductCollectorInvariantError("body_too_large", "Relay request body is too large.");
|
|
696
|
+
}
|
|
697
|
+
chunks.push(buffer);
|
|
698
|
+
}
|
|
699
|
+
if (chunks.length === 0)
|
|
700
|
+
throw new Error("Relay request body is empty.");
|
|
701
|
+
return JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
702
|
+
}
|
|
703
|
+
function writeRelayJson(res, statusCode, value, origin) {
|
|
704
|
+
const body = `${JSON.stringify(value)}\n`;
|
|
705
|
+
res.writeHead(statusCode, {
|
|
706
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
707
|
+
"Content-Length": Buffer.byteLength(body),
|
|
708
|
+
"Cache-Control": "no-store",
|
|
709
|
+
"Access-Control-Allow-Origin": origin === "https://www.linkedin.com" ? origin : "https://www.linkedin.com",
|
|
710
|
+
"Access-Control-Allow-Private-Network": "true",
|
|
711
|
+
Vary: "Origin"
|
|
712
|
+
});
|
|
713
|
+
res.end(body);
|
|
714
|
+
}
|
|
715
|
+
function isAllowedRelayOrigin(req) {
|
|
716
|
+
const origin = req.headers.origin;
|
|
717
|
+
return origin === undefined || origin === "https://www.linkedin.com";
|
|
718
|
+
}
|
|
719
|
+
function closeServer(server) {
|
|
720
|
+
return new Promise((resolve) => server.close(() => resolve()));
|
|
721
|
+
}
|
|
722
|
+
export async function collectLinkedInProductsViaBrowserRelay(options) {
|
|
723
|
+
const queryUrl = normalizeLinkedInProductSearchUrl(options.queryUrl);
|
|
724
|
+
const expectedQueryHash = sha256(queryUrl);
|
|
725
|
+
const resolvedPaths = [options.checkpointPath, options.rawJsonlPath, options.outPath].map((value) => path.resolve(value));
|
|
726
|
+
if (new Set(resolvedPaths).size !== resolvedPaths.length) {
|
|
727
|
+
throw new LinkedInProductCollectorInvariantError("artifact_path_conflict", "Checkpoint, raw JSONL, and complete artifact paths must be distinct.");
|
|
728
|
+
}
|
|
729
|
+
const existingCheckpoint = await readCheckpoint(options.checkpointPath);
|
|
730
|
+
const [rawExists, outExists] = await Promise.all([
|
|
731
|
+
fileExists(options.rawJsonlPath),
|
|
732
|
+
fileExists(options.outPath)
|
|
733
|
+
]);
|
|
734
|
+
if (!existingCheckpoint && (rawExists || outExists)) {
|
|
735
|
+
throw new LinkedInProductCollectorInvariantError("orphaned_artifact", "A new collection requires unused raw JSONL and output paths. Choose new paths or restore the matching checkpoint.");
|
|
736
|
+
}
|
|
737
|
+
if (existingCheckpoint && existingCheckpoint.pagesAccepted > 0 && !rawExists) {
|
|
738
|
+
throw new LinkedInProductCollectorInvariantError("missing_raw_evidence", "The checkpoint has accepted pages but its raw JSONL evidence is missing.");
|
|
739
|
+
}
|
|
740
|
+
if (existingCheckpoint && rawExists) {
|
|
741
|
+
await assertRawEvidenceMatchesCheckpoint(options.rawJsonlPath, existingCheckpoint);
|
|
742
|
+
}
|
|
743
|
+
if (existingCheckpoint && existingCheckpoint.status !== "complete" && outExists) {
|
|
744
|
+
throw new LinkedInProductCollectorInvariantError("stale_complete_artifact", "An incomplete checkpoint cannot share a path with an existing complete artifact. Choose a new output path.");
|
|
745
|
+
}
|
|
746
|
+
let checkpoint = existingCheckpoint ?? createLinkedInProductCollectorCheckpoint(queryUrl);
|
|
747
|
+
if (checkpoint.queryUrl !== queryUrl || checkpoint.queryHash !== expectedQueryHash) {
|
|
748
|
+
throw new LinkedInProductCollectorInvariantError("checkpoint_query_mismatch", "The checkpoint belongs to a different LinkedIn product search URL.");
|
|
749
|
+
}
|
|
750
|
+
await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
|
|
751
|
+
if (checkpoint.status === "complete") {
|
|
752
|
+
const artifact = buildLinkedInProductCollectionArtifact(checkpoint);
|
|
753
|
+
await writePrivateJsonAtomic(options.outPath, artifact);
|
|
754
|
+
return {
|
|
755
|
+
status: "complete",
|
|
756
|
+
complete: true,
|
|
757
|
+
queryUrl,
|
|
758
|
+
queryHash: expectedQueryHash,
|
|
759
|
+
checkpointPath: path.resolve(options.checkpointPath),
|
|
760
|
+
rawJsonlPath: path.resolve(options.rawJsonlPath),
|
|
761
|
+
outPath: path.resolve(options.outPath),
|
|
762
|
+
pagesAccepted: checkpoint.pagesAccepted,
|
|
763
|
+
productsCollected: checkpoint.products.length,
|
|
764
|
+
nextPageNumber: checkpoint.nextPageNumber,
|
|
765
|
+
orderChecksum: checkpoint.orderChecksum,
|
|
766
|
+
terminalPage: checkpoint.terminalPage
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
if (checkpoint.status === "failed") {
|
|
770
|
+
throw new LinkedInProductCollectorInvariantError("failed_checkpoint", "The checkpoint is failed. Inspect it and choose a new checkpoint path before retrying.");
|
|
771
|
+
}
|
|
772
|
+
let uniqueTasksIssuedThisRun = 0;
|
|
773
|
+
let lastIssuedPage = null;
|
|
774
|
+
let settled = false;
|
|
775
|
+
let acceptingRequests = true;
|
|
776
|
+
let idleTimer;
|
|
777
|
+
let incompleteStopPromise;
|
|
778
|
+
let resolveResult;
|
|
779
|
+
let rejectResult;
|
|
780
|
+
const resultPromise = new Promise((resolve, reject) => {
|
|
781
|
+
resolveResult = resolve;
|
|
782
|
+
rejectResult = reject;
|
|
783
|
+
});
|
|
784
|
+
let requestQueue = Promise.resolve();
|
|
785
|
+
const enqueueExclusive = (operation) => {
|
|
786
|
+
const queued = requestQueue.then(operation, operation);
|
|
787
|
+
requestQueue = queued.catch(() => undefined);
|
|
788
|
+
return queued;
|
|
789
|
+
};
|
|
790
|
+
const server = createServer((req, res) => {
|
|
791
|
+
if (!acceptingRequests) {
|
|
792
|
+
writeRelayJson(res, 410, { status: "closed" }, req.headers.origin);
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
const execute = async () => {
|
|
796
|
+
try {
|
|
797
|
+
await handleRequest(req, res);
|
|
798
|
+
}
|
|
799
|
+
catch (error) {
|
|
800
|
+
const browserInputError = error instanceof LinkedInProductCollectorInvariantError || error instanceof z.ZodError || error instanceof SyntaxError;
|
|
801
|
+
if (!res.headersSent) {
|
|
802
|
+
writeRelayJson(res, browserInputError ? 409 : 500, {
|
|
803
|
+
status: "rejected",
|
|
804
|
+
error: error instanceof Error ? error.message : String(error)
|
|
805
|
+
}, req.headers.origin);
|
|
806
|
+
}
|
|
807
|
+
if (!browserInputError)
|
|
808
|
+
await abortWithError(error);
|
|
809
|
+
}
|
|
810
|
+
};
|
|
811
|
+
void enqueueExclusive(execute);
|
|
812
|
+
});
|
|
813
|
+
const makeResult = (status, reason) => ({
|
|
814
|
+
status,
|
|
815
|
+
complete: status === "complete",
|
|
816
|
+
...(reason ? { reason } : {}),
|
|
817
|
+
queryUrl,
|
|
818
|
+
queryHash: expectedQueryHash,
|
|
819
|
+
checkpointPath: path.resolve(options.checkpointPath),
|
|
820
|
+
rawJsonlPath: path.resolve(options.rawJsonlPath),
|
|
821
|
+
outPath: status === "complete" ? path.resolve(options.outPath) : null,
|
|
822
|
+
pagesAccepted: checkpoint.pagesAccepted,
|
|
823
|
+
productsCollected: checkpoint.products.length,
|
|
824
|
+
nextPageNumber: checkpoint.nextPageNumber,
|
|
825
|
+
...(checkpoint.orderChecksum ? { orderChecksum: checkpoint.orderChecksum } : {}),
|
|
826
|
+
...(checkpoint.terminalPage ? { terminalPage: checkpoint.terminalPage } : {})
|
|
827
|
+
});
|
|
828
|
+
const finish = async (status, reason) => {
|
|
829
|
+
if (settled)
|
|
830
|
+
return;
|
|
831
|
+
settled = true;
|
|
832
|
+
acceptingRequests = false;
|
|
833
|
+
if (idleTimer)
|
|
834
|
+
clearTimeout(idleTimer);
|
|
835
|
+
process.off("SIGINT", handleSigint);
|
|
836
|
+
process.off("SIGTERM", handleSigterm);
|
|
837
|
+
await closeServer(server);
|
|
838
|
+
resolveResult?.(makeResult(status, reason));
|
|
839
|
+
};
|
|
840
|
+
const abortWithError = async (error) => {
|
|
841
|
+
if (settled)
|
|
842
|
+
return;
|
|
843
|
+
settled = true;
|
|
844
|
+
acceptingRequests = false;
|
|
845
|
+
if (idleTimer)
|
|
846
|
+
clearTimeout(idleTimer);
|
|
847
|
+
process.off("SIGINT", handleSigint);
|
|
848
|
+
process.off("SIGTERM", handleSigterm);
|
|
849
|
+
await closeServer(server);
|
|
850
|
+
rejectResult?.(error);
|
|
851
|
+
};
|
|
852
|
+
const stopIncomplete = (reason, message) => {
|
|
853
|
+
acceptingRequests = false;
|
|
854
|
+
if (incompleteStopPromise)
|
|
855
|
+
return incompleteStopPromise;
|
|
856
|
+
incompleteStopPromise = (async () => {
|
|
857
|
+
if (settled)
|
|
858
|
+
return;
|
|
859
|
+
const now = new Date().toISOString();
|
|
860
|
+
checkpoint = {
|
|
861
|
+
...checkpoint,
|
|
862
|
+
updatedAt: now,
|
|
863
|
+
lastStop: {
|
|
864
|
+
reason,
|
|
865
|
+
at: now,
|
|
866
|
+
pageNumber: checkpoint.nextPageNumber,
|
|
867
|
+
...(message ? { message } : {})
|
|
868
|
+
}
|
|
869
|
+
};
|
|
870
|
+
await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
|
|
871
|
+
await finish("incomplete", reason);
|
|
872
|
+
})();
|
|
873
|
+
return incompleteStopPromise;
|
|
874
|
+
};
|
|
875
|
+
const requestIncompleteStop = (reason, message) => {
|
|
876
|
+
acceptingRequests = false;
|
|
877
|
+
return enqueueExclusive(async () => {
|
|
878
|
+
if (settled)
|
|
879
|
+
return;
|
|
880
|
+
if (checkpoint.status === "complete") {
|
|
881
|
+
await finish("complete");
|
|
882
|
+
return;
|
|
883
|
+
}
|
|
884
|
+
await stopIncomplete(reason, message);
|
|
885
|
+
});
|
|
886
|
+
};
|
|
887
|
+
function resetIdleTimer() {
|
|
888
|
+
if (idleTimer)
|
|
889
|
+
clearTimeout(idleTimer);
|
|
890
|
+
idleTimer = setTimeout(() => {
|
|
891
|
+
void requestIncompleteStop("idle_timeout", "No browser relay activity arrived before the local timeout.").catch((error) => {
|
|
892
|
+
rejectResult?.(error);
|
|
893
|
+
});
|
|
894
|
+
}, options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS);
|
|
895
|
+
idleTimer.unref();
|
|
896
|
+
}
|
|
897
|
+
async function handleRequest(req, res) {
|
|
898
|
+
if (!acceptingRequests) {
|
|
899
|
+
writeRelayJson(res, 410, { status: "closed" }, req.headers.origin);
|
|
900
|
+
return;
|
|
901
|
+
}
|
|
902
|
+
if (!isAllowedRelayOrigin(req)) {
|
|
903
|
+
writeRelayJson(res, 403, { status: "rejected", error: "Relay origin is not allowed." });
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
resetIdleTimer();
|
|
907
|
+
const requestUrl = new URL(req.url ?? "/", "http://127.0.0.1");
|
|
908
|
+
if (req.method === "OPTIONS") {
|
|
909
|
+
res.writeHead(204, {
|
|
910
|
+
"Access-Control-Allow-Origin": "https://www.linkedin.com",
|
|
911
|
+
"Access-Control-Allow-Methods": "GET, POST, OPTIONS",
|
|
912
|
+
"Access-Control-Allow-Headers": "Content-Type",
|
|
913
|
+
"Access-Control-Allow-Private-Network": "true",
|
|
914
|
+
"Access-Control-Max-Age": "600",
|
|
915
|
+
Vary: "Origin"
|
|
916
|
+
});
|
|
917
|
+
res.end();
|
|
918
|
+
return;
|
|
919
|
+
}
|
|
920
|
+
if (req.method === "GET" && requestUrl.pathname === "/health") {
|
|
921
|
+
writeRelayJson(res, 200, { status: "ok", queryHash: expectedQueryHash }, req.headers.origin);
|
|
922
|
+
return;
|
|
923
|
+
}
|
|
924
|
+
if (req.method === "GET" && requestUrl.pathname === "/task") {
|
|
925
|
+
if (options.maxPages !== undefined && checkpoint.nextPageNumber > options.maxPages) {
|
|
926
|
+
acceptingRequests = false;
|
|
927
|
+
writeRelayJson(res, 200, { status: "incomplete", reason: "page_limit" }, req.headers.origin);
|
|
928
|
+
res.once("finish", () => void requestIncompleteStop("page_limit"));
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
931
|
+
if (options.maxRequests !== undefined &&
|
|
932
|
+
uniqueTasksIssuedThisRun >= options.maxRequests &&
|
|
933
|
+
lastIssuedPage !== checkpoint.nextPageNumber) {
|
|
934
|
+
acceptingRequests = false;
|
|
935
|
+
writeRelayJson(res, 200, { status: "incomplete", reason: "request_limit" }, req.headers.origin);
|
|
936
|
+
res.once("finish", () => void requestIncompleteStop("request_limit"));
|
|
937
|
+
return;
|
|
938
|
+
}
|
|
939
|
+
if (lastIssuedPage !== checkpoint.nextPageNumber) {
|
|
940
|
+
uniqueTasksIssuedThisRun += 1;
|
|
941
|
+
lastIssuedPage = checkpoint.nextPageNumber;
|
|
942
|
+
checkpoint = { ...checkpoint, tasksIssued: checkpoint.tasksIssued + 1, updatedAt: new Date().toISOString() };
|
|
943
|
+
await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
|
|
944
|
+
}
|
|
945
|
+
writeRelayJson(res, 200, {
|
|
946
|
+
status: "task",
|
|
947
|
+
queryHash: expectedQueryHash,
|
|
948
|
+
pageNumber: checkpoint.nextPageNumber,
|
|
949
|
+
currentUrl: buildLinkedInProductSearchPageUrl(queryUrl, checkpoint.nextPageNumber),
|
|
950
|
+
productsAccepted: checkpoint.products.length
|
|
951
|
+
}, req.headers.origin);
|
|
952
|
+
return;
|
|
953
|
+
}
|
|
954
|
+
if (req.method === "POST" && requestUrl.pathname === "/page") {
|
|
955
|
+
const submission = linkedInProductPageSubmissionSchema.parse(await readRelayJson(req));
|
|
956
|
+
if (submission.pageNumber !== checkpoint.nextPageNumber) {
|
|
957
|
+
throw new LinkedInProductCollectorInvariantError("out_of_order_page", `Expected page ${checkpoint.nextPageNumber}, received page ${submission.pageNumber}.`);
|
|
958
|
+
}
|
|
959
|
+
if (lastIssuedPage !== submission.pageNumber) {
|
|
960
|
+
throw new LinkedInProductCollectorInvariantError("unleased_page", "Poll GET /task before posting the expected browser page.");
|
|
961
|
+
}
|
|
962
|
+
if ("error" in submission) {
|
|
963
|
+
acceptingRequests = false;
|
|
964
|
+
const sanitizedCurrentUrl = sanitizeTerminalErrorUrl(submission.currentUrl);
|
|
965
|
+
const receivedAt = new Date().toISOString();
|
|
966
|
+
await appendPrivateJsonLine(options.rawJsonlPath, {
|
|
967
|
+
type: "terminal_error",
|
|
968
|
+
queryHash: expectedQueryHash,
|
|
969
|
+
pageNumber: submission.pageNumber,
|
|
970
|
+
currentUrl: sanitizedCurrentUrl,
|
|
971
|
+
error: { code: submission.error.code },
|
|
972
|
+
receivedAt
|
|
973
|
+
});
|
|
974
|
+
writeRelayJson(res, 202, { status: "checkpointed", complete: false }, req.headers.origin);
|
|
975
|
+
res.once("finish", () => {
|
|
976
|
+
void requestIncompleteStop(submission.error.code, terminalStopMessage(submission.error.code)).catch((error) => rejectResult?.(error));
|
|
977
|
+
});
|
|
978
|
+
return;
|
|
979
|
+
}
|
|
980
|
+
const sanitizedCurrentUrl = assertExpectedLinkedInSearchPage(submission.currentUrl, checkpoint.queryUrl, submission.pageNumber);
|
|
981
|
+
let applied;
|
|
982
|
+
try {
|
|
983
|
+
applied = applyLinkedInProductSearchPage(checkpoint, submission);
|
|
984
|
+
}
|
|
985
|
+
catch (error) {
|
|
986
|
+
if (error instanceof LinkedInProductCollectorInvariantError && error.code === "identity_drift") {
|
|
987
|
+
acceptingRequests = false;
|
|
988
|
+
const now = new Date().toISOString();
|
|
989
|
+
checkpoint = {
|
|
990
|
+
...checkpoint,
|
|
991
|
+
status: "failed",
|
|
992
|
+
updatedAt: now,
|
|
993
|
+
lastStop: { reason: error.code, at: now, pageNumber: checkpoint.nextPageNumber, message: error.message }
|
|
994
|
+
};
|
|
995
|
+
await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
|
|
996
|
+
writeRelayJson(res, 409, { status: "rejected", complete: false, error: error.message }, req.headers.origin);
|
|
997
|
+
res.once("finish", () => void finish("incomplete", error.code));
|
|
998
|
+
return;
|
|
999
|
+
}
|
|
1000
|
+
throw error;
|
|
1001
|
+
}
|
|
1002
|
+
await appendPrivateJsonLine(options.rawJsonlPath, {
|
|
1003
|
+
type: "page",
|
|
1004
|
+
queryHash: expectedQueryHash,
|
|
1005
|
+
pageNumber: submission.pageNumber,
|
|
1006
|
+
currentUrl: sanitizedCurrentUrl,
|
|
1007
|
+
reportedTotal: submission.reportedTotal ?? null,
|
|
1008
|
+
hasNext: submission.hasNext,
|
|
1009
|
+
items: submission.items.map(rawVisibleProductCard),
|
|
1010
|
+
visibleItemCount: submission.items.length,
|
|
1011
|
+
newProductCount: applied.newProductCount,
|
|
1012
|
+
duplicateCount: applied.duplicateCount,
|
|
1013
|
+
receivedAt: applied.checkpoint.updatedAt
|
|
1014
|
+
});
|
|
1015
|
+
checkpoint = applied.checkpoint;
|
|
1016
|
+
await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
|
|
1017
|
+
if (checkpoint.status === "complete") {
|
|
1018
|
+
const artifact = buildLinkedInProductCollectionArtifact(checkpoint);
|
|
1019
|
+
await writePrivateJsonAtomic(options.outPath, artifact);
|
|
1020
|
+
acceptingRequests = false;
|
|
1021
|
+
}
|
|
1022
|
+
lastIssuedPage = null;
|
|
1023
|
+
writeRelayJson(res, 202, {
|
|
1024
|
+
status: checkpoint.status === "complete" ? "complete" : "accepted",
|
|
1025
|
+
complete: checkpoint.status === "complete",
|
|
1026
|
+
pageNumber: submission.pageNumber,
|
|
1027
|
+
newProductCount: applied.newProductCount,
|
|
1028
|
+
duplicateCount: applied.duplicateCount,
|
|
1029
|
+
productsAccepted: checkpoint.products.length,
|
|
1030
|
+
nextPageNumber: checkpoint.nextPageNumber
|
|
1031
|
+
}, req.headers.origin);
|
|
1032
|
+
if (checkpoint.status === "complete") {
|
|
1033
|
+
res.once("finish", () => void finish("complete"));
|
|
1034
|
+
}
|
|
1035
|
+
else if (options.maxPages !== undefined && checkpoint.nextPageNumber > options.maxPages) {
|
|
1036
|
+
acceptingRequests = false;
|
|
1037
|
+
res.once("finish", () => void requestIncompleteStop("page_limit"));
|
|
1038
|
+
}
|
|
1039
|
+
else if (options.maxRequests !== undefined && uniqueTasksIssuedThisRun >= options.maxRequests) {
|
|
1040
|
+
acceptingRequests = false;
|
|
1041
|
+
res.once("finish", () => void requestIncompleteStop("request_limit"));
|
|
1042
|
+
}
|
|
1043
|
+
return;
|
|
1044
|
+
}
|
|
1045
|
+
writeRelayJson(res, 404, { status: "not_found" }, req.headers.origin);
|
|
1046
|
+
}
|
|
1047
|
+
const handleSigint = () => {
|
|
1048
|
+
void requestIncompleteStop("interrupted", "The collector received SIGINT.").catch((error) => rejectResult?.(error));
|
|
1049
|
+
};
|
|
1050
|
+
const handleSigterm = () => {
|
|
1051
|
+
void requestIncompleteStop("interrupted", "The collector received SIGTERM.").catch((error) => rejectResult?.(error));
|
|
1052
|
+
};
|
|
1053
|
+
process.once("SIGINT", handleSigint);
|
|
1054
|
+
process.once("SIGTERM", handleSigterm);
|
|
1055
|
+
try {
|
|
1056
|
+
await new Promise((resolve, reject) => {
|
|
1057
|
+
server.once("error", reject);
|
|
1058
|
+
server.listen(options.relayPort, "127.0.0.1", () => resolve());
|
|
1059
|
+
});
|
|
1060
|
+
resetIdleTimer();
|
|
1061
|
+
const address = server.address();
|
|
1062
|
+
if (!address || typeof address === "string")
|
|
1063
|
+
throw new Error("The browser relay did not bind to TCP.");
|
|
1064
|
+
options.onListening?.({
|
|
1065
|
+
host: "127.0.0.1",
|
|
1066
|
+
port: address.port,
|
|
1067
|
+
taskUrl: `http://127.0.0.1:${address.port}/task`,
|
|
1068
|
+
pageUrl: `http://127.0.0.1:${address.port}/page`
|
|
1069
|
+
});
|
|
1070
|
+
return await resultPromise;
|
|
1071
|
+
}
|
|
1072
|
+
catch (error) {
|
|
1073
|
+
if (idleTimer)
|
|
1074
|
+
clearTimeout(idleTimer);
|
|
1075
|
+
process.off("SIGINT", handleSigint);
|
|
1076
|
+
process.off("SIGTERM", handleSigterm);
|
|
1077
|
+
if (server.listening)
|
|
1078
|
+
await closeServer(server);
|
|
1079
|
+
throw error;
|
|
1080
|
+
}
|
|
1081
|
+
}
|
package/package.json
CHANGED