salesprompter-cli 0.1.65 → 0.1.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/dist/cli.js +97 -2
- package/dist/linkedin-product-search.js +1081 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -73,6 +73,11 @@ salesprompter contacts:resolve-emails --in ./contacts.tsv --out-dir ./email-run
|
|
|
73
73
|
salesprompter leads:collect \
|
|
74
74
|
--linkedin-url "$SALES_NAV_PEOPLE_URL"
|
|
75
75
|
|
|
76
|
+
# For large discovery runs, store the leads now and enrich selected imports later
|
|
77
|
+
salesprompter leads:collect \
|
|
78
|
+
--linkedin-url "$SALES_NAV_PEOPLE_URL" \
|
|
79
|
+
--defer-email-enrichment
|
|
80
|
+
|
|
76
81
|
# Enrich one or more stored imports: Sales Navigator company -> official domain -> Hunter email
|
|
77
82
|
# This updates workspace data only. It does not create or start outreach.
|
|
78
83
|
# Stale ids and raw legal-name misses use conservative exact-name recovery.
|
|
@@ -120,6 +125,19 @@ salesprompter packs:list
|
|
|
120
125
|
salesprompter --help
|
|
121
126
|
```
|
|
122
127
|
|
|
128
|
+
Collect an exact signed-in LinkedIn product-search order locally, without uploading it:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
salesprompter products:collect \
|
|
132
|
+
--query-url "$LINKEDIN_PRODUCTS_URL" \
|
|
133
|
+
--checkpoint ./data/products.checkpoint.json \
|
|
134
|
+
--raw-jsonl ./data/products.raw.jsonl \
|
|
135
|
+
--out ./data/products.complete.json \
|
|
136
|
+
--relay-port 43117
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Use a signed-in in-app-browser worker with the loopback `GET /task` and `POST /page` protocol. Interrupted, bounded, challenged, or rate-limited runs remain checkpointed and do not create the complete artifact.
|
|
140
|
+
|
|
123
141
|
Download stored workspace leads without starting a new Sales Navigator scrape:
|
|
124
142
|
|
|
125
143
|
```bash
|
package/dist/cli.js
CHANGED
|
@@ -28,6 +28,7 @@ import { buildHistoricalVendorIcp, buildVendorIcp } from "./icp-templates.js";
|
|
|
28
28
|
import { InstantlySyncProvider } from "./instantly.js";
|
|
29
29
|
import { backfillLinkedInCompanies } from "./linkedin-companies.js";
|
|
30
30
|
import { parseLinkedInCompanyPage } from "./linkedin-companies.js";
|
|
31
|
+
import { collectLinkedInProductsViaBrowserRelay } from "./linkedin-product-search.js";
|
|
31
32
|
import { crawlLinkedInProductCategory } from "./linkedin-products.js";
|
|
32
33
|
import { claimLinkedInSessionCookieForCli, claimValidatedSalesNavigatorSessionCookieForCli, createLinkedInSessionSupabaseClient, recordLinkedInSessionCookieAudit, resolveConfiguredEnvValue } from "./linkedin-session.js";
|
|
33
34
|
import { buildLeadlistsFunnelQueries } from "./leadlists-funnel.js";
|
|
@@ -296,6 +297,23 @@ const SalesNavigatorLocalImportResponseSchema = z.object({
|
|
|
296
297
|
})
|
|
297
298
|
.nullable()
|
|
298
299
|
.optional(),
|
|
300
|
+
emailEnrichment: z
|
|
301
|
+
.object({
|
|
302
|
+
scheduled: z.boolean(),
|
|
303
|
+
deferred: z.boolean().optional(),
|
|
304
|
+
provider: z.string().min(1).optional(),
|
|
305
|
+
fallbackProvider: z.string().min(1).optional(),
|
|
306
|
+
})
|
|
307
|
+
.optional(),
|
|
308
|
+
enrichmentDeferred: z.boolean().optional(),
|
|
309
|
+
outreachStarted: z.literal(false).optional(),
|
|
310
|
+
});
|
|
311
|
+
const SalesNavigatorLocalImportCapabilitiesResponseSchema = z.object({
|
|
312
|
+
status: z.literal("ok"),
|
|
313
|
+
capabilities: z.object({
|
|
314
|
+
deferredEmailEnrichment: z.literal(true),
|
|
315
|
+
}),
|
|
316
|
+
outreachStarted: z.literal(false),
|
|
299
317
|
});
|
|
300
318
|
const PhantombusterContainersSyncResponseSchema = z.object({
|
|
301
319
|
status: z.literal("ok"),
|
|
@@ -581,7 +599,7 @@ const cliPacks = [
|
|
|
581
599
|
slug: "research",
|
|
582
600
|
title: "Research",
|
|
583
601
|
summary: "Scrape markets and enrich companies before outreach.",
|
|
584
|
-
commands: ["market:scrape", "companies:enrich"],
|
|
602
|
+
commands: ["products:collect", "market:scrape", "companies:enrich"],
|
|
585
603
|
installStatus: "included"
|
|
586
604
|
},
|
|
587
605
|
{
|
|
@@ -631,6 +649,7 @@ const helpAliasByCommandName = new Map([
|
|
|
631
649
|
["linkedin-companies:backfill", "companies:enrich"],
|
|
632
650
|
["linkedin-companies:scrape-local", "companies:scrape-linkedin"],
|
|
633
651
|
["dealroom-companies:scrape-local", "companies:scrape-dealroom"],
|
|
652
|
+
["linkedin-products:collect", "products:collect"],
|
|
634
653
|
["linkedin-products:scrape", "market:scrape"],
|
|
635
654
|
["salesnav:from-product-category", "leads:discover"],
|
|
636
655
|
["salesnav:people:collect", "leads:collect"],
|
|
@@ -681,6 +700,7 @@ const helpVisibleCommandNames = new Set([
|
|
|
681
700
|
"linkedin-companies:backfill",
|
|
682
701
|
"linkedin-companies:scrape-local",
|
|
683
702
|
"dealroom-companies:scrape-local",
|
|
703
|
+
"linkedin-products:collect",
|
|
684
704
|
"linkedin-products:scrape",
|
|
685
705
|
"salesnav:from-product-category",
|
|
686
706
|
"salesnav:people:collect",
|
|
@@ -9275,6 +9295,9 @@ async function importLocalSalesNavigatorPeopleViaApp(session, payload) {
|
|
|
9275
9295
|
totalResults: payload.totalResults,
|
|
9276
9296
|
fetchedPages: payload.fetchedPages,
|
|
9277
9297
|
people: payload.people,
|
|
9298
|
+
...(payload.startEmailEnrichment === false
|
|
9299
|
+
? { startEmailEnrichment: false }
|
|
9300
|
+
: {}),
|
|
9278
9301
|
rawPayload: {
|
|
9279
9302
|
...payload.rawPayload,
|
|
9280
9303
|
clientId: workspaceClientId ?? undefined,
|
|
@@ -9305,6 +9328,20 @@ async function importLocalSalesNavigatorPeopleViaApp(session, payload) {
|
|
|
9305
9328
|
}, SalesNavigatorLocalImportResponseSchema);
|
|
9306
9329
|
return value;
|
|
9307
9330
|
}
|
|
9331
|
+
async function requireDeferredEmailEnrichmentCapability(session) {
|
|
9332
|
+
try {
|
|
9333
|
+
const result = await fetchCliJson(session, (currentSession) => fetch(`${currentSession.apiBaseUrl}/api/cli/salesnav/local-import`, {
|
|
9334
|
+
method: "GET",
|
|
9335
|
+
headers: {
|
|
9336
|
+
Authorization: `Bearer ${currentSession.accessToken}`,
|
|
9337
|
+
},
|
|
9338
|
+
}), SalesNavigatorLocalImportCapabilitiesResponseSchema);
|
|
9339
|
+
return result.session;
|
|
9340
|
+
}
|
|
9341
|
+
catch {
|
|
9342
|
+
throw new Error("This Salesprompter workspace cannot safely defer email enrichment yet. Update the app before collecting leads with --defer-email-enrichment.");
|
|
9343
|
+
}
|
|
9344
|
+
}
|
|
9308
9345
|
const SALESPROMPTER_SUPABASE_TABLES = {
|
|
9309
9346
|
accountSearchRuns: "salesprompter_linkedin_sales_nav_account_search_runs",
|
|
9310
9347
|
accountSearchSlices: "salesprompter_linkedin_sales_nav_account_search_slices",
|
|
@@ -13813,6 +13850,7 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
|
|
|
13813
13850
|
.max(accessibleResultLimit)
|
|
13814
13851
|
.parse(options.maxResults);
|
|
13815
13852
|
const complete = Boolean(options.complete || options.maxResults == null);
|
|
13853
|
+
const emailEnrichmentDeferred = Boolean(options.deferEmailEnrichment);
|
|
13816
13854
|
const checkpointPath = complete
|
|
13817
13855
|
? options.checkpoint
|
|
13818
13856
|
? path.resolve(options.checkpoint)
|
|
@@ -13827,13 +13865,18 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
|
|
|
13827
13865
|
accessibleResultLimit,
|
|
13828
13866
|
complete,
|
|
13829
13867
|
resumable: complete,
|
|
13868
|
+
emailEnrichmentDeferred,
|
|
13869
|
+
outreachStarted: false,
|
|
13830
13870
|
destination: "/leads/cli-imports"
|
|
13831
13871
|
};
|
|
13832
13872
|
if (options.out)
|
|
13833
13873
|
await writeJsonFile(options.out, payload);
|
|
13834
13874
|
return payload;
|
|
13835
13875
|
}
|
|
13836
|
-
|
|
13876
|
+
let session = await requireAuthSession();
|
|
13877
|
+
if (emailEnrichmentDeferred) {
|
|
13878
|
+
session = await requireDeferredEmailEnrichmentCapability(session);
|
|
13879
|
+
}
|
|
13837
13880
|
const pageDelayMinMs = z.coerce.number().int().min(0).parse(options.pageDelayMinMs);
|
|
13838
13881
|
const pageDelayMaxMs = z.coerce.number().int().min(pageDelayMinMs).parse(options.pageDelayMaxMs);
|
|
13839
13882
|
const browserRelayPort = options.browserRelayPort == null
|
|
@@ -13899,6 +13942,9 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
|
|
|
13899
13942
|
slicePreset: adaptiveCollection
|
|
13900
13943
|
? "local-salesnav-people-adaptive"
|
|
13901
13944
|
: "local-salesnav-people",
|
|
13945
|
+
...(emailEnrichmentDeferred
|
|
13946
|
+
? { startEmailEnrichment: false }
|
|
13947
|
+
: {}),
|
|
13902
13948
|
rawPayload: adaptiveCollection
|
|
13903
13949
|
? {
|
|
13904
13950
|
collectionMode: adaptiveCollection.collectionMode,
|
|
@@ -13914,6 +13960,13 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
|
|
|
13914
13960
|
accessibleResultLimit,
|
|
13915
13961
|
},
|
|
13916
13962
|
});
|
|
13963
|
+
if (emailEnrichmentDeferred &&
|
|
13964
|
+
!(imported.enrichmentDeferred === true &&
|
|
13965
|
+
imported.emailEnrichment?.deferred === true &&
|
|
13966
|
+
imported.emailEnrichment.scheduled === false &&
|
|
13967
|
+
imported.outreachStarted === false)) {
|
|
13968
|
+
throw new Error(`Import ${imported.runId} was stored, but the workspace did not confirm that email enrichment was deferred. Do not repeat the collection until that run's enrichment state is verified.`);
|
|
13969
|
+
}
|
|
13917
13970
|
const payload = {
|
|
13918
13971
|
status: "ok",
|
|
13919
13972
|
dryRun: false,
|
|
@@ -13942,6 +13995,10 @@ async function runSalesNavigatorPeopleCollectCommand(options) {
|
|
|
13942
13995
|
runId: imported.runId,
|
|
13943
13996
|
imported: imported.imported,
|
|
13944
13997
|
upserted: imported.upserted,
|
|
13998
|
+
emailEnrichmentDeferred: imported.enrichmentDeferred ??
|
|
13999
|
+
imported.emailEnrichment?.deferred ??
|
|
14000
|
+
false,
|
|
14001
|
+
outreachStarted: imported.outreachStarted ?? false,
|
|
13945
14002
|
workspaceUrl: `${session.apiBaseUrl}/leads/cli-imports?runId=` +
|
|
13946
14003
|
encodeURIComponent(imported.runId)
|
|
13947
14004
|
};
|
|
@@ -14807,6 +14864,7 @@ program
|
|
|
14807
14864
|
.option("--max-results <number>", "Bounded collection size; defaults to exhaustive collection with a 2500-person window")
|
|
14808
14865
|
.option("--complete", "Split oversized searches by company headcount and deduplicate the complete result", false)
|
|
14809
14866
|
.option("--checkpoint <path>", "Optional resume checkpoint; exhaustive runs use a per-query checkpoint by default")
|
|
14867
|
+
.option("--defer-email-enrichment", "Store leads without starting email enrichment", false)
|
|
14810
14868
|
.option("--curl-file <path>", "Optional copied Sales Navigator Lead Search curl request")
|
|
14811
14869
|
.option("--browser-relay-port <number>", "Use the signed-in browser through a loopback relay")
|
|
14812
14870
|
.option("--page-size <number>", "Direct Sales Navigator page size", "100")
|
|
@@ -15128,6 +15186,43 @@ program
|
|
|
15128
15186
|
profile,
|
|
15129
15187
|
});
|
|
15130
15188
|
});
|
|
15189
|
+
program
|
|
15190
|
+
.command("linkedin-products:collect")
|
|
15191
|
+
.alias("products:collect")
|
|
15192
|
+
.description("Collect an exact LinkedIn product-search order locally through a loopback browser relay.")
|
|
15193
|
+
.requiredOption("--query-url <url>", "Exact LinkedIn /search/results/products/ query URL")
|
|
15194
|
+
.requiredOption("--checkpoint <path>", "Private resumable checkpoint JSON path")
|
|
15195
|
+
.requiredOption("--raw-jsonl <path>", "Private append-only normalized page evidence path")
|
|
15196
|
+
.requiredOption("--out <path>", "Complete public-fields-only ranked artifact path")
|
|
15197
|
+
.requiredOption("--relay-port <number>", "Loopback relay port for GET /task and POST /page")
|
|
15198
|
+
.option("--max-pages <number>", "Optional highest page number to accept in this run")
|
|
15199
|
+
.option("--max-requests <number>", "Optional number of unique browser tasks to issue in this run")
|
|
15200
|
+
.action(async (options) => {
|
|
15201
|
+
const relayPort = z.coerce.number().int().min(1).max(65_535).parse(options.relayPort);
|
|
15202
|
+
const maxPages = options.maxPages === undefined
|
|
15203
|
+
? undefined
|
|
15204
|
+
: z.coerce.number().int().min(1).max(100_000).parse(options.maxPages);
|
|
15205
|
+
const maxRequests = options.maxRequests === undefined
|
|
15206
|
+
? undefined
|
|
15207
|
+
: z.coerce.number().int().min(1).max(100_000).parse(options.maxRequests);
|
|
15208
|
+
const idleTimeoutMs = process.env.SALESPROMPTER_LINKEDIN_PRODUCTS_RELAY_IDLE_TIMEOUT_MS
|
|
15209
|
+
? z.coerce.number().int().min(100).max(86_400_000).parse(process.env.SALESPROMPTER_LINKEDIN_PRODUCTS_RELAY_IDLE_TIMEOUT_MS)
|
|
15210
|
+
: undefined;
|
|
15211
|
+
const result = await collectLinkedInProductsViaBrowserRelay({
|
|
15212
|
+
queryUrl: String(options.queryUrl),
|
|
15213
|
+
checkpointPath: path.resolve(String(options.checkpoint)),
|
|
15214
|
+
rawJsonlPath: path.resolve(String(options.rawJsonl)),
|
|
15215
|
+
outPath: path.resolve(String(options.out)),
|
|
15216
|
+
relayPort,
|
|
15217
|
+
maxPages,
|
|
15218
|
+
maxRequests,
|
|
15219
|
+
idleTimeoutMs,
|
|
15220
|
+
onListening: ({ taskUrl, pageUrl }) => {
|
|
15221
|
+
writeProgress(`LinkedIn product relay ready: GET ${taskUrl} and POST ${pageUrl}`);
|
|
15222
|
+
}
|
|
15223
|
+
});
|
|
15224
|
+
printOutput(result);
|
|
15225
|
+
});
|
|
15131
15226
|
program
|
|
15132
15227
|
.command("linkedin-products:scrape")
|
|
15133
15228
|
.alias("market:scrape")
|
|
@@ -0,0 +1,1081 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { access, appendFile, chmod, mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
const CHECKPOINT_SCHEMA_VERSION = 1;
|
|
7
|
+
const ARTIFACT_SCHEMA_VERSION = 1;
|
|
8
|
+
const MAX_RELAY_BODY_BYTES = 5 * 1024 * 1024;
|
|
9
|
+
const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
|
|
10
|
+
const nonEmptyText = (max) => z.string().trim().min(1).max(max);
|
|
11
|
+
const optionalPublicUrl = z.string().trim().url().max(4_096).optional();
|
|
12
|
+
export const visibleLinkedInProductCardSchema = z
|
|
13
|
+
.object({
|
|
14
|
+
identity: z.string().trim().min(1).max(256).optional(),
|
|
15
|
+
name: nonEmptyText(500),
|
|
16
|
+
linkedinUrl: nonEmptyText(4_096),
|
|
17
|
+
imageUrl: optionalPublicUrl,
|
|
18
|
+
vendor: nonEmptyText(500).optional(),
|
|
19
|
+
category: nonEmptyText(500).optional(),
|
|
20
|
+
description: nonEmptyText(10_000).optional()
|
|
21
|
+
})
|
|
22
|
+
.strict();
|
|
23
|
+
const pageSuccessSchema = z
|
|
24
|
+
.object({
|
|
25
|
+
pageNumber: z.number().int().min(1),
|
|
26
|
+
currentUrl: nonEmptyText(8_192),
|
|
27
|
+
reportedTotal: z.number().int().min(0).nullable().optional(),
|
|
28
|
+
hasNext: z.boolean(),
|
|
29
|
+
items: z.array(visibleLinkedInProductCardSchema).max(1_000)
|
|
30
|
+
})
|
|
31
|
+
.strict();
|
|
32
|
+
const terminalErrorSchema = z
|
|
33
|
+
.object({
|
|
34
|
+
pageNumber: z.number().int().min(1),
|
|
35
|
+
currentUrl: nonEmptyText(8_192),
|
|
36
|
+
error: z
|
|
37
|
+
.object({
|
|
38
|
+
code: z.enum([
|
|
39
|
+
"auth_required",
|
|
40
|
+
"challenge",
|
|
41
|
+
"rate_limited",
|
|
42
|
+
"interrupted",
|
|
43
|
+
"unexpected_page"
|
|
44
|
+
]),
|
|
45
|
+
message: z.string().trim().max(1_000).optional()
|
|
46
|
+
})
|
|
47
|
+
.strict()
|
|
48
|
+
})
|
|
49
|
+
.strict();
|
|
50
|
+
export const linkedInProductPageSubmissionSchema = z.union([pageSuccessSchema, terminalErrorSchema]);
|
|
51
|
+
const rawVisibleProductCardSchema = visibleLinkedInProductCardSchema
|
|
52
|
+
.extend({ linkedinSlug: nonEmptyText(500) })
|
|
53
|
+
.strict();
|
|
54
|
+
const rawPageEvidenceSchema = z
|
|
55
|
+
.object({
|
|
56
|
+
type: z.literal("page"),
|
|
57
|
+
queryHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
58
|
+
pageNumber: z.number().int().min(1),
|
|
59
|
+
currentUrl: nonEmptyText(8_192),
|
|
60
|
+
reportedTotal: z.number().int().min(0).nullable(),
|
|
61
|
+
hasNext: z.boolean(),
|
|
62
|
+
items: z.array(rawVisibleProductCardSchema).max(1_000),
|
|
63
|
+
visibleItemCount: z.number().int().min(0),
|
|
64
|
+
newProductCount: z.number().int().min(0),
|
|
65
|
+
duplicateCount: z.number().int().min(0),
|
|
66
|
+
receivedAt: nonEmptyText(100)
|
|
67
|
+
})
|
|
68
|
+
.strict();
|
|
69
|
+
const rawTerminalEvidenceSchema = z
|
|
70
|
+
.object({
|
|
71
|
+
type: z.literal("terminal_error"),
|
|
72
|
+
queryHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
73
|
+
pageNumber: z.number().int().min(1),
|
|
74
|
+
currentUrl: nonEmptyText(8_192),
|
|
75
|
+
error: z.object({ code: terminalErrorSchema.shape.error.shape.code }).strict(),
|
|
76
|
+
receivedAt: nonEmptyText(100)
|
|
77
|
+
})
|
|
78
|
+
.strict();
|
|
79
|
+
const rawEvidenceRecordSchema = z.union([rawPageEvidenceSchema, rawTerminalEvidenceSchema]);
|
|
80
|
+
export class LinkedInProductCollectorInvariantError extends Error {
|
|
81
|
+
code;
|
|
82
|
+
constructor(code, message) {
|
|
83
|
+
super(message);
|
|
84
|
+
this.code = code;
|
|
85
|
+
this.name = "LinkedInProductCollectorInvariantError";
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
function sha256(value) {
|
|
89
|
+
return createHash("sha256").update(value, "utf8").digest("hex");
|
|
90
|
+
}
|
|
91
|
+
function cleanText(value) {
|
|
92
|
+
const cleaned = value?.replace(/\s+/g, " ").trim();
|
|
93
|
+
return cleaned || undefined;
|
|
94
|
+
}
|
|
95
|
+
function canonicalizePublicUrl(value) {
|
|
96
|
+
if (!value)
|
|
97
|
+
return undefined;
|
|
98
|
+
const parsed = new URL(value);
|
|
99
|
+
if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
|
|
100
|
+
throw new LinkedInProductCollectorInvariantError("invalid_public_url", `Unsupported public URL: ${value}`);
|
|
101
|
+
}
|
|
102
|
+
parsed.username = "";
|
|
103
|
+
parsed.password = "";
|
|
104
|
+
parsed.hash = "";
|
|
105
|
+
return parsed.toString();
|
|
106
|
+
}
|
|
107
|
+
export function normalizeLinkedInProductSearchUrl(input) {
|
|
108
|
+
let parsed;
|
|
109
|
+
try {
|
|
110
|
+
parsed = new URL(input);
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
throw new LinkedInProductCollectorInvariantError("invalid_query_url", "The query URL is not a valid URL.");
|
|
114
|
+
}
|
|
115
|
+
if (parsed.protocol !== "https:" ||
|
|
116
|
+
parsed.port !== "" ||
|
|
117
|
+
parsed.username !== "" ||
|
|
118
|
+
parsed.password !== "" ||
|
|
119
|
+
!["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
|
|
120
|
+
throw new LinkedInProductCollectorInvariantError("invalid_query_url", "The query URL must use https://www.linkedin.com.");
|
|
121
|
+
}
|
|
122
|
+
if (parsed.pathname !== "/search/results/products" && parsed.pathname !== "/search/results/products/") {
|
|
123
|
+
throw new LinkedInProductCollectorInvariantError("invalid_query_url", "The query URL must point to /search/results/products/.");
|
|
124
|
+
}
|
|
125
|
+
const suppliedPages = parsed.searchParams.getAll("page");
|
|
126
|
+
const suppliedPage = suppliedPages[0];
|
|
127
|
+
if (suppliedPages.length > 1 || (suppliedPage && suppliedPage !== "1")) {
|
|
128
|
+
throw new LinkedInProductCollectorInvariantError("invalid_query_url", "An exhaustive collection must begin on page 1; remove the page parameter.");
|
|
129
|
+
}
|
|
130
|
+
parsed.protocol = "https:";
|
|
131
|
+
parsed.hostname = "www.linkedin.com";
|
|
132
|
+
parsed.port = "";
|
|
133
|
+
parsed.username = "";
|
|
134
|
+
parsed.password = "";
|
|
135
|
+
parsed.pathname = "/search/results/products/";
|
|
136
|
+
parsed.hash = "";
|
|
137
|
+
parsed.searchParams.delete("page");
|
|
138
|
+
return parsed.toString();
|
|
139
|
+
}
|
|
140
|
+
export function buildLinkedInProductSearchPageUrl(queryUrl, pageNumber) {
|
|
141
|
+
if (!Number.isInteger(pageNumber) || pageNumber < 1) {
|
|
142
|
+
throw new LinkedInProductCollectorInvariantError("invalid_page", "Page numbers must be positive integers.");
|
|
143
|
+
}
|
|
144
|
+
const parsed = new URL(normalizeLinkedInProductSearchUrl(queryUrl));
|
|
145
|
+
if (pageNumber > 1)
|
|
146
|
+
parsed.searchParams.set("page", String(pageNumber));
|
|
147
|
+
return parsed.toString();
|
|
148
|
+
}
|
|
149
|
+
function assertExpectedLinkedInSearchPage(currentUrl, queryUrl, pageNumber) {
|
|
150
|
+
let actual;
|
|
151
|
+
try {
|
|
152
|
+
actual = new URL(currentUrl);
|
|
153
|
+
}
|
|
154
|
+
catch {
|
|
155
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported an invalid page URL.");
|
|
156
|
+
}
|
|
157
|
+
const source = new URL(normalizeLinkedInProductSearchUrl(queryUrl));
|
|
158
|
+
if (actual.protocol !== "https:" ||
|
|
159
|
+
actual.port !== "" ||
|
|
160
|
+
actual.username !== "" ||
|
|
161
|
+
actual.password !== "" ||
|
|
162
|
+
!["linkedin.com", "www.linkedin.com"].includes(actual.hostname.toLowerCase()) ||
|
|
163
|
+
(actual.pathname !== "/search/results/products" && actual.pathname !== "/search/results/products/")) {
|
|
164
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser is not on the expected LinkedIn product-search page.");
|
|
165
|
+
}
|
|
166
|
+
for (const key of new Set(source.searchParams.keys())) {
|
|
167
|
+
const expectedValues = source.searchParams.getAll(key);
|
|
168
|
+
const actualValues = actual.searchParams.getAll(key);
|
|
169
|
+
if (expectedValues.length !== actualValues.length || expectedValues.some((value, index) => value !== actualValues[index])) {
|
|
170
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", `The browser page changed the original ${key} query filter.`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
const sourceKeys = new Set(source.searchParams.keys());
|
|
174
|
+
for (const key of new Set(actual.searchParams.keys())) {
|
|
175
|
+
if (!sourceKeys.has(key) && key !== "page" && key !== "spellCorrectionEnabled") {
|
|
176
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", `The browser page added the unapproved ${key} query parameter.`);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
const spellCorrectionValues = actual.searchParams.getAll("spellCorrectionEnabled");
|
|
180
|
+
if (spellCorrectionValues.length > 1 ||
|
|
181
|
+
spellCorrectionValues.some((value) => value !== "true" && value !== "false")) {
|
|
182
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported an invalid spellCorrectionEnabled parameter.");
|
|
183
|
+
}
|
|
184
|
+
const actualPages = actual.searchParams.getAll("page");
|
|
185
|
+
const actualPage = actualPages[0] ?? null;
|
|
186
|
+
if (actualPages.length > 1) {
|
|
187
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported multiple page numbers.");
|
|
188
|
+
}
|
|
189
|
+
if (pageNumber === 1) {
|
|
190
|
+
if (actualPage && actualPage !== "1") {
|
|
191
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported the wrong page number.");
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
else if (actualPage !== String(pageNumber)) {
|
|
195
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported the wrong page number.");
|
|
196
|
+
}
|
|
197
|
+
return buildLinkedInProductSearchPageUrl(queryUrl, pageNumber);
|
|
198
|
+
}
|
|
199
|
+
export function canonicalizeLinkedInProductUrl(value) {
|
|
200
|
+
let parsed;
|
|
201
|
+
try {
|
|
202
|
+
parsed = new URL(value, "https://www.linkedin.com");
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid LinkedIn product URL: ${value}`);
|
|
206
|
+
}
|
|
207
|
+
if (parsed.protocol !== "https:" ||
|
|
208
|
+
parsed.port !== "" ||
|
|
209
|
+
parsed.username !== "" ||
|
|
210
|
+
parsed.password !== "" ||
|
|
211
|
+
!["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
|
|
212
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Product URL is outside LinkedIn: ${value}`);
|
|
213
|
+
}
|
|
214
|
+
const match = parsed.pathname.match(/^\/products\/([^/]+)\/?$/i);
|
|
215
|
+
if (!match?.[1]) {
|
|
216
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Product URL must match /products/<slug>/: ${value}`);
|
|
217
|
+
}
|
|
218
|
+
let decodedSlug;
|
|
219
|
+
try {
|
|
220
|
+
decodedSlug = decodeURIComponent(match[1]).trim().normalize("NFC").toLowerCase();
|
|
221
|
+
}
|
|
222
|
+
catch {
|
|
223
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid encoded product slug: ${value}`);
|
|
224
|
+
}
|
|
225
|
+
if (!decodedSlug || /[\u0000-\u001f\u007f/\\]/.test(decodedSlug)) {
|
|
226
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid LinkedIn product slug: ${match[1]}`);
|
|
227
|
+
}
|
|
228
|
+
const slug = encodeURIComponent(decodedSlug)
|
|
229
|
+
.replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).padStart(2, "0")}`)
|
|
230
|
+
.toLowerCase();
|
|
231
|
+
if (slug.length > 720 || !/^(?:[a-z0-9._~-]|%[0-9a-f]{2})+$/.test(slug)) {
|
|
232
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product_url", `Invalid LinkedIn product slug: ${slug}`);
|
|
233
|
+
}
|
|
234
|
+
return {
|
|
235
|
+
linkedinUrl: `https://www.linkedin.com/products/${slug}`,
|
|
236
|
+
linkedinSlug: slug
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
function normalizeIdentity(value) {
|
|
240
|
+
const identity = cleanText(value);
|
|
241
|
+
if (!identity)
|
|
242
|
+
return undefined;
|
|
243
|
+
if (!/^[A-Za-z0-9:._-]+$/.test(identity)) {
|
|
244
|
+
throw new LinkedInProductCollectorInvariantError("invalid_identity", "Product identity contains unsupported characters.");
|
|
245
|
+
}
|
|
246
|
+
return identity;
|
|
247
|
+
}
|
|
248
|
+
function normalizeVisibleProductCard(card, rank) {
|
|
249
|
+
const { linkedinUrl, linkedinSlug } = canonicalizeLinkedInProductUrl(card.linkedinUrl);
|
|
250
|
+
const name = cleanText(card.name);
|
|
251
|
+
if (!name) {
|
|
252
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product", "A visible product card is missing its name.");
|
|
253
|
+
}
|
|
254
|
+
const imageUrl = canonicalizePublicUrl(card.imageUrl);
|
|
255
|
+
const vendor = cleanText(card.vendor);
|
|
256
|
+
const category = cleanText(card.category);
|
|
257
|
+
const description = cleanText(card.description);
|
|
258
|
+
const identity = normalizeIdentity(card.identity);
|
|
259
|
+
return {
|
|
260
|
+
rank,
|
|
261
|
+
name,
|
|
262
|
+
linkedinUrl,
|
|
263
|
+
linkedinSlug,
|
|
264
|
+
...(imageUrl ? { imageUrl } : {}),
|
|
265
|
+
...(vendor ? { vendor } : {}),
|
|
266
|
+
...(category ? { category } : {}),
|
|
267
|
+
...(description ? { description } : {}),
|
|
268
|
+
...(identity ? { identity } : {})
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
export function createLinkedInProductCollectorCheckpoint(queryUrl, now = new Date()) {
|
|
272
|
+
const normalizedQueryUrl = normalizeLinkedInProductSearchUrl(queryUrl);
|
|
273
|
+
const timestamp = now.toISOString();
|
|
274
|
+
return {
|
|
275
|
+
schemaVersion: CHECKPOINT_SCHEMA_VERSION,
|
|
276
|
+
queryUrl: normalizedQueryUrl,
|
|
277
|
+
queryHash: sha256(normalizedQueryUrl),
|
|
278
|
+
startedAt: timestamp,
|
|
279
|
+
updatedAt: timestamp,
|
|
280
|
+
status: "collecting",
|
|
281
|
+
nextPageNumber: 1,
|
|
282
|
+
pagesAccepted: 0,
|
|
283
|
+
tasksIssued: 0,
|
|
284
|
+
approximateReportedTotal: null,
|
|
285
|
+
products: []
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
export function applyLinkedInProductSearchPage(checkpoint, submission, now = new Date()) {
|
|
289
|
+
if (checkpoint.status !== "collecting") {
|
|
290
|
+
throw new LinkedInProductCollectorInvariantError("checkpoint_closed", "The checkpoint is not collecting.");
|
|
291
|
+
}
|
|
292
|
+
if (submission.pageNumber !== checkpoint.nextPageNumber) {
|
|
293
|
+
throw new LinkedInProductCollectorInvariantError("out_of_order_page", `Expected page ${checkpoint.nextPageNumber}, received page ${submission.pageNumber}.`);
|
|
294
|
+
}
|
|
295
|
+
const currentUrl = assertExpectedLinkedInSearchPage(submission.currentUrl, checkpoint.queryUrl, submission.pageNumber);
|
|
296
|
+
if (submission.hasNext && submission.items.length === 0) {
|
|
297
|
+
throw new LinkedInProductCollectorInvariantError("empty_page_with_next", "A page that claims to have a next page must contain at least one visible product card.");
|
|
298
|
+
}
|
|
299
|
+
const products = checkpoint.products.map((product) => ({ ...product }));
|
|
300
|
+
const byUrl = new Map(products.map((product) => [product.linkedinUrl, product]));
|
|
301
|
+
const byIdentity = new Map();
|
|
302
|
+
for (const product of products) {
|
|
303
|
+
if (product.identity)
|
|
304
|
+
byIdentity.set(product.identity, product);
|
|
305
|
+
}
|
|
306
|
+
let duplicateCount = 0;
|
|
307
|
+
let newProductCount = 0;
|
|
308
|
+
for (const input of submission.items) {
|
|
309
|
+
const candidate = normalizeVisibleProductCard(input, products.length + 1);
|
|
310
|
+
const existingByUrl = byUrl.get(candidate.linkedinUrl);
|
|
311
|
+
const existingByIdentity = candidate.identity ? byIdentity.get(candidate.identity) : undefined;
|
|
312
|
+
if (existingByIdentity && existingByIdentity.linkedinUrl !== candidate.linkedinUrl) {
|
|
313
|
+
throw new LinkedInProductCollectorInvariantError("identity_drift", `Product identity ${candidate.identity} moved to a different canonical URL.`);
|
|
314
|
+
}
|
|
315
|
+
if (existingByUrl) {
|
|
316
|
+
if (candidate.identity && existingByUrl.identity && candidate.identity !== existingByUrl.identity) {
|
|
317
|
+
throw new LinkedInProductCollectorInvariantError("identity_drift", `Canonical URL ${candidate.linkedinUrl} changed product identity.`);
|
|
318
|
+
}
|
|
319
|
+
if (existingByUrl.name !== candidate.name) {
|
|
320
|
+
throw new LinkedInProductCollectorInvariantError("identity_drift", `Canonical URL ${candidate.linkedinUrl} changed product name.`);
|
|
321
|
+
}
|
|
322
|
+
if (!existingByUrl.identity && candidate.identity) {
|
|
323
|
+
existingByUrl.identity = candidate.identity;
|
|
324
|
+
byIdentity.set(candidate.identity, existingByUrl);
|
|
325
|
+
}
|
|
326
|
+
duplicateCount += 1;
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
products.push(candidate);
|
|
330
|
+
byUrl.set(candidate.linkedinUrl, candidate);
|
|
331
|
+
if (candidate.identity)
|
|
332
|
+
byIdentity.set(candidate.identity, candidate);
|
|
333
|
+
newProductCount += 1;
|
|
334
|
+
}
|
|
335
|
+
products.forEach((product, index) => {
|
|
336
|
+
if (product.rank !== index + 1) {
|
|
337
|
+
throw new LinkedInProductCollectorInvariantError("rank_drift", "Checkpoint ranks are not contiguous.");
|
|
338
|
+
}
|
|
339
|
+
});
|
|
340
|
+
const capturedAt = now.toISOString();
|
|
341
|
+
const reportedTotal = submission.reportedTotal ?? checkpoint.approximateReportedTotal;
|
|
342
|
+
const terminalPage = submission.hasNext
|
|
343
|
+
? undefined
|
|
344
|
+
: {
|
|
345
|
+
pageNumber: submission.pageNumber,
|
|
346
|
+
currentUrl,
|
|
347
|
+
reportedTotal: submission.reportedTotal ?? null,
|
|
348
|
+
hasNext: false,
|
|
349
|
+
visibleItemCount: submission.items.length,
|
|
350
|
+
newProductCount,
|
|
351
|
+
capturedAt
|
|
352
|
+
};
|
|
353
|
+
const nextCheckpoint = {
|
|
354
|
+
...checkpoint,
|
|
355
|
+
updatedAt: capturedAt,
|
|
356
|
+
status: submission.hasNext ? "collecting" : "complete",
|
|
357
|
+
nextPageNumber: submission.hasNext ? submission.pageNumber + 1 : submission.pageNumber,
|
|
358
|
+
pagesAccepted: checkpoint.pagesAccepted + 1,
|
|
359
|
+
tasksIssued: Math.max(checkpoint.tasksIssued, checkpoint.pagesAccepted + 1),
|
|
360
|
+
approximateReportedTotal: reportedTotal ?? null,
|
|
361
|
+
products,
|
|
362
|
+
...(terminalPage ? { terminalPage, completedAt: capturedAt } : {}),
|
|
363
|
+
lastStop: undefined
|
|
364
|
+
};
|
|
365
|
+
if (terminalPage)
|
|
366
|
+
nextCheckpoint.orderChecksum = computeLinkedInProductOrderChecksum(products);
|
|
367
|
+
return { checkpoint: nextCheckpoint, newProductCount, duplicateCount };
|
|
368
|
+
}
|
|
369
|
+
export function computeLinkedInProductOrderChecksum(products) {
|
|
370
|
+
const manifest = products.map((product) => `${product.rank}\t${product.linkedinUrl}`).join("\n");
|
|
371
|
+
return sha256(manifest);
|
|
372
|
+
}
|
|
373
|
+
export function buildLinkedInProductCollectionArtifact(checkpoint) {
|
|
374
|
+
assertValidCheckpoint(checkpoint);
|
|
375
|
+
if (checkpoint.status !== "complete" || !checkpoint.terminalPage || !checkpoint.completedAt) {
|
|
376
|
+
throw new LinkedInProductCollectorInvariantError("incomplete_checkpoint", "A complete artifact requires natural terminal-page evidence.");
|
|
377
|
+
}
|
|
378
|
+
const products = checkpoint.products.map((stored) => {
|
|
379
|
+
const canonical = canonicalizeLinkedInProductUrl(stored.linkedinUrl);
|
|
380
|
+
if (canonical.linkedinSlug !== stored.linkedinSlug) {
|
|
381
|
+
throw new LinkedInProductCollectorInvariantError("canonical_url_drift", "Checkpoint product URL and slug disagree.");
|
|
382
|
+
}
|
|
383
|
+
const name = cleanText(stored.name);
|
|
384
|
+
if (!name)
|
|
385
|
+
throw new LinkedInProductCollectorInvariantError("invalid_product", "Checkpoint product name is empty.");
|
|
386
|
+
const imageUrl = canonicalizePublicUrl(stored.imageUrl);
|
|
387
|
+
const vendor = cleanText(stored.vendor);
|
|
388
|
+
const category = cleanText(stored.category);
|
|
389
|
+
const description = cleanText(stored.description);
|
|
390
|
+
return {
|
|
391
|
+
rank: stored.rank,
|
|
392
|
+
name,
|
|
393
|
+
linkedinUrl: canonical.linkedinUrl,
|
|
394
|
+
linkedinSlug: canonical.linkedinSlug,
|
|
395
|
+
...(imageUrl ? { imageUrl } : {}),
|
|
396
|
+
...(vendor ? { vendor } : {}),
|
|
397
|
+
...(category ? { category } : {}),
|
|
398
|
+
...(description ? { description } : {})
|
|
399
|
+
};
|
|
400
|
+
});
|
|
401
|
+
products.forEach((product, index) => {
|
|
402
|
+
if (product.rank !== index + 1) {
|
|
403
|
+
throw new LinkedInProductCollectorInvariantError("rank_drift", "Artifact ranks are not contiguous.");
|
|
404
|
+
}
|
|
405
|
+
});
|
|
406
|
+
const orderChecksum = computeLinkedInProductOrderChecksum(products);
|
|
407
|
+
if (checkpoint.orderChecksum && checkpoint.orderChecksum !== orderChecksum) {
|
|
408
|
+
throw new LinkedInProductCollectorInvariantError("checksum_drift", "Checkpoint order checksum changed.");
|
|
409
|
+
}
|
|
410
|
+
const terminalPage = {
|
|
411
|
+
pageNumber: checkpoint.terminalPage.pageNumber,
|
|
412
|
+
currentUrl: checkpoint.terminalPage.currentUrl,
|
|
413
|
+
reportedTotal: checkpoint.terminalPage.reportedTotal,
|
|
414
|
+
hasNext: false,
|
|
415
|
+
visibleItemCount: checkpoint.terminalPage.visibleItemCount,
|
|
416
|
+
newProductCount: checkpoint.terminalPage.newProductCount,
|
|
417
|
+
capturedAt: checkpoint.terminalPage.capturedAt
|
|
418
|
+
};
|
|
419
|
+
return {
|
|
420
|
+
schemaVersion: ARTIFACT_SCHEMA_VERSION,
|
|
421
|
+
kind: "linkedin_product_search_snapshot",
|
|
422
|
+
complete: true,
|
|
423
|
+
source: {
|
|
424
|
+
queryUrl: checkpoint.queryUrl,
|
|
425
|
+
queryHash: checkpoint.queryHash
|
|
426
|
+
},
|
|
427
|
+
capture: {
|
|
428
|
+
startedAt: checkpoint.startedAt,
|
|
429
|
+
completedAt: checkpoint.completedAt,
|
|
430
|
+
approximateReportedTotal: checkpoint.approximateReportedTotal,
|
|
431
|
+
pagesAccepted: checkpoint.pagesAccepted,
|
|
432
|
+
terminalPage
|
|
433
|
+
},
|
|
434
|
+
productCount: products.length,
|
|
435
|
+
orderChecksumAlgorithm: "sha256",
|
|
436
|
+
orderChecksum,
|
|
437
|
+
products
|
|
438
|
+
};
|
|
439
|
+
}
|
|
440
|
+
function rawVisibleProductCard(card) {
|
|
441
|
+
const { rank: _rank, ...normalized } = normalizeVisibleProductCard(card, 1);
|
|
442
|
+
return normalized;
|
|
443
|
+
}
|
|
444
|
+
function sanitizeTerminalErrorUrl(currentUrl) {
|
|
445
|
+
let parsed;
|
|
446
|
+
try {
|
|
447
|
+
parsed = new URL(currentUrl);
|
|
448
|
+
}
|
|
449
|
+
catch {
|
|
450
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "The browser reported an invalid terminal URL.");
|
|
451
|
+
}
|
|
452
|
+
if (parsed.protocol !== "https:" ||
|
|
453
|
+
parsed.port !== "" ||
|
|
454
|
+
parsed.username !== "" ||
|
|
455
|
+
parsed.password !== "" ||
|
|
456
|
+
!["linkedin.com", "www.linkedin.com"].includes(parsed.hostname.toLowerCase())) {
|
|
457
|
+
throw new LinkedInProductCollectorInvariantError("unexpected_page", "A terminal browser error must come from a LinkedIn page.");
|
|
458
|
+
}
|
|
459
|
+
return `https://www.linkedin.com${parsed.pathname}`;
|
|
460
|
+
}
|
|
461
|
+
function terminalStopMessage(code) {
|
|
462
|
+
switch (code) {
|
|
463
|
+
case "auth_required":
|
|
464
|
+
return "The signed-in LinkedIn session is no longer available.";
|
|
465
|
+
case "challenge":
|
|
466
|
+
return "LinkedIn displayed a checkpoint or challenge.";
|
|
467
|
+
case "rate_limited":
|
|
468
|
+
return "LinkedIn rate-limited the product search.";
|
|
469
|
+
case "interrupted":
|
|
470
|
+
return "The browser worker was interrupted.";
|
|
471
|
+
case "unexpected_page":
|
|
472
|
+
return "The browser reached an unexpected LinkedIn page.";
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
async function ensureParent(filePath) {
|
|
476
|
+
await mkdir(path.dirname(path.resolve(filePath)), { recursive: true });
|
|
477
|
+
}
|
|
478
|
+
async function writePrivateJsonAtomic(filePath, value) {
|
|
479
|
+
const resolved = path.resolve(filePath);
|
|
480
|
+
await ensureParent(resolved);
|
|
481
|
+
const temporary = `${resolved}.${process.pid}.${randomUUID()}.tmp`;
|
|
482
|
+
await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
483
|
+
await chmod(temporary, 0o600);
|
|
484
|
+
await rename(temporary, resolved);
|
|
485
|
+
await chmod(resolved, 0o600);
|
|
486
|
+
}
|
|
487
|
+
async function appendPrivateJsonLine(filePath, value) {
|
|
488
|
+
const resolved = path.resolve(filePath);
|
|
489
|
+
await ensureParent(resolved);
|
|
490
|
+
await appendFile(resolved, `${JSON.stringify(value)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
491
|
+
await chmod(resolved, 0o600);
|
|
492
|
+
}
|
|
493
|
+
async function readCheckpoint(filePath) {
|
|
494
|
+
try {
|
|
495
|
+
const value = JSON.parse(await readFile(path.resolve(filePath), "utf8"));
|
|
496
|
+
if (value.schemaVersion !== CHECKPOINT_SCHEMA_VERSION || !Array.isArray(value.products)) {
|
|
497
|
+
throw new Error("Unsupported LinkedIn product collector checkpoint.");
|
|
498
|
+
}
|
|
499
|
+
assertValidCheckpoint(value);
|
|
500
|
+
return value;
|
|
501
|
+
}
|
|
502
|
+
catch (error) {
|
|
503
|
+
if (error.code === "ENOENT")
|
|
504
|
+
return null;
|
|
505
|
+
throw error;
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
function assertValidCheckpoint(checkpoint) {
|
|
509
|
+
if (normalizeLinkedInProductSearchUrl(checkpoint.queryUrl) !== checkpoint.queryUrl) {
|
|
510
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint query URL is invalid.");
|
|
511
|
+
}
|
|
512
|
+
if (!/^[a-f0-9]{64}$/.test(checkpoint.queryHash)) {
|
|
513
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint query hash is invalid.");
|
|
514
|
+
}
|
|
515
|
+
if (!["collecting", "complete", "failed"].includes(checkpoint.status)) {
|
|
516
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint status is invalid.");
|
|
517
|
+
}
|
|
518
|
+
if (!Number.isInteger(checkpoint.nextPageNumber) ||
|
|
519
|
+
checkpoint.nextPageNumber < 1 ||
|
|
520
|
+
!Number.isInteger(checkpoint.pagesAccepted) ||
|
|
521
|
+
checkpoint.pagesAccepted < 0 ||
|
|
522
|
+
!Number.isInteger(checkpoint.tasksIssued) ||
|
|
523
|
+
checkpoint.tasksIssued < 0) {
|
|
524
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint counters are invalid.");
|
|
525
|
+
}
|
|
526
|
+
if (checkpoint.tasksIssued < checkpoint.pagesAccepted) {
|
|
527
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint has more accepted pages than issued browser tasks.");
|
|
528
|
+
}
|
|
529
|
+
assertIsoTimestamp(checkpoint.startedAt, "startedAt");
|
|
530
|
+
assertIsoTimestamp(checkpoint.updatedAt, "updatedAt");
|
|
531
|
+
if (checkpoint.updatedAt < checkpoint.startedAt) {
|
|
532
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint timestamps are out of order.");
|
|
533
|
+
}
|
|
534
|
+
if (checkpoint.approximateReportedTotal !== null &&
|
|
535
|
+
(!Number.isInteger(checkpoint.approximateReportedTotal) || checkpoint.approximateReportedTotal < 0)) {
|
|
536
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint approximate reported total is invalid.");
|
|
537
|
+
}
|
|
538
|
+
const seenUrls = new Set();
|
|
539
|
+
const seenIdentities = new Set();
|
|
540
|
+
checkpoint.products.forEach((product, index) => {
|
|
541
|
+
if (product.rank !== index + 1 || !Number.isInteger(product.rank)) {
|
|
542
|
+
throw new LinkedInProductCollectorInvariantError("rank_drift", "Checkpoint ranks are not contiguous.");
|
|
543
|
+
}
|
|
544
|
+
const canonical = canonicalizeLinkedInProductUrl(product.linkedinUrl);
|
|
545
|
+
if (canonical.linkedinUrl !== product.linkedinUrl || canonical.linkedinSlug !== product.linkedinSlug) {
|
|
546
|
+
throw new LinkedInProductCollectorInvariantError("canonical_url_drift", "Checkpoint contains a non-canonical product URL.");
|
|
547
|
+
}
|
|
548
|
+
if (seenUrls.has(product.linkedinUrl)) {
|
|
549
|
+
throw new LinkedInProductCollectorInvariantError("duplicate_product", "Checkpoint contains a duplicate product URL.");
|
|
550
|
+
}
|
|
551
|
+
seenUrls.add(product.linkedinUrl);
|
|
552
|
+
if (product.identity) {
|
|
553
|
+
const identity = normalizeIdentity(product.identity);
|
|
554
|
+
if (!identity || seenIdentities.has(identity)) {
|
|
555
|
+
throw new LinkedInProductCollectorInvariantError("identity_drift", "Checkpoint contains a duplicate or invalid product identity.");
|
|
556
|
+
}
|
|
557
|
+
seenIdentities.add(identity);
|
|
558
|
+
}
|
|
559
|
+
});
|
|
560
|
+
if (checkpoint.status === "complete") {
|
|
561
|
+
if (checkpoint.pagesAccepted < 1 ||
|
|
562
|
+
checkpoint.nextPageNumber !== checkpoint.pagesAccepted ||
|
|
563
|
+
!checkpoint.terminalPage ||
|
|
564
|
+
!checkpoint.completedAt ||
|
|
565
|
+
checkpoint.lastStop !== undefined ||
|
|
566
|
+
!/^[a-f0-9]{64}$/.test(checkpoint.orderChecksum ?? "")) {
|
|
567
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "A complete checkpoint is missing terminal evidence or its checksum.");
|
|
568
|
+
}
|
|
569
|
+
const terminal = checkpoint.terminalPage;
|
|
570
|
+
if (terminal.hasNext !== false ||
|
|
571
|
+
!Number.isInteger(terminal.pageNumber) ||
|
|
572
|
+
terminal.pageNumber !== checkpoint.pagesAccepted ||
|
|
573
|
+
terminal.currentUrl !== buildLinkedInProductSearchPageUrl(checkpoint.queryUrl, terminal.pageNumber) ||
|
|
574
|
+
!Number.isInteger(terminal.visibleItemCount) ||
|
|
575
|
+
terminal.visibleItemCount < 0 ||
|
|
576
|
+
!Number.isInteger(terminal.newProductCount) ||
|
|
577
|
+
terminal.newProductCount < 0 ||
|
|
578
|
+
terminal.newProductCount > terminal.visibleItemCount ||
|
|
579
|
+
(terminal.reportedTotal !== null &&
|
|
580
|
+
(!Number.isInteger(terminal.reportedTotal) || terminal.reportedTotal < 0))) {
|
|
581
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Complete checkpoint terminal-page evidence is invalid.");
|
|
582
|
+
}
|
|
583
|
+
assertIsoTimestamp(terminal.capturedAt, "terminalPage.capturedAt");
|
|
584
|
+
assertIsoTimestamp(checkpoint.completedAt, "completedAt");
|
|
585
|
+
if (terminal.capturedAt !== checkpoint.completedAt || checkpoint.completedAt !== checkpoint.updatedAt) {
|
|
586
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Complete checkpoint capture timestamps disagree.");
|
|
587
|
+
}
|
|
588
|
+
if (computeLinkedInProductOrderChecksum(checkpoint.products) !== checkpoint.orderChecksum) {
|
|
589
|
+
throw new LinkedInProductCollectorInvariantError("checksum_drift", "Checkpoint order checksum changed.");
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
else {
|
|
593
|
+
if (checkpoint.nextPageNumber !== checkpoint.pagesAccepted + 1 ||
|
|
594
|
+
checkpoint.terminalPage !== undefined ||
|
|
595
|
+
checkpoint.completedAt !== undefined ||
|
|
596
|
+
checkpoint.orderChecksum !== undefined) {
|
|
597
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Incomplete checkpoint state is internally inconsistent.");
|
|
598
|
+
}
|
|
599
|
+
if (checkpoint.lastStop) {
|
|
600
|
+
if (typeof checkpoint.lastStop.reason !== "string" ||
|
|
601
|
+
checkpoint.lastStop.reason.trim() === "" ||
|
|
602
|
+
!Number.isInteger(checkpoint.lastStop.pageNumber) ||
|
|
603
|
+
checkpoint.lastStop.pageNumber !== checkpoint.nextPageNumber) {
|
|
604
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", "Checkpoint stop evidence is invalid.");
|
|
605
|
+
}
|
|
606
|
+
assertIsoTimestamp(checkpoint.lastStop.at, "lastStop.at");
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
function assertIsoTimestamp(value, field) {
|
|
611
|
+
if (typeof value !== "string") {
|
|
612
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", `Checkpoint ${field} is not a timestamp.`);
|
|
613
|
+
}
|
|
614
|
+
const parsed = new Date(value);
|
|
615
|
+
if (Number.isNaN(parsed.valueOf()) || parsed.toISOString() !== value) {
|
|
616
|
+
throw new LinkedInProductCollectorInvariantError("invalid_checkpoint", `Checkpoint ${field} is not canonical ISO time.`);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
async function fileExists(filePath) {
|
|
620
|
+
try {
|
|
621
|
+
await access(path.resolve(filePath));
|
|
622
|
+
return true;
|
|
623
|
+
}
|
|
624
|
+
catch {
|
|
625
|
+
return false;
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
async function assertRawEvidenceMatchesCheckpoint(filePath, checkpoint) {
|
|
629
|
+
const raw = await readFile(path.resolve(filePath), "utf8");
|
|
630
|
+
const lines = raw.split(/\r?\n/).filter((line) => line.trim() !== "");
|
|
631
|
+
let replay = createLinkedInProductCollectorCheckpoint(checkpoint.queryUrl, new Date(checkpoint.startedAt));
|
|
632
|
+
let acceptedPageRecords = 0;
|
|
633
|
+
for (const [index, line] of lines.entries()) {
|
|
634
|
+
let parsedJson;
|
|
635
|
+
try {
|
|
636
|
+
parsedJson = JSON.parse(line);
|
|
637
|
+
}
|
|
638
|
+
catch {
|
|
639
|
+
throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw JSONL line ${index + 1} is not valid JSON.`);
|
|
640
|
+
}
|
|
641
|
+
const record = rawEvidenceRecordSchema.parse(parsedJson);
|
|
642
|
+
if (record.queryHash !== checkpoint.queryHash) {
|
|
643
|
+
throw new LinkedInProductCollectorInvariantError("raw_query_mismatch", `Raw JSONL line ${index + 1} belongs to another query.`);
|
|
644
|
+
}
|
|
645
|
+
assertIsoTimestamp(record.receivedAt, `raw line ${index + 1} receivedAt`);
|
|
646
|
+
if (record.type === "terminal_error") {
|
|
647
|
+
if (replay.status !== "collecting" || record.pageNumber !== replay.nextPageNumber) {
|
|
648
|
+
throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw terminal error on line ${index + 1} is out of sequence.`);
|
|
649
|
+
}
|
|
650
|
+
continue;
|
|
651
|
+
}
|
|
652
|
+
if (record.pageNumber !== replay.nextPageNumber) {
|
|
653
|
+
throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw page on line ${index + 1} is duplicate or out of sequence.`);
|
|
654
|
+
}
|
|
655
|
+
const items = record.items.map(({ linkedinSlug, ...item }) => {
|
|
656
|
+
const canonical = canonicalizeLinkedInProductUrl(item.linkedinUrl);
|
|
657
|
+
if (canonical.linkedinSlug !== linkedinSlug) {
|
|
658
|
+
throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw product slug on line ${index + 1} disagrees with its URL.`);
|
|
659
|
+
}
|
|
660
|
+
return item;
|
|
661
|
+
});
|
|
662
|
+
const applied = applyLinkedInProductSearchPage(replay, {
|
|
663
|
+
pageNumber: record.pageNumber,
|
|
664
|
+
currentUrl: record.currentUrl,
|
|
665
|
+
reportedTotal: record.reportedTotal,
|
|
666
|
+
hasNext: record.hasNext,
|
|
667
|
+
items
|
|
668
|
+
}, new Date(record.receivedAt));
|
|
669
|
+
if (record.visibleItemCount !== record.items.length ||
|
|
670
|
+
record.newProductCount !== applied.newProductCount ||
|
|
671
|
+
record.duplicateCount !== applied.duplicateCount) {
|
|
672
|
+
throw new LinkedInProductCollectorInvariantError("invalid_raw_evidence", `Raw page counts on line ${index + 1} do not match its normalized items.`);
|
|
673
|
+
}
|
|
674
|
+
replay = applied.checkpoint;
|
|
675
|
+
acceptedPageRecords += 1;
|
|
676
|
+
}
|
|
677
|
+
if (acceptedPageRecords !== checkpoint.pagesAccepted) {
|
|
678
|
+
throw new LinkedInProductCollectorInvariantError("missing_raw_evidence", `Raw JSONL proves ${acceptedPageRecords} pages, but the checkpoint claims ${checkpoint.pagesAccepted}.`);
|
|
679
|
+
}
|
|
680
|
+
if (JSON.stringify(replay.products) !== JSON.stringify(checkpoint.products)) {
|
|
681
|
+
throw new LinkedInProductCollectorInvariantError("raw_order_mismatch", "Raw JSONL product order does not reproduce the checkpoint.");
|
|
682
|
+
}
|
|
683
|
+
if (replay.approximateReportedTotal !== checkpoint.approximateReportedTotal ||
|
|
684
|
+
(checkpoint.status === "complete") !== (replay.status === "complete")) {
|
|
685
|
+
throw new LinkedInProductCollectorInvariantError("raw_state_mismatch", "Raw JSONL completion metadata does not reproduce the checkpoint.");
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
async function readRelayJson(req) {
|
|
689
|
+
const chunks = [];
|
|
690
|
+
let size = 0;
|
|
691
|
+
for await (const chunk of req) {
|
|
692
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
693
|
+
size += buffer.length;
|
|
694
|
+
if (size > MAX_RELAY_BODY_BYTES) {
|
|
695
|
+
throw new LinkedInProductCollectorInvariantError("body_too_large", "Relay request body is too large.");
|
|
696
|
+
}
|
|
697
|
+
chunks.push(buffer);
|
|
698
|
+
}
|
|
699
|
+
if (chunks.length === 0)
|
|
700
|
+
throw new Error("Relay request body is empty.");
|
|
701
|
+
return JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
702
|
+
}
|
|
703
|
+
function writeRelayJson(res, statusCode, value, origin) {
|
|
704
|
+
const body = `${JSON.stringify(value)}\n`;
|
|
705
|
+
res.writeHead(statusCode, {
|
|
706
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
707
|
+
"Content-Length": Buffer.byteLength(body),
|
|
708
|
+
"Cache-Control": "no-store",
|
|
709
|
+
"Access-Control-Allow-Origin": origin === "https://www.linkedin.com" ? origin : "https://www.linkedin.com",
|
|
710
|
+
"Access-Control-Allow-Private-Network": "true",
|
|
711
|
+
Vary: "Origin"
|
|
712
|
+
});
|
|
713
|
+
res.end(body);
|
|
714
|
+
}
|
|
715
|
+
function isAllowedRelayOrigin(req) {
|
|
716
|
+
const origin = req.headers.origin;
|
|
717
|
+
return origin === undefined || origin === "https://www.linkedin.com";
|
|
718
|
+
}
|
|
719
|
+
function closeServer(server) {
|
|
720
|
+
return new Promise((resolve) => server.close(() => resolve()));
|
|
721
|
+
}
|
|
722
|
+
export async function collectLinkedInProductsViaBrowserRelay(options) {
|
|
723
|
+
const queryUrl = normalizeLinkedInProductSearchUrl(options.queryUrl);
|
|
724
|
+
const expectedQueryHash = sha256(queryUrl);
|
|
725
|
+
const resolvedPaths = [options.checkpointPath, options.rawJsonlPath, options.outPath].map((value) => path.resolve(value));
|
|
726
|
+
if (new Set(resolvedPaths).size !== resolvedPaths.length) {
|
|
727
|
+
throw new LinkedInProductCollectorInvariantError("artifact_path_conflict", "Checkpoint, raw JSONL, and complete artifact paths must be distinct.");
|
|
728
|
+
}
|
|
729
|
+
const existingCheckpoint = await readCheckpoint(options.checkpointPath);
|
|
730
|
+
const [rawExists, outExists] = await Promise.all([
|
|
731
|
+
fileExists(options.rawJsonlPath),
|
|
732
|
+
fileExists(options.outPath)
|
|
733
|
+
]);
|
|
734
|
+
if (!existingCheckpoint && (rawExists || outExists)) {
|
|
735
|
+
throw new LinkedInProductCollectorInvariantError("orphaned_artifact", "A new collection requires unused raw JSONL and output paths. Choose new paths or restore the matching checkpoint.");
|
|
736
|
+
}
|
|
737
|
+
if (existingCheckpoint && existingCheckpoint.pagesAccepted > 0 && !rawExists) {
|
|
738
|
+
throw new LinkedInProductCollectorInvariantError("missing_raw_evidence", "The checkpoint has accepted pages but its raw JSONL evidence is missing.");
|
|
739
|
+
}
|
|
740
|
+
if (existingCheckpoint && rawExists) {
|
|
741
|
+
await assertRawEvidenceMatchesCheckpoint(options.rawJsonlPath, existingCheckpoint);
|
|
742
|
+
}
|
|
743
|
+
if (existingCheckpoint && existingCheckpoint.status !== "complete" && outExists) {
|
|
744
|
+
throw new LinkedInProductCollectorInvariantError("stale_complete_artifact", "An incomplete checkpoint cannot share a path with an existing complete artifact. Choose a new output path.");
|
|
745
|
+
}
|
|
746
|
+
let checkpoint = existingCheckpoint ?? createLinkedInProductCollectorCheckpoint(queryUrl);
|
|
747
|
+
if (checkpoint.queryUrl !== queryUrl || checkpoint.queryHash !== expectedQueryHash) {
|
|
748
|
+
throw new LinkedInProductCollectorInvariantError("checkpoint_query_mismatch", "The checkpoint belongs to a different LinkedIn product search URL.");
|
|
749
|
+
}
|
|
750
|
+
await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
|
|
751
|
+
if (checkpoint.status === "complete") {
|
|
752
|
+
const artifact = buildLinkedInProductCollectionArtifact(checkpoint);
|
|
753
|
+
await writePrivateJsonAtomic(options.outPath, artifact);
|
|
754
|
+
return {
|
|
755
|
+
status: "complete",
|
|
756
|
+
complete: true,
|
|
757
|
+
queryUrl,
|
|
758
|
+
queryHash: expectedQueryHash,
|
|
759
|
+
checkpointPath: path.resolve(options.checkpointPath),
|
|
760
|
+
rawJsonlPath: path.resolve(options.rawJsonlPath),
|
|
761
|
+
outPath: path.resolve(options.outPath),
|
|
762
|
+
pagesAccepted: checkpoint.pagesAccepted,
|
|
763
|
+
productsCollected: checkpoint.products.length,
|
|
764
|
+
nextPageNumber: checkpoint.nextPageNumber,
|
|
765
|
+
orderChecksum: checkpoint.orderChecksum,
|
|
766
|
+
terminalPage: checkpoint.terminalPage
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
if (checkpoint.status === "failed") {
|
|
770
|
+
throw new LinkedInProductCollectorInvariantError("failed_checkpoint", "The checkpoint is failed. Inspect it and choose a new checkpoint path before retrying.");
|
|
771
|
+
}
|
|
772
|
+
let uniqueTasksIssuedThisRun = 0;
|
|
773
|
+
let lastIssuedPage = null;
|
|
774
|
+
let settled = false;
|
|
775
|
+
let acceptingRequests = true;
|
|
776
|
+
let idleTimer;
|
|
777
|
+
let incompleteStopPromise;
|
|
778
|
+
let resolveResult;
|
|
779
|
+
let rejectResult;
|
|
780
|
+
const resultPromise = new Promise((resolve, reject) => {
|
|
781
|
+
resolveResult = resolve;
|
|
782
|
+
rejectResult = reject;
|
|
783
|
+
});
|
|
784
|
+
let requestQueue = Promise.resolve();
|
|
785
|
+
const enqueueExclusive = (operation) => {
|
|
786
|
+
const queued = requestQueue.then(operation, operation);
|
|
787
|
+
requestQueue = queued.catch(() => undefined);
|
|
788
|
+
return queued;
|
|
789
|
+
};
|
|
790
|
+
const server = createServer((req, res) => {
|
|
791
|
+
if (!acceptingRequests) {
|
|
792
|
+
writeRelayJson(res, 410, { status: "closed" }, req.headers.origin);
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
const execute = async () => {
|
|
796
|
+
try {
|
|
797
|
+
await handleRequest(req, res);
|
|
798
|
+
}
|
|
799
|
+
catch (error) {
|
|
800
|
+
const browserInputError = error instanceof LinkedInProductCollectorInvariantError || error instanceof z.ZodError || error instanceof SyntaxError;
|
|
801
|
+
if (!res.headersSent) {
|
|
802
|
+
writeRelayJson(res, browserInputError ? 409 : 500, {
|
|
803
|
+
status: "rejected",
|
|
804
|
+
error: error instanceof Error ? error.message : String(error)
|
|
805
|
+
}, req.headers.origin);
|
|
806
|
+
}
|
|
807
|
+
if (!browserInputError)
|
|
808
|
+
await abortWithError(error);
|
|
809
|
+
}
|
|
810
|
+
};
|
|
811
|
+
void enqueueExclusive(execute);
|
|
812
|
+
});
|
|
813
|
+
const makeResult = (status, reason) => ({
|
|
814
|
+
status,
|
|
815
|
+
complete: status === "complete",
|
|
816
|
+
...(reason ? { reason } : {}),
|
|
817
|
+
queryUrl,
|
|
818
|
+
queryHash: expectedQueryHash,
|
|
819
|
+
checkpointPath: path.resolve(options.checkpointPath),
|
|
820
|
+
rawJsonlPath: path.resolve(options.rawJsonlPath),
|
|
821
|
+
outPath: status === "complete" ? path.resolve(options.outPath) : null,
|
|
822
|
+
pagesAccepted: checkpoint.pagesAccepted,
|
|
823
|
+
productsCollected: checkpoint.products.length,
|
|
824
|
+
nextPageNumber: checkpoint.nextPageNumber,
|
|
825
|
+
...(checkpoint.orderChecksum ? { orderChecksum: checkpoint.orderChecksum } : {}),
|
|
826
|
+
...(checkpoint.terminalPage ? { terminalPage: checkpoint.terminalPage } : {})
|
|
827
|
+
});
|
|
828
|
+
const finish = async (status, reason) => {
|
|
829
|
+
if (settled)
|
|
830
|
+
return;
|
|
831
|
+
settled = true;
|
|
832
|
+
acceptingRequests = false;
|
|
833
|
+
if (idleTimer)
|
|
834
|
+
clearTimeout(idleTimer);
|
|
835
|
+
process.off("SIGINT", handleSigint);
|
|
836
|
+
process.off("SIGTERM", handleSigterm);
|
|
837
|
+
await closeServer(server);
|
|
838
|
+
resolveResult?.(makeResult(status, reason));
|
|
839
|
+
};
|
|
840
|
+
const abortWithError = async (error) => {
|
|
841
|
+
if (settled)
|
|
842
|
+
return;
|
|
843
|
+
settled = true;
|
|
844
|
+
acceptingRequests = false;
|
|
845
|
+
if (idleTimer)
|
|
846
|
+
clearTimeout(idleTimer);
|
|
847
|
+
process.off("SIGINT", handleSigint);
|
|
848
|
+
process.off("SIGTERM", handleSigterm);
|
|
849
|
+
await closeServer(server);
|
|
850
|
+
rejectResult?.(error);
|
|
851
|
+
};
|
|
852
|
+
const stopIncomplete = (reason, message) => {
|
|
853
|
+
acceptingRequests = false;
|
|
854
|
+
if (incompleteStopPromise)
|
|
855
|
+
return incompleteStopPromise;
|
|
856
|
+
incompleteStopPromise = (async () => {
|
|
857
|
+
if (settled)
|
|
858
|
+
return;
|
|
859
|
+
const now = new Date().toISOString();
|
|
860
|
+
checkpoint = {
|
|
861
|
+
...checkpoint,
|
|
862
|
+
updatedAt: now,
|
|
863
|
+
lastStop: {
|
|
864
|
+
reason,
|
|
865
|
+
at: now,
|
|
866
|
+
pageNumber: checkpoint.nextPageNumber,
|
|
867
|
+
...(message ? { message } : {})
|
|
868
|
+
}
|
|
869
|
+
};
|
|
870
|
+
await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
|
|
871
|
+
await finish("incomplete", reason);
|
|
872
|
+
})();
|
|
873
|
+
return incompleteStopPromise;
|
|
874
|
+
};
|
|
875
|
+
const requestIncompleteStop = (reason, message) => {
|
|
876
|
+
acceptingRequests = false;
|
|
877
|
+
return enqueueExclusive(async () => {
|
|
878
|
+
if (settled)
|
|
879
|
+
return;
|
|
880
|
+
if (checkpoint.status === "complete") {
|
|
881
|
+
await finish("complete");
|
|
882
|
+
return;
|
|
883
|
+
}
|
|
884
|
+
await stopIncomplete(reason, message);
|
|
885
|
+
});
|
|
886
|
+
};
|
|
887
|
+
function resetIdleTimer() {
|
|
888
|
+
if (idleTimer)
|
|
889
|
+
clearTimeout(idleTimer);
|
|
890
|
+
idleTimer = setTimeout(() => {
|
|
891
|
+
void requestIncompleteStop("idle_timeout", "No browser relay activity arrived before the local timeout.").catch((error) => {
|
|
892
|
+
rejectResult?.(error);
|
|
893
|
+
});
|
|
894
|
+
}, options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS);
|
|
895
|
+
idleTimer.unref();
|
|
896
|
+
}
|
|
897
|
+
async function handleRequest(req, res) {
|
|
898
|
+
if (!acceptingRequests) {
|
|
899
|
+
writeRelayJson(res, 410, { status: "closed" }, req.headers.origin);
|
|
900
|
+
return;
|
|
901
|
+
}
|
|
902
|
+
if (!isAllowedRelayOrigin(req)) {
|
|
903
|
+
writeRelayJson(res, 403, { status: "rejected", error: "Relay origin is not allowed." });
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
resetIdleTimer();
|
|
907
|
+
const requestUrl = new URL(req.url ?? "/", "http://127.0.0.1");
|
|
908
|
+
if (req.method === "OPTIONS") {
|
|
909
|
+
res.writeHead(204, {
|
|
910
|
+
"Access-Control-Allow-Origin": "https://www.linkedin.com",
|
|
911
|
+
"Access-Control-Allow-Methods": "GET, POST, OPTIONS",
|
|
912
|
+
"Access-Control-Allow-Headers": "Content-Type",
|
|
913
|
+
"Access-Control-Allow-Private-Network": "true",
|
|
914
|
+
"Access-Control-Max-Age": "600",
|
|
915
|
+
Vary: "Origin"
|
|
916
|
+
});
|
|
917
|
+
res.end();
|
|
918
|
+
return;
|
|
919
|
+
}
|
|
920
|
+
if (req.method === "GET" && requestUrl.pathname === "/health") {
|
|
921
|
+
writeRelayJson(res, 200, { status: "ok", queryHash: expectedQueryHash }, req.headers.origin);
|
|
922
|
+
return;
|
|
923
|
+
}
|
|
924
|
+
if (req.method === "GET" && requestUrl.pathname === "/task") {
|
|
925
|
+
if (options.maxPages !== undefined && checkpoint.nextPageNumber > options.maxPages) {
|
|
926
|
+
acceptingRequests = false;
|
|
927
|
+
writeRelayJson(res, 200, { status: "incomplete", reason: "page_limit" }, req.headers.origin);
|
|
928
|
+
res.once("finish", () => void requestIncompleteStop("page_limit"));
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
931
|
+
if (options.maxRequests !== undefined &&
|
|
932
|
+
uniqueTasksIssuedThisRun >= options.maxRequests &&
|
|
933
|
+
lastIssuedPage !== checkpoint.nextPageNumber) {
|
|
934
|
+
acceptingRequests = false;
|
|
935
|
+
writeRelayJson(res, 200, { status: "incomplete", reason: "request_limit" }, req.headers.origin);
|
|
936
|
+
res.once("finish", () => void requestIncompleteStop("request_limit"));
|
|
937
|
+
return;
|
|
938
|
+
}
|
|
939
|
+
if (lastIssuedPage !== checkpoint.nextPageNumber) {
|
|
940
|
+
uniqueTasksIssuedThisRun += 1;
|
|
941
|
+
lastIssuedPage = checkpoint.nextPageNumber;
|
|
942
|
+
checkpoint = { ...checkpoint, tasksIssued: checkpoint.tasksIssued + 1, updatedAt: new Date().toISOString() };
|
|
943
|
+
await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
|
|
944
|
+
}
|
|
945
|
+
writeRelayJson(res, 200, {
|
|
946
|
+
status: "task",
|
|
947
|
+
queryHash: expectedQueryHash,
|
|
948
|
+
pageNumber: checkpoint.nextPageNumber,
|
|
949
|
+
currentUrl: buildLinkedInProductSearchPageUrl(queryUrl, checkpoint.nextPageNumber),
|
|
950
|
+
productsAccepted: checkpoint.products.length
|
|
951
|
+
}, req.headers.origin);
|
|
952
|
+
return;
|
|
953
|
+
}
|
|
954
|
+
if (req.method === "POST" && requestUrl.pathname === "/page") {
|
|
955
|
+
const submission = linkedInProductPageSubmissionSchema.parse(await readRelayJson(req));
|
|
956
|
+
if (submission.pageNumber !== checkpoint.nextPageNumber) {
|
|
957
|
+
throw new LinkedInProductCollectorInvariantError("out_of_order_page", `Expected page ${checkpoint.nextPageNumber}, received page ${submission.pageNumber}.`);
|
|
958
|
+
}
|
|
959
|
+
if (lastIssuedPage !== submission.pageNumber) {
|
|
960
|
+
throw new LinkedInProductCollectorInvariantError("unleased_page", "Poll GET /task before posting the expected browser page.");
|
|
961
|
+
}
|
|
962
|
+
if ("error" in submission) {
|
|
963
|
+
acceptingRequests = false;
|
|
964
|
+
const sanitizedCurrentUrl = sanitizeTerminalErrorUrl(submission.currentUrl);
|
|
965
|
+
const receivedAt = new Date().toISOString();
|
|
966
|
+
await appendPrivateJsonLine(options.rawJsonlPath, {
|
|
967
|
+
type: "terminal_error",
|
|
968
|
+
queryHash: expectedQueryHash,
|
|
969
|
+
pageNumber: submission.pageNumber,
|
|
970
|
+
currentUrl: sanitizedCurrentUrl,
|
|
971
|
+
error: { code: submission.error.code },
|
|
972
|
+
receivedAt
|
|
973
|
+
});
|
|
974
|
+
writeRelayJson(res, 202, { status: "checkpointed", complete: false }, req.headers.origin);
|
|
975
|
+
res.once("finish", () => {
|
|
976
|
+
void requestIncompleteStop(submission.error.code, terminalStopMessage(submission.error.code)).catch((error) => rejectResult?.(error));
|
|
977
|
+
});
|
|
978
|
+
return;
|
|
979
|
+
}
|
|
980
|
+
const sanitizedCurrentUrl = assertExpectedLinkedInSearchPage(submission.currentUrl, checkpoint.queryUrl, submission.pageNumber);
|
|
981
|
+
let applied;
|
|
982
|
+
try {
|
|
983
|
+
applied = applyLinkedInProductSearchPage(checkpoint, submission);
|
|
984
|
+
}
|
|
985
|
+
catch (error) {
|
|
986
|
+
if (error instanceof LinkedInProductCollectorInvariantError && error.code === "identity_drift") {
|
|
987
|
+
acceptingRequests = false;
|
|
988
|
+
const now = new Date().toISOString();
|
|
989
|
+
checkpoint = {
|
|
990
|
+
...checkpoint,
|
|
991
|
+
status: "failed",
|
|
992
|
+
updatedAt: now,
|
|
993
|
+
lastStop: { reason: error.code, at: now, pageNumber: checkpoint.nextPageNumber, message: error.message }
|
|
994
|
+
};
|
|
995
|
+
await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
|
|
996
|
+
writeRelayJson(res, 409, { status: "rejected", complete: false, error: error.message }, req.headers.origin);
|
|
997
|
+
res.once("finish", () => void finish("incomplete", error.code));
|
|
998
|
+
return;
|
|
999
|
+
}
|
|
1000
|
+
throw error;
|
|
1001
|
+
}
|
|
1002
|
+
await appendPrivateJsonLine(options.rawJsonlPath, {
|
|
1003
|
+
type: "page",
|
|
1004
|
+
queryHash: expectedQueryHash,
|
|
1005
|
+
pageNumber: submission.pageNumber,
|
|
1006
|
+
currentUrl: sanitizedCurrentUrl,
|
|
1007
|
+
reportedTotal: submission.reportedTotal ?? null,
|
|
1008
|
+
hasNext: submission.hasNext,
|
|
1009
|
+
items: submission.items.map(rawVisibleProductCard),
|
|
1010
|
+
visibleItemCount: submission.items.length,
|
|
1011
|
+
newProductCount: applied.newProductCount,
|
|
1012
|
+
duplicateCount: applied.duplicateCount,
|
|
1013
|
+
receivedAt: applied.checkpoint.updatedAt
|
|
1014
|
+
});
|
|
1015
|
+
checkpoint = applied.checkpoint;
|
|
1016
|
+
await writePrivateJsonAtomic(options.checkpointPath, checkpoint);
|
|
1017
|
+
if (checkpoint.status === "complete") {
|
|
1018
|
+
const artifact = buildLinkedInProductCollectionArtifact(checkpoint);
|
|
1019
|
+
await writePrivateJsonAtomic(options.outPath, artifact);
|
|
1020
|
+
acceptingRequests = false;
|
|
1021
|
+
}
|
|
1022
|
+
lastIssuedPage = null;
|
|
1023
|
+
writeRelayJson(res, 202, {
|
|
1024
|
+
status: checkpoint.status === "complete" ? "complete" : "accepted",
|
|
1025
|
+
complete: checkpoint.status === "complete",
|
|
1026
|
+
pageNumber: submission.pageNumber,
|
|
1027
|
+
newProductCount: applied.newProductCount,
|
|
1028
|
+
duplicateCount: applied.duplicateCount,
|
|
1029
|
+
productsAccepted: checkpoint.products.length,
|
|
1030
|
+
nextPageNumber: checkpoint.nextPageNumber
|
|
1031
|
+
}, req.headers.origin);
|
|
1032
|
+
if (checkpoint.status === "complete") {
|
|
1033
|
+
res.once("finish", () => void finish("complete"));
|
|
1034
|
+
}
|
|
1035
|
+
else if (options.maxPages !== undefined && checkpoint.nextPageNumber > options.maxPages) {
|
|
1036
|
+
acceptingRequests = false;
|
|
1037
|
+
res.once("finish", () => void requestIncompleteStop("page_limit"));
|
|
1038
|
+
}
|
|
1039
|
+
else if (options.maxRequests !== undefined && uniqueTasksIssuedThisRun >= options.maxRequests) {
|
|
1040
|
+
acceptingRequests = false;
|
|
1041
|
+
res.once("finish", () => void requestIncompleteStop("request_limit"));
|
|
1042
|
+
}
|
|
1043
|
+
return;
|
|
1044
|
+
}
|
|
1045
|
+
writeRelayJson(res, 404, { status: "not_found" }, req.headers.origin);
|
|
1046
|
+
}
|
|
1047
|
+
const handleSigint = () => {
|
|
1048
|
+
void requestIncompleteStop("interrupted", "The collector received SIGINT.").catch((error) => rejectResult?.(error));
|
|
1049
|
+
};
|
|
1050
|
+
const handleSigterm = () => {
|
|
1051
|
+
void requestIncompleteStop("interrupted", "The collector received SIGTERM.").catch((error) => rejectResult?.(error));
|
|
1052
|
+
};
|
|
1053
|
+
process.once("SIGINT", handleSigint);
|
|
1054
|
+
process.once("SIGTERM", handleSigterm);
|
|
1055
|
+
try {
|
|
1056
|
+
await new Promise((resolve, reject) => {
|
|
1057
|
+
server.once("error", reject);
|
|
1058
|
+
server.listen(options.relayPort, "127.0.0.1", () => resolve());
|
|
1059
|
+
});
|
|
1060
|
+
resetIdleTimer();
|
|
1061
|
+
const address = server.address();
|
|
1062
|
+
if (!address || typeof address === "string")
|
|
1063
|
+
throw new Error("The browser relay did not bind to TCP.");
|
|
1064
|
+
options.onListening?.({
|
|
1065
|
+
host: "127.0.0.1",
|
|
1066
|
+
port: address.port,
|
|
1067
|
+
taskUrl: `http://127.0.0.1:${address.port}/task`,
|
|
1068
|
+
pageUrl: `http://127.0.0.1:${address.port}/page`
|
|
1069
|
+
});
|
|
1070
|
+
return await resultPromise;
|
|
1071
|
+
}
|
|
1072
|
+
catch (error) {
|
|
1073
|
+
if (idleTimer)
|
|
1074
|
+
clearTimeout(idleTimer);
|
|
1075
|
+
process.off("SIGINT", handleSigint);
|
|
1076
|
+
process.off("SIGTERM", handleSigterm);
|
|
1077
|
+
if (server.listening)
|
|
1078
|
+
await closeServer(server);
|
|
1079
|
+
throw error;
|
|
1080
|
+
}
|
|
1081
|
+
}
|
package/package.json
CHANGED