unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +79 -33
- package/dist/index.js +2 -6
- package/dist/mcp.js +73 -22
- package/dist/server.js +25994 -0
- package/package.json +1 -2
- package/vendor/kuri/manifest.json +1 -1
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/analytics-session.ts +0 -55
- package/runtime-src/api/browse-index.ts +0 -343
- package/runtime-src/api/browse-session.ts +0 -572
- package/runtime-src/api/browse-submit-prereqs.ts +0 -48
- package/runtime-src/api/browse-submit.ts +0 -1184
- package/runtime-src/api/routes.ts +0 -1833
- package/runtime-src/auth/browser-cookies.ts +0 -423
- package/runtime-src/auth/index.ts +0 -535
- package/runtime-src/auth/runtime.ts +0 -116
- package/runtime-src/browser/index.ts +0 -659
- package/runtime-src/browser/types.ts +0 -41
- package/runtime-src/build-info.generated.ts +0 -6
- package/runtime-src/capture/index.ts +0 -1929
- package/runtime-src/capture/prefetch.ts +0 -95
- package/runtime-src/capture/rsc.ts +0 -45
- package/runtime-src/cli/shortcuts.ts +0 -273
- package/runtime-src/cli.ts +0 -1734
- package/runtime-src/client/graph-client.ts +0 -100
- package/runtime-src/client/index.ts +0 -1425
- package/runtime-src/debug-trace.ts +0 -18
- package/runtime-src/domain.ts +0 -38
- package/runtime-src/execution/index.ts +0 -3407
- package/runtime-src/execution/retry.ts +0 -46
- package/runtime-src/execution/robots.ts +0 -167
- package/runtime-src/execution/search-forms.ts +0 -188
- package/runtime-src/execution/token-resolver.ts +0 -135
- package/runtime-src/extraction/index.ts +0 -1507
- package/runtime-src/foundry/publish-bundle.ts +0 -392
- package/runtime-src/graph/agent-augment.ts +0 -315
- package/runtime-src/graph/index.ts +0 -1532
- package/runtime-src/graph/local-fixtures.ts +0 -393
- package/runtime-src/graph/local-harness.ts +0 -646
- package/runtime-src/graph/planner.ts +0 -411
- package/runtime-src/graph/session.ts +0 -294
- package/runtime-src/graph/trace-store.ts +0 -136
- package/runtime-src/impact-log.ts +0 -227
- package/runtime-src/index.ts +0 -24
- package/runtime-src/indexer/index.ts +0 -465
- package/runtime-src/intent-match.ts +0 -1515
- package/runtime-src/kuri/client.ts +0 -1839
- package/runtime-src/logger.ts +0 -30
- package/runtime-src/marketplace/index.ts +0 -111
- package/runtime-src/mcp.ts +0 -1911
- package/runtime-src/orchestrator/browser-agent.ts +0 -374
- package/runtime-src/orchestrator/dag-advisor.ts +0 -59
- package/runtime-src/orchestrator/dag-feedback.ts +0 -257
- package/runtime-src/orchestrator/first-pass-action.ts +0 -403
- package/runtime-src/orchestrator/index.ts +0 -4480
- package/runtime-src/orchestrator/passive-publish.ts +0 -187
- package/runtime-src/orchestrator/timing-economics.ts +0 -80
- package/runtime-src/payments/cascade.ts +0 -137
- package/runtime-src/payments/index.ts +0 -270
- package/runtime-src/payments/lobster-pay.ts +0 -182
- package/runtime-src/payments/wallet.ts +0 -98
- package/runtime-src/publish/review-context.ts +0 -93
- package/runtime-src/publish/sanitize.ts +0 -197
- package/runtime-src/publish/schema-review.ts +0 -192
- package/runtime-src/publish-admission.ts +0 -388
- package/runtime-src/ratelimit/index.ts +0 -23
- package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
- package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
- package/runtime-src/reverse-engineer/index.ts +0 -1551
- package/runtime-src/reverse-engineer/token-sources.ts +0 -379
- package/runtime-src/router.ts +0 -17
- package/runtime-src/routing-telemetry.ts +0 -395
- package/runtime-src/runtime/browser-access.ts +0 -11
- package/runtime-src/runtime/browser-auth.ts +0 -12
- package/runtime-src/runtime/browser-host.ts +0 -48
- package/runtime-src/runtime/lifecycle.ts +0 -17
- package/runtime-src/runtime/local-server.ts +0 -311
- package/runtime-src/runtime/paths.ts +0 -99
- package/runtime-src/runtime/setup.ts +0 -251
- package/runtime-src/runtime/supervisor.ts +0 -69
- package/runtime-src/runtime/update-hints.ts +0 -351
- package/runtime-src/server.ts +0 -100
- package/runtime-src/session-logs.ts +0 -142
- package/runtime-src/settings.ts +0 -221
- package/runtime-src/single-binary.ts +0 -143
- package/runtime-src/site-policy.ts +0 -54
- package/runtime-src/stale-cleanup-runner.ts +0 -144
- package/runtime-src/stale-cleanup.ts +0 -133
- package/runtime-src/telemetry-attribution.ts +0 -120
- package/runtime-src/telemetry.ts +0 -253
- package/runtime-src/template-params.ts +0 -141
- package/runtime-src/transform/drift.ts +0 -60
- package/runtime-src/transform/index.ts +0 -277
- package/runtime-src/types/index.ts +0 -1
- package/runtime-src/types/skill.ts +0 -931
- package/runtime-src/vault/index.ts +0 -196
- package/runtime-src/verification/auth-gate.ts +0 -8
- package/runtime-src/verification/candidates.ts +0 -27
- package/runtime-src/verification/index.ts +0 -120
- package/runtime-src/verification/matrix.ts +0 -30
- package/runtime-src/version.ts +0 -148
- package/runtime-src/workflow/artifact.ts +0 -161
- package/runtime-src/workflow/compile.ts +0 -808
- package/runtime-src/workflow/publish.ts +0 -225
- package/runtime-src/workflow/runtime.ts +0 -213
|
@@ -1,1507 +0,0 @@
|
|
|
1
|
-
import * as cheerio from "cheerio";
|
|
2
|
-
import { assessIntentResult } from "../intent-match.js";
|
|
3
|
-
|
|
4
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- cheerio v1.x doesn't export Element directly
|
|
5
|
-
type CheerioEl = any;
|
|
6
|
-
|
|
7
|
-
// --- Tag / attribute removal sets ---
|
|
8
|
-
|
|
9
|
-
const STRIP_TAGS = new Set(["script", "style", "noscript", "svg", "iframe"]);
|
|
10
|
-
const CHROME_TAGS = new Set(["nav", "footer", "header"]);
|
|
11
|
-
|
|
12
|
-
const AD_PATTERNS = /\b(ad|ads|advert|advertisement|tracking|tracker|cookie-banner|cookie-consent|cookie-notice|popup|modal-overlay|gdpr|consent|banner-promo)\b/i;
|
|
13
|
-
const HIDDEN_ATTRS: Array<{ attr: string; value?: string }> = [
|
|
14
|
-
{ attr: "aria-hidden", value: "true" },
|
|
15
|
-
{ attr: "hidden" },
|
|
16
|
-
];
|
|
17
|
-
|
|
18
|
-
// Selectors for "main content" regions — tried in priority order
|
|
19
|
-
const CONTENT_SELECTORS = [
|
|
20
|
-
"main",
|
|
21
|
-
"article",
|
|
22
|
-
"[role=\"main\"]",
|
|
23
|
-
"#content",
|
|
24
|
-
".content",
|
|
25
|
-
];
|
|
26
|
-
|
|
27
|
-
// Common repeating-element selectors for card detection
|
|
28
|
-
const CARD_SELECTORS = [
|
|
29
|
-
".card", ".item", ".result", ".product", ".listing",
|
|
30
|
-
".entry", ".post", ".tile", ".row",
|
|
31
|
-
"[class*='card']", "[class*='item']", "[class*='result']",
|
|
32
|
-
"[class*='product']", "[class*='listing']",
|
|
33
|
-
".cds-ProductCard-card", ".cds-ProductCard", "[class*='ProductCard-card']", "[class*='ProductCard']",
|
|
34
|
-
// Semantic HTML patterns — articles/sections as repeated items
|
|
35
|
-
"article", "section > div > div",
|
|
36
|
-
// Common e-commerce / catalog patterns
|
|
37
|
-
"[class*='pod']", "[class*='grid-item']", "[class*='col-']",
|
|
38
|
-
];
|
|
39
|
-
|
|
40
|
-
// ---------------------------------------------------------------------------
|
|
41
|
-
// extractSPAData — parse SPA-embedded JSON before cleanDOM strips scripts
|
|
42
|
-
// ---------------------------------------------------------------------------
|
|
43
|
-
|
|
44
|
-
interface SPAExtraction extends ExtractedStructure {
|
|
45
|
-
type: "spa-nextjs" | "spa-nuxt" | "spa-initial-state" | "spa-preloaded-state";
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function extractFlashNoticeSpecial(html: string, intent: string): ExtractedStructure[] {
|
|
49
|
-
if (!/\b(flash|message|messages|alert|success|error|warning)\b/i.test(intent)) return [];
|
|
50
|
-
const $ = cheerio.load(html);
|
|
51
|
-
const flash = $("#flash, .flash, .alert, [role='alert']").first();
|
|
52
|
-
if (flash.length === 0) return [];
|
|
53
|
-
const flashText = flash.text().replace(/\s+/g, " ").replace(/[×x]\s*$/, "").trim();
|
|
54
|
-
if (!flashText || flashText.length < 4) return [];
|
|
55
|
-
const title = $("main h1, main h2, article h1, article h2, h1, h2").first().text().trim();
|
|
56
|
-
return [{
|
|
57
|
-
type: "key-value",
|
|
58
|
-
data: {
|
|
59
|
-
...(title ? { title } : {}),
|
|
60
|
-
flash: flashText,
|
|
61
|
-
message: flashText,
|
|
62
|
-
},
|
|
63
|
-
element_count: title ? 2 : 1,
|
|
64
|
-
selector: buildReplaySelector(flash),
|
|
65
|
-
}];
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
/**
|
|
69
|
-
* Extract structured data embedded by SPA frameworks BEFORE cleanDOM strips scripts.
|
|
70
|
-
* Must be called on raw HTML.
|
|
71
|
-
*/
|
|
72
|
-
export function extractSPAData(html: string): SPAExtraction[] {
|
|
73
|
-
const results: SPAExtraction[] = [];
|
|
74
|
-
|
|
75
|
-
// --- Next.js: <script id="__NEXT_DATA__" type="application/json"> ---
|
|
76
|
-
const nextDataMatch = html.match(/<script\s+id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/i);
|
|
77
|
-
if (nextDataMatch) {
|
|
78
|
-
try {
|
|
79
|
-
const parsed = JSON.parse(nextDataMatch[1]);
|
|
80
|
-
const pageProps = parsed?.props?.pageProps;
|
|
81
|
-
if (pageProps && typeof pageProps === "object" && Object.keys(pageProps).length > 0) {
|
|
82
|
-
results.push({
|
|
83
|
-
type: "spa-nextjs",
|
|
84
|
-
data: pageProps,
|
|
85
|
-
element_count: countDataElements(pageProps),
|
|
86
|
-
});
|
|
87
|
-
}
|
|
88
|
-
} catch { /* malformed __NEXT_DATA__ */ }
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
// --- Nuxt.js: window.__NUXT__={...} or <script>window.__NUXT__=... ---
|
|
92
|
-
const nuxtMatch = html.match(/window\.__NUXT__\s*=\s*(\{[\s\S]*?\});?\s*(?:<\/script>|$)/i);
|
|
93
|
-
if (nuxtMatch) {
|
|
94
|
-
try {
|
|
95
|
-
const parsed = JSON.parse(nuxtMatch[1]);
|
|
96
|
-
const data = parsed?.data?.[0] ?? parsed?.state ?? parsed;
|
|
97
|
-
if (data && typeof data === "object" && Object.keys(data).length > 0) {
|
|
98
|
-
results.push({
|
|
99
|
-
type: "spa-nuxt",
|
|
100
|
-
data,
|
|
101
|
-
element_count: countDataElements(data),
|
|
102
|
-
});
|
|
103
|
-
}
|
|
104
|
-
} catch { /* malformed __NUXT__ — often not pure JSON, skip */ }
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
// --- Generic: window.__INITIAL_STATE__ ---
|
|
108
|
-
const initialStateMatch = html.match(/window\.__INITIAL_STATE__\s*=\s*(\{[\s\S]*?\});?\s*(?:<\/script>|$)/i);
|
|
109
|
-
if (initialStateMatch) {
|
|
110
|
-
try {
|
|
111
|
-
const parsed = JSON.parse(initialStateMatch[1]);
|
|
112
|
-
if (parsed && typeof parsed === "object" && Object.keys(parsed).length > 0) {
|
|
113
|
-
results.push({
|
|
114
|
-
type: "spa-initial-state",
|
|
115
|
-
data: parsed,
|
|
116
|
-
element_count: countDataElements(parsed),
|
|
117
|
-
});
|
|
118
|
-
}
|
|
119
|
-
} catch { /* malformed __INITIAL_STATE__ */ }
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
// --- Generic: window.__PRELOADED_STATE__ ---
|
|
123
|
-
const preloadedMatch = html.match(/window\.__PRELOADED_STATE__\s*=\s*(\{[\s\S]*?\});?\s*(?:<\/script>|$)/i);
|
|
124
|
-
if (preloadedMatch) {
|
|
125
|
-
try {
|
|
126
|
-
const parsed = JSON.parse(preloadedMatch[1]);
|
|
127
|
-
if (parsed && typeof parsed === "object" && Object.keys(parsed).length > 0) {
|
|
128
|
-
results.push({
|
|
129
|
-
type: "spa-preloaded-state",
|
|
130
|
-
data: parsed,
|
|
131
|
-
element_count: countDataElements(parsed),
|
|
132
|
-
});
|
|
133
|
-
}
|
|
134
|
-
} catch { /* malformed __PRELOADED_STATE__ */ }
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
return results;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/** Count meaningful data elements in a nested structure */
|
|
141
|
-
function countDataElements(obj: unknown, depth = 0): number {
|
|
142
|
-
if (depth > 5) return 0;
|
|
143
|
-
if (Array.isArray(obj)) return obj.reduce((sum, item) => sum + Math.max(1, countDataElements(item, depth + 1)), 0);
|
|
144
|
-
if (obj && typeof obj === "object") {
|
|
145
|
-
const keys = Object.keys(obj);
|
|
146
|
-
return keys.reduce((sum, k) => sum + countDataElements((obj as Record<string, unknown>)[k], depth + 1), 0);
|
|
147
|
-
}
|
|
148
|
-
return 1;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
// ---------------------------------------------------------------------------
|
|
152
|
-
// cleanDOM
|
|
153
|
-
// ---------------------------------------------------------------------------
|
|
154
|
-
|
|
155
|
-
/**
|
|
156
|
-
* Strip noise from raw page HTML — remove scripts, styles, nav chrome,
|
|
157
|
-
* ads, hidden elements. Prefer content inside main/article regions.
|
|
158
|
-
*/
|
|
159
|
-
export function cleanDOM(html: string): string {
|
|
160
|
-
const $ = cheerio.load(html);
|
|
161
|
-
|
|
162
|
-
// 1. Remove script/style/svg/iframe/noscript tags entirely
|
|
163
|
-
// Preserve JSON-LD scripts — they contain structured data
|
|
164
|
-
for (const tag of STRIP_TAGS) {
|
|
165
|
-
if (tag === "script") {
|
|
166
|
-
$("script").not('[type="application/ld+json"]').remove();
|
|
167
|
-
} else {
|
|
168
|
-
$(tag).remove();
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
// 2. Remove navigation chrome
|
|
173
|
-
for (const tag of CHROME_TAGS) {
|
|
174
|
-
$(tag).remove();
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
// 3. Remove ad/tracking elements by class/id
|
|
178
|
-
$("*").each((_, el) => {
|
|
179
|
-
const $el = $(el);
|
|
180
|
-
const cls = $el.attr("class") ?? "";
|
|
181
|
-
const id = $el.attr("id") ?? "";
|
|
182
|
-
if (AD_PATTERNS.test(cls) || AD_PATTERNS.test(id)) {
|
|
183
|
-
$el.remove();
|
|
184
|
-
}
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
// 4. Remove hidden elements
|
|
188
|
-
$("[style]").each((_, el) => {
|
|
189
|
-
const $el = $(el);
|
|
190
|
-
const style = ($el.attr("style") ?? "").replace(/\s/g, "");
|
|
191
|
-
if (style.includes("display:none") || style.includes("visibility:hidden")) {
|
|
192
|
-
$el.remove();
|
|
193
|
-
}
|
|
194
|
-
});
|
|
195
|
-
for (const { attr, value } of HIDDEN_ATTRS) {
|
|
196
|
-
const selector = value ? `[${attr}="${value}"]` : `[${attr}]`;
|
|
197
|
-
$(selector).remove();
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
// 5. Prefer content region if available (but only if it's a single container,
|
|
201
|
-
// not multiple repeating elements like <article> per product)
|
|
202
|
-
for (const sel of CONTENT_SELECTORS) {
|
|
203
|
-
const region = $(sel);
|
|
204
|
-
if (region.length === 1 && region.text().trim().length > 100) {
|
|
205
|
-
return region.html() ?? $.html();
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
return $("body").html() ?? $.html();
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
// ---------------------------------------------------------------------------
|
|
213
|
-
// parseStructured
|
|
214
|
-
// ---------------------------------------------------------------------------
|
|
215
|
-
|
|
216
|
-
interface ExtractedStructure {
|
|
217
|
-
type: string;
|
|
218
|
-
data: unknown;
|
|
219
|
-
element_count: number;
|
|
220
|
-
selector?: string;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
function hasMessageLikeRecord(value: unknown): boolean {
|
|
224
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
225
|
-
const record = value as Record<string, unknown>;
|
|
226
|
-
return (
|
|
227
|
-
(typeof record.title === "string" || typeof record.heading_1 === "string" || typeof record.heading === "string") &&
|
|
228
|
-
(typeof record.message === "string" || typeof record.description === "string" || typeof record.flash === "string")
|
|
229
|
-
);
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
function isMessageLikeStructure(structure: ExtractedStructure, intent: string): boolean {
|
|
233
|
-
if (!/\b(message|messages|flash|alert|success|error|warning)\b/i.test(intent)) return false;
|
|
234
|
-
if (Array.isArray(structure.data)) return structure.data.some((item) => hasMessageLikeRecord(item));
|
|
235
|
-
return hasMessageLikeRecord(structure.data);
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
function pruneRowsForIntent(rows: Record<string, string>[], intent: string): Record<string, string>[] {
|
|
239
|
-
const lower = intent.toLowerCase();
|
|
240
|
-
const keep = (predicate: (row: Record<string, string>) => boolean): Record<string, string>[] => rows.filter(predicate);
|
|
241
|
-
|
|
242
|
-
if (/\b(question|questions)\b/.test(lower)) {
|
|
243
|
-
return keep((row) =>
|
|
244
|
-
!!(row.title || row.name) &&
|
|
245
|
-
!!(row.url || row.link) &&
|
|
246
|
-
!!(row.score || row.answer_count || row.author || row.date || row.meta || row.description) &&
|
|
247
|
-
String(row.title ?? row.name ?? "").trim().length > 12
|
|
248
|
-
);
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
if (/\b(post|posts|tweet|tweets|status|statuses)\b/.test(lower)) {
|
|
252
|
-
return keep((row) =>
|
|
253
|
-
!!(row.title || row.text || row.description) &&
|
|
254
|
-
!!(row.url || row.link) &&
|
|
255
|
-
!!(row.author || row.score || row.date || row.meta || row.description || row.text)
|
|
256
|
-
);
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
if (/\b(doc|docs|documentation)\b/.test(lower)) {
|
|
260
|
-
return keep((row) =>
|
|
261
|
-
!!(row.title || row.name) &&
|
|
262
|
-
!!(row.url || row.link) &&
|
|
263
|
-
!!(row.summary || row.description || row.slug || row.meta)
|
|
264
|
-
);
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
if (/\b(paper|papers)\b/.test(lower)) {
|
|
268
|
-
return keep((row) =>
|
|
269
|
-
!!(row.title || row.name) &&
|
|
270
|
-
!!(row.url || row.link) &&
|
|
271
|
-
!!(row.summary || row.description || row.author || row.date || row.meta)
|
|
272
|
-
);
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
if (/\b(definition|dictionary|meaning)\b/.test(lower)) {
|
|
276
|
-
return keep((row) =>
|
|
277
|
-
!!(row.term || row.title || row.name) &&
|
|
278
|
-
!!(row.definition || row.description)
|
|
279
|
-
);
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
if (/\b(recipe|recipes)\b/.test(lower)) {
|
|
283
|
-
return keep((row) =>
|
|
284
|
-
!!(row.title || row.name) &&
|
|
285
|
-
!!(row.url || row.link) &&
|
|
286
|
-
!!(row.rating || row.description || row.author || row.meta)
|
|
287
|
-
);
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
if (/\b(course|courses)\b/.test(lower)) {
|
|
291
|
-
return keep((row) =>
|
|
292
|
-
!!(row.title || row.name) &&
|
|
293
|
-
!!(row.url || row.link) &&
|
|
294
|
-
!!(row.rating || row.description || row.author || row.meta)
|
|
295
|
-
);
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
return rows;
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
function normalizeStructureForIntent(structure: ExtractedStructure, intent: string): ExtractedStructure {
|
|
302
|
-
if (structure.type !== "repeated-elements" || !Array.isArray(structure.data)) return structure;
|
|
303
|
-
const objectRows = (structure.data as unknown[]).filter((row): row is Record<string, string> => !!row && typeof row === "object" && !Array.isArray(row));
|
|
304
|
-
if (objectRows.length === 0) return structure;
|
|
305
|
-
const pruned = pruneRowsForIntent(objectRows, intent);
|
|
306
|
-
if (pruned.length >= 1 && pruned.length < objectRows.length) {
|
|
307
|
-
return {
|
|
308
|
-
...structure,
|
|
309
|
-
data: pruned,
|
|
310
|
-
element_count: pruned.length,
|
|
311
|
-
};
|
|
312
|
-
}
|
|
313
|
-
return structure;
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
function normalizeGitHubPath(href: string | undefined): string | null {
|
|
317
|
-
if (!href) return null;
|
|
318
|
-
const clean = href.split("?")[0].replace(/\/+$/, "");
|
|
319
|
-
const match = clean.match(/^\/([^/]+)\/([^/]+)$/);
|
|
320
|
-
if (!match) return null;
|
|
321
|
-
const owner = match[1];
|
|
322
|
-
const repo = match[2];
|
|
323
|
-
if (/^(features|topics|collections|marketplace|orgs|users|settings|login|signup|sponsors|pricing|search|notifications|explore|pulls|issues)$/.test(owner)) return null;
|
|
324
|
-
return `${owner}/${repo}`;
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
function cleanText(value: string): string {
|
|
328
|
-
return value.replace(/\s+/g, " ").trim();
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
function normalizeLinkedInProfilePath(href: string | undefined): string | null {
|
|
332
|
-
if (!href) return null;
|
|
333
|
-
const clean = href.split("?")[0].replace(/\/+$/, "");
|
|
334
|
-
const match = clean.match(/\/in\/([^/]+)$/);
|
|
335
|
-
return match ? match[1] : null;
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
function extractGitHubSpecial(html: string, intent: string): ExtractedStructure[] {
|
|
339
|
-
if (
|
|
340
|
-
!/github/i.test(html) &&
|
|
341
|
-
!/href=["']\/[^/"']+\/[^/"']+["']/i.test(html) &&
|
|
342
|
-
!/data-target="react-app\.embeddedData"/i.test(html)
|
|
343
|
-
) return [];
|
|
344
|
-
const $ = cheerio.load(html);
|
|
345
|
-
const results: ExtractedStructure[] = [];
|
|
346
|
-
const intentLower = intent.toLowerCase();
|
|
347
|
-
|
|
348
|
-
const embeddedDataMatch = html.match(/<script[^>]+data-target="react-app\.embeddedData"[^>]*>([\s\S]*?)<\/script>/i);
|
|
349
|
-
if (embeddedDataMatch && intentLower.includes("search")) {
|
|
350
|
-
try {
|
|
351
|
-
const parsed = JSON.parse(embeddedDataMatch[1]);
|
|
352
|
-
const embeddedResults = parsed?.payload?.results;
|
|
353
|
-
if (Array.isArray(embeddedResults) && embeddedResults.length >= 2) {
|
|
354
|
-
const repos = embeddedResults
|
|
355
|
-
.map((item: Record<string, unknown>) => {
|
|
356
|
-
const repo = item.repo as { repository?: { owner_login?: string; name?: string } } | undefined;
|
|
357
|
-
const owner = repo?.repository?.owner_login;
|
|
358
|
-
const name = repo?.repository?.name;
|
|
359
|
-
if (!owner || !name) return null;
|
|
360
|
-
const row: Record<string, string> = {
|
|
361
|
-
full_name: `${owner}/${name}`,
|
|
362
|
-
url: `https://github.com/${owner}/${name}`,
|
|
363
|
-
};
|
|
364
|
-
const description = String(item.hl_trunc_description ?? "").replace(/<[^>]+>/g, "").trim();
|
|
365
|
-
const language = String(item.language ?? "").trim();
|
|
366
|
-
const stars = item.followers != null ? String(item.followers) : "";
|
|
367
|
-
if (description) row.description = description;
|
|
368
|
-
if (language) row.language = language;
|
|
369
|
-
if (stars) row.stargazers_count = stars;
|
|
370
|
-
return row;
|
|
371
|
-
})
|
|
372
|
-
.filter((row): row is Record<string, string> => !!row);
|
|
373
|
-
if (repos.length >= 2) {
|
|
374
|
-
results.push({ type: "repeated-elements", data: repos.slice(0, 20), element_count: repos.length });
|
|
375
|
-
}
|
|
376
|
-
}
|
|
377
|
-
} catch { /* malformed embedded data */ }
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
const repoNwo = $('meta[name="octolytics-dimension-repository_nwo"]').attr("content")?.trim();
|
|
381
|
-
if (repoNwo && (intentLower.includes("repository") || intentLower.includes("repo"))) {
|
|
382
|
-
const ogDesc = $('meta[property="og:description"]').attr("content")?.trim() || "";
|
|
383
|
-
const stars = $("#repo-stars-counter-star").first().text().trim()
|
|
384
|
-
|| $('a[href$="/stargazers"]').first().text().replace(/\s+/g, " ").trim();
|
|
385
|
-
const forks = $('a[href$="/forks"]').first().text().replace(/\s+/g, " ").trim();
|
|
386
|
-
const about = $("h2").filter((_, el) => $(el).text().trim() === "About").first()
|
|
387
|
-
.parent().text().replace(/\s+/g, " ").trim();
|
|
388
|
-
const data: Record<string, string> = {
|
|
389
|
-
full_name: repoNwo,
|
|
390
|
-
description: ogDesc || $('meta[name="description"]').attr("content")?.trim() || "",
|
|
391
|
-
url: $('meta[property="og:url"]').attr("content")?.trim() || `https://github.com/${repoNwo}`,
|
|
392
|
-
};
|
|
393
|
-
if (stars) data.stars = stars;
|
|
394
|
-
if (forks) data.forks = forks;
|
|
395
|
-
if (about && about.length > 20 && about.length < 500) data.about = about;
|
|
396
|
-
results.push({ type: "key-value", data, element_count: Object.keys(data).length });
|
|
397
|
-
}
|
|
398
|
-
|
|
399
|
-
if ((/search-results-page/.test(html) || /\/search\?/.test(html) || /resultsrepositories/i.test(html) || /href=["']\/[^/"']+\/[^/"']+["']/i.test(html)) && intentLower.includes("search")) {
|
|
400
|
-
const seen = new Set<string>();
|
|
401
|
-
const repos: Record<string, string>[] = [];
|
|
402
|
-
$(".search-title a[href], [data-testid='results-list'] a[href], .search-results-container a[href], a[href^='/']").each((_, el) => {
|
|
403
|
-
const $a = $(el);
|
|
404
|
-
const href = $a.attr("href");
|
|
405
|
-
const fullName = normalizeGitHubPath(href);
|
|
406
|
-
if (!fullName || seen.has(fullName)) return;
|
|
407
|
-
const title = cleanText($a.text());
|
|
408
|
-
if (!title || title.length > 120) return;
|
|
409
|
-
const card = $a.closest("div, li");
|
|
410
|
-
const cardText = cleanText(card.text());
|
|
411
|
-
const desc = cleanText(card.find("p").first().text());
|
|
412
|
-
const lang = cleanText(card.find("[itemprop='programmingLanguage']").first().text());
|
|
413
|
-
if (!/star|fork|updated|results?|language|repository/i.test(cardText) && !desc && !lang) return;
|
|
414
|
-
seen.add(fullName);
|
|
415
|
-
const stars = cleanText(card.find("a[href$='/stargazers']").first().text());
|
|
416
|
-
const row: Record<string, string> = {
|
|
417
|
-
full_name: fullName,
|
|
418
|
-
url: `https://github.com/${fullName}`,
|
|
419
|
-
};
|
|
420
|
-
if (desc) row.description = desc;
|
|
421
|
-
if (lang) row.language = lang;
|
|
422
|
-
if (stars) row.stars = stars;
|
|
423
|
-
repos.push(row);
|
|
424
|
-
});
|
|
425
|
-
if (repos.length >= 2) {
|
|
426
|
-
results.push({ type: "repeated-elements", data: repos.slice(0, 10), element_count: repos.length });
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
if (/trending/i.test(intentLower) || /\/trending\b/.test(html)) {
|
|
431
|
-
const seen = new Set<string>();
|
|
432
|
-
const repos: Record<string, string>[] = [];
|
|
433
|
-
$("article.Box-row, article, .Box-row").each((_, el) => {
|
|
434
|
-
const $el = $(el);
|
|
435
|
-
const repoLink = $el.find("h1 a[href], h2 a[href], a[href^='/']").filter((_, a) => !!normalizeGitHubPath($(a).attr("href"))).first();
|
|
436
|
-
const fullName = normalizeGitHubPath(repoLink.attr("href"));
|
|
437
|
-
if (!fullName || seen.has(fullName)) return;
|
|
438
|
-
const desc = $el.find("p").first().text().replace(/\s+/g, " ").trim();
|
|
439
|
-
const lang = $el.find('[itemprop="programmingLanguage"]').first().text().trim();
|
|
440
|
-
const stars = $el.find('a[href$="/stargazers"]').first().text().replace(/\s+/g, " ").trim();
|
|
441
|
-
seen.add(fullName);
|
|
442
|
-
const row: Record<string, string> = {
|
|
443
|
-
full_name: fullName,
|
|
444
|
-
url: `https://github.com/${fullName}`,
|
|
445
|
-
};
|
|
446
|
-
if (desc) row.description = desc;
|
|
447
|
-
if (lang) row.language = lang;
|
|
448
|
-
if (stars) row.stars = stars;
|
|
449
|
-
repos.push(row);
|
|
450
|
-
});
|
|
451
|
-
if (repos.length >= 2) {
|
|
452
|
-
results.push({ type: "repeated-elements", data: repos.slice(0, 20), element_count: repos.length });
|
|
453
|
-
}
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
return results;
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
function extractLinkedInSpecial(html: string, intent: string): ExtractedStructure[] {
|
|
460
|
-
if (!/linkedin/i.test(html)) return [];
|
|
461
|
-
const intentLower = intent.toLowerCase();
|
|
462
|
-
if (!/(search|people|person|profile|member)/.test(intentLower)) return [];
|
|
463
|
-
const $ = cheerio.load(html);
|
|
464
|
-
const seen = new Set<string>();
|
|
465
|
-
const people: Record<string, string>[] = [];
|
|
466
|
-
|
|
467
|
-
$("a[href*='/in/']").each((_, el) => {
|
|
468
|
-
const $a = $(el);
|
|
469
|
-
const handle = normalizeLinkedInProfilePath($a.attr("href"));
|
|
470
|
-
if (!handle || seen.has(handle)) return;
|
|
471
|
-
const name = cleanText($a.text());
|
|
472
|
-
if (!name || name.length < 3 || name.length > 120) return;
|
|
473
|
-
const card = $a.closest("li, div");
|
|
474
|
-
const cardText = cleanText(card.text());
|
|
475
|
-
if (cardText.length < name.length + 5) return;
|
|
476
|
-
const headline = cleanText(
|
|
477
|
-
card.find("div, span, p")
|
|
478
|
-
.map((_, node) => cleanText($(node).text()))
|
|
479
|
-
.get()
|
|
480
|
-
.find((text) =>
|
|
481
|
-
!!text &&
|
|
482
|
-
text !== name &&
|
|
483
|
-
text.length >= 8 &&
|
|
484
|
-
text.length <= 220 &&
|
|
485
|
-
!/^(message|connect|follow|premium|linkedin|see more|show all)$/i.test(text)
|
|
486
|
-
) ?? ""
|
|
487
|
-
);
|
|
488
|
-
const row: Record<string, string> = {
|
|
489
|
-
name,
|
|
490
|
-
url: `https://www.linkedin.com/in/${handle}`,
|
|
491
|
-
public_identifier: handle,
|
|
492
|
-
};
|
|
493
|
-
if (headline) row.headline = headline;
|
|
494
|
-
people.push(row);
|
|
495
|
-
seen.add(handle);
|
|
496
|
-
});
|
|
497
|
-
|
|
498
|
-
return people.length >= 2 ? [{ type: "repeated-elements", data: people.slice(0, 10), element_count: people.length }] : [];
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
function extractPackageSearchSpecial(html: string, intent: string): ExtractedStructure[] {
|
|
502
|
-
const intentLower = intent.toLowerCase();
|
|
503
|
-
if (!/\bsearch\b/.test(intentLower) || !/\b(package|packages|crate|crates)\b/.test(intentLower)) return [];
|
|
504
|
-
if (!/package-snippet/i.test(html)) return [];
|
|
505
|
-
const $ = cheerio.load(html);
|
|
506
|
-
const rows: Record<string, string>[] = [];
|
|
507
|
-
const seen = new Set<string>();
|
|
508
|
-
|
|
509
|
-
$("a.package-snippet[href], .package-snippet").each((_, el) => {
|
|
510
|
-
const $el = $(el);
|
|
511
|
-
const name = cleanText($el.find(".package-snippet__name").first().text());
|
|
512
|
-
if (!name || seen.has(name)) return;
|
|
513
|
-
const version = cleanText($el.find(".package-snippet__version").first().text());
|
|
514
|
-
const description = cleanText($el.find(".package-snippet__description").first().text());
|
|
515
|
-
const href = $el.attr("href") ?? "";
|
|
516
|
-
rows.push({
|
|
517
|
-
name,
|
|
518
|
-
...(version ? { version } : {}),
|
|
519
|
-
...(description ? { description } : {}),
|
|
520
|
-
url: href ? new URL(href, "https://pypi.org").toString() : `https://pypi.org/project/${encodeURIComponent(name)}/`,
|
|
521
|
-
});
|
|
522
|
-
seen.add(name);
|
|
523
|
-
});
|
|
524
|
-
|
|
525
|
-
return rows.length >= 2 ? [{ type: "repeated-elements", data: rows.slice(0, 20), element_count: rows.length }] : [];
|
|
526
|
-
}
|
|
527
|
-
|
|
528
|
-
function extractXProfileSpecial(html: string, intent: string): ExtractedStructure[] {
|
|
529
|
-
const intentLower = intent.toLowerCase();
|
|
530
|
-
if (!/(person|people|profile|profiles|user|users|member)/.test(intentLower)) return [];
|
|
531
|
-
if (!/(twitter|x\.com|twitter:|og:)/i.test(html)) return [];
|
|
532
|
-
const $ = cheerio.load(html);
|
|
533
|
-
const title = cleanText($("title").first().text());
|
|
534
|
-
const ogTitle = cleanText($('meta[property="og:title"]').attr("content") ?? $('meta[name="twitter:title"]').attr("content") ?? "");
|
|
535
|
-
const description = cleanText($('meta[name="description"]').attr("content") ?? $('meta[property="og:description"]').attr("content") ?? "");
|
|
536
|
-
const canonical = ($('link[rel="canonical"]').attr("href") ?? $('meta[property="og:url"]').attr("content") ?? "").trim();
|
|
537
|
-
|
|
538
|
-
const source = ogTitle || title;
|
|
539
|
-
const titleMatch = source.match(/^(.*?)\s*\(@?([A-Za-z0-9_]{1,30})\)/);
|
|
540
|
-
const handleFromUrl = canonical.match(/https?:\/\/(?:www\.)?x\.com\/([A-Za-z0-9_]{1,30})(?:\/|$)/)?.[1]
|
|
541
|
-
?? canonical.match(/https?:\/\/(?:www\.)?twitter\.com\/([A-Za-z0-9_]{1,30})(?:\/|$)/)?.[1];
|
|
542
|
-
const username = titleMatch?.[2] ?? handleFromUrl ?? "";
|
|
543
|
-
const name = cleanText(titleMatch?.[1] ?? source.replace(/\s*\/\s*[XT]$/i, ""));
|
|
544
|
-
|
|
545
|
-
if (!name || !username) return [];
|
|
546
|
-
|
|
547
|
-
const row: Record<string, string> = {
|
|
548
|
-
name,
|
|
549
|
-
username,
|
|
550
|
-
public_identifier: username,
|
|
551
|
-
url: canonical || `https://x.com/${username}`,
|
|
552
|
-
};
|
|
553
|
-
if (description) row.description = description;
|
|
554
|
-
return [{ type: "key-value", data: row, element_count: 1 }];
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
function extractPostSpecial(html: string, intent: string): ExtractedStructure[] {
|
|
558
|
-
const intentLower = intent.toLowerCase();
|
|
559
|
-
if (!/(post|posts|tweet|tweets|status|statuses)/.test(intentLower)) return [];
|
|
560
|
-
const $ = cheerio.load(html);
|
|
561
|
-
const seen = new Set<string>();
|
|
562
|
-
const posts: Record<string, string>[] = [];
|
|
563
|
-
|
|
564
|
-
$("article, [role='article'], li, div").each((_, el) => {
|
|
565
|
-
const $el = $(el);
|
|
566
|
-
const link = $el.find("a[href*='/status/'], a[href*='/statuses/'], a[href*='/posts/'], a[href*='/@'], a[href*='/s/']").first();
|
|
567
|
-
const href = link.attr("href");
|
|
568
|
-
if (!href || href.length > 300) return;
|
|
569
|
-
const canonical = href.split("?")[0];
|
|
570
|
-
if (seen.has(canonical)) return;
|
|
571
|
-
const title = cleanText(link.text());
|
|
572
|
-
|
|
573
|
-
const text = cleanText(
|
|
574
|
-
$el.find("p, span, div")
|
|
575
|
-
.map((__, node) => cleanText($(node).text()))
|
|
576
|
-
.get()
|
|
577
|
-
.filter((value) =>
|
|
578
|
-
value.length >= 20 &&
|
|
579
|
-
value.length <= 700 &&
|
|
580
|
-
!/^(reply|repost|like|share|show more|show less|follow|message)$/i.test(value)
|
|
581
|
-
)
|
|
582
|
-
.sort((a, b) => b.length - a.length)[0] ?? ""
|
|
583
|
-
);
|
|
584
|
-
|
|
585
|
-
const mastodonMatch = canonical.match(/\/@([^/]+)\/(\d+)/);
|
|
586
|
-
const statusMatch = canonical.match(/\/status\/(\d+)/);
|
|
587
|
-
const lobstersMatch = canonical.match(/\/s\/([^/]+)/);
|
|
588
|
-
const username = mastodonMatch?.[1]
|
|
589
|
-
?? canonical.match(/\/([^/@]+)\/status\/\d+/)?.[1]
|
|
590
|
-
?? cleanText($el.find("[class*='author'], [class*='byline'], .u-author").first().text())
|
|
591
|
-
?? "";
|
|
592
|
-
const id = mastodonMatch?.[2] ?? statusMatch?.[1] ?? lobstersMatch?.[1] ?? canonical.split("/").pop() ?? "";
|
|
593
|
-
const score = cleanText($el.find("[class*='score'], [class*='points']").first().text());
|
|
594
|
-
|
|
595
|
-
if (!text && !username && !title) return;
|
|
596
|
-
|
|
597
|
-
posts.push({
|
|
598
|
-
...(id ? { id } : {}),
|
|
599
|
-
...(username ? { username } : {}),
|
|
600
|
-
url: canonical,
|
|
601
|
-
...(title ? { title } : {}),
|
|
602
|
-
...(text ? { text } : {}),
|
|
603
|
-
...(score ? { score } : {}),
|
|
604
|
-
...(username ? { author: username } : {}),
|
|
605
|
-
});
|
|
606
|
-
seen.add(canonical);
|
|
607
|
-
});
|
|
608
|
-
|
|
609
|
-
return posts.length >= 1 ? [{ type: "repeated-elements", data: posts.slice(0, 20), element_count: posts.length }] : [];
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
function extractDefinitionSpecial(html: string, intent: string): ExtractedStructure[] {
|
|
613
|
-
const intentLower = intent.toLowerCase();
|
|
614
|
-
if (!/(definition|dictionary|meaning)/.test(intentLower)) return [];
|
|
615
|
-
const $ = cheerio.load(html);
|
|
616
|
-
const root = $("main, article, [role='main'], .entry-body, .di-body").first();
|
|
617
|
-
const scope = root.length > 0 ? root : $("body");
|
|
618
|
-
const term = cleanText(scope.find("h1").first().text()) || cleanText($("h1").first().text());
|
|
619
|
-
let definition = cleanText(
|
|
620
|
-
scope.find("dd, [class*='def'], [class*='meaning'], [class*='definition']").first().text(),
|
|
621
|
-
);
|
|
622
|
-
let normalizedTerm = term;
|
|
623
|
-
if ((!normalizedTerm || !definition || definition.length < 10)) {
|
|
624
|
-
const ogTitle = cleanText($('meta[property="og:title"]').attr("content") ?? "");
|
|
625
|
-
const metaDescription = cleanText($('meta[name="description"]').attr("content") ?? $('meta[itemprop="headline"]').attr("content") ?? "");
|
|
626
|
-
const canonical = cleanText($('link[rel="canonical"]').attr("href") ?? "");
|
|
627
|
-
if (!normalizedTerm) {
|
|
628
|
-
normalizedTerm = ogTitle
|
|
629
|
-
|| canonical.split("/").filter(Boolean).pop()?.replace(/[-_]+/g, " ")
|
|
630
|
-
|| "";
|
|
631
|
-
}
|
|
632
|
-
if (!definition && metaDescription) {
|
|
633
|
-
definition = metaDescription
|
|
634
|
-
.replace(/^[A-Z0-9 _-]+\s+definition:\s*/i, "")
|
|
635
|
-
.replace(/\s*Learn more\.?$/i, "")
|
|
636
|
-
.replace(/…/g, "...")
|
|
637
|
-
.trim();
|
|
638
|
-
}
|
|
639
|
-
}
|
|
640
|
-
if (!normalizedTerm || !definition || definition.length < 10) return [];
|
|
641
|
-
return [{
|
|
642
|
-
type: "key-value",
|
|
643
|
-
data: {
|
|
644
|
-
term: normalizedTerm,
|
|
645
|
-
title: normalizedTerm,
|
|
646
|
-
definition,
|
|
647
|
-
},
|
|
648
|
-
element_count: 1,
|
|
649
|
-
}];
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
function extractCourseSearchSpecial(html: string, intent: string): ExtractedStructure[] {
|
|
653
|
-
if (!/\b(course|courses)\b/i.test(intent)) return [];
|
|
654
|
-
if (!/ProductCard|CommonCard-titleLink|RatingStat|partnerNames/i.test(html)) return [];
|
|
655
|
-
const $ = cheerio.load(html);
|
|
656
|
-
const rows: Record<string, string>[] = [];
|
|
657
|
-
const seen = new Set<string>();
|
|
658
|
-
|
|
659
|
-
$(".cds-ProductCard-card, .cds-ProductCard, [class*='ProductCard-card'], [class*='ProductCard']").each((_, el) => {
|
|
660
|
-
const $el = $(el);
|
|
661
|
-
const fields = extractCardFields($, $el);
|
|
662
|
-
const title = fields.title?.trim();
|
|
663
|
-
const url = (fields.url ?? fields.link ?? "").trim();
|
|
664
|
-
if (!title || !url || title === "All Results") return;
|
|
665
|
-
const stable = `${title}|${url}`;
|
|
666
|
-
if (seen.has(stable)) return;
|
|
667
|
-
if (!fields.rating && !fields.partner && !fields.description) return;
|
|
668
|
-
rows.push(fields);
|
|
669
|
-
seen.add(stable);
|
|
670
|
-
});
|
|
671
|
-
|
|
672
|
-
return rows.length >= 2
|
|
673
|
-
? [{ type: "repeated-elements", data: rows.slice(0, 20), element_count: rows.length, selector: ".cds-ProductCard-card" }]
|
|
674
|
-
: [];
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
function extractTrendSpecial(html: string, intent: string): ExtractedStructure[] {
|
|
678
|
-
const intentLower = intent.toLowerCase();
|
|
679
|
-
if (!/(trend|trending|topic|topics|hashtag|hashtags)/.test(intentLower)) return [];
|
|
680
|
-
const $ = cheerio.load(html);
|
|
681
|
-
const roots = $("main, [role='main'], section");
|
|
682
|
-
const scope = roots.length > 0 ? roots.first() : $("body");
|
|
683
|
-
const seen = new Set<string>();
|
|
684
|
-
const topics: Record<string, string>[] = [];
|
|
685
|
-
|
|
686
|
-
scope.find("a[href]").each((_, el) => {
|
|
687
|
-
const $a = $(el);
|
|
688
|
-
const href = ($a.attr("href") ?? "").trim();
|
|
689
|
-
const name = cleanText($a.text());
|
|
690
|
-
if (!href || !name || name.length > 80 || name.length < 2) return;
|
|
691
|
-
if (/^(home|explore|notifications|messages|lists|profile|more|show more|settings|terms|privacy)$/i.test(name)) return;
|
|
692
|
-
const nearby = cleanText($a.closest("div, li, article, section").text());
|
|
693
|
-
const trendish = name.startsWith("#")
|
|
694
|
-
|| /hashtag|trend|trending|topic/i.test(nearby)
|
|
695
|
-
|| /search\?q=|explore|hashtag/i.test(href);
|
|
696
|
-
if (!trendish) return;
|
|
697
|
-
const key = `${name}|${href.split("?")[0]}`;
|
|
698
|
-
if (seen.has(key)) return;
|
|
699
|
-
topics.push({ name, url: href });
|
|
700
|
-
seen.add(key);
|
|
701
|
-
});
|
|
702
|
-
|
|
703
|
-
return topics.length >= 2 ? [{ type: "repeated-elements", data: topics.slice(0, 20), element_count: topics.length }] : [];
|
|
704
|
-
}
|
|
705
|
-
|
|
706
|
-
/**
|
|
707
|
-
* Heuristic extraction of structured data from HTML.
|
|
708
|
-
* Returns an array of discovered data structures.
|
|
709
|
-
*/
|
|
710
|
-
export function parseStructured(html: string): ExtractedStructure[] {
|
|
711
|
-
const $ = cheerio.load(html);
|
|
712
|
-
const results: ExtractedStructure[] = [];
|
|
713
|
-
|
|
714
|
-
// --- JSON-LD ---
|
|
715
|
-
$('script[type="application/ld+json"]').each((_, el) => {
|
|
716
|
-
try {
|
|
717
|
-
const raw = $(el).html();
|
|
718
|
-
if (!raw) return;
|
|
719
|
-
const parsed = JSON.parse(raw);
|
|
720
|
-
results.push({ type: "json-ld", data: parsed, element_count: 1 });
|
|
721
|
-
} catch { /* malformed JSON-LD */ }
|
|
722
|
-
});
|
|
723
|
-
|
|
724
|
-
// --- Meta tags (Open Graph + schema.org) ---
|
|
725
|
-
const meta: Record<string, string> = {};
|
|
726
|
-
$("meta[property], meta[name]").each((_, el) => {
|
|
727
|
-
const $el = $(el);
|
|
728
|
-
const key = $el.attr("property") ?? $el.attr("name") ?? "";
|
|
729
|
-
const content = $el.attr("content") ?? "";
|
|
730
|
-
if ((key.startsWith("og:") || key.startsWith("article:") ||
|
|
731
|
-
key.startsWith("twitter:") || key.startsWith("schema:")) && content) {
|
|
732
|
-
meta[key] = content;
|
|
733
|
-
}
|
|
734
|
-
});
|
|
735
|
-
if (Object.keys(meta).length > 0) {
|
|
736
|
-
results.push({ type: "meta", data: meta, element_count: Object.keys(meta).length });
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
// --- Itemlist tables (HN-style: tr.athing with story rows) ---
|
|
740
|
-
$("table").each((_, table) => {
|
|
741
|
-
const $table = $(table);
|
|
742
|
-
const athings = $table.find("tr.athing");
|
|
743
|
-
if (athings.length >= 3) {
|
|
744
|
-
const items: Record<string, string>[] = [];
|
|
745
|
-
athings.each((_, tr) => {
|
|
746
|
-
const $tr = $(tr);
|
|
747
|
-
const item: Record<string, string> = {};
|
|
748
|
-
const titleLink = $tr.find("span.titleline > a, td.title > span > a, td.title a.storylink").first();
|
|
749
|
-
if (titleLink.length) {
|
|
750
|
-
item.title = titleLink.text().trim();
|
|
751
|
-
item.link = titleLink.attr("href") || "";
|
|
752
|
-
}
|
|
753
|
-
const rank = $tr.find("span.rank").text().trim().replace(".", "");
|
|
754
|
-
if (rank) item.rank = rank;
|
|
755
|
-
const $sub = $tr.next("tr");
|
|
756
|
-
const score = $sub.find("span.score").text().trim();
|
|
757
|
-
if (score) item.score = score;
|
|
758
|
-
const age = $sub.find("span.age").text().trim();
|
|
759
|
-
if (age) item.age = age;
|
|
760
|
-
const author = $sub.find("a.hnuser").text().trim();
|
|
761
|
-
if (author) item.author = author;
|
|
762
|
-
const commentsLink = $sub.find("a").last().text().trim();
|
|
763
|
-
if (commentsLink && commentsLink.includes("comment")) item.comments = commentsLink;
|
|
764
|
-
if (item.title) items.push(item);
|
|
765
|
-
});
|
|
766
|
-
if (items.length >= 3) {
|
|
767
|
-
results.push({ type: "itemlist", data: items, element_count: items.length });
|
|
768
|
-
$table.remove();
|
|
769
|
-
return;
|
|
770
|
-
}
|
|
771
|
-
}
|
|
772
|
-
});
|
|
773
|
-
|
|
774
|
-
// --- Tables ---
|
|
775
|
-
$("table").each((_, table) => {
|
|
776
|
-
const rows = parseTable($, $(table));
|
|
777
|
-
if (rows.length > 0) {
|
|
778
|
-
results.push({ type: "table", data: rows, element_count: rows.length });
|
|
779
|
-
}
|
|
780
|
-
});
|
|
781
|
-
|
|
782
|
-
// --- Definition lists (key-value pairs) ---
|
|
783
|
-
$("dl").each((_, dl) => {
|
|
784
|
-
const pairs = parseDL($, $(dl));
|
|
785
|
-
if (Object.keys(pairs).length > 0) {
|
|
786
|
-
results.push({ type: "key-value", data: pairs, element_count: Object.keys(pairs).length });
|
|
787
|
-
}
|
|
788
|
-
});
|
|
789
|
-
|
|
790
|
-
// --- Ordered/unordered lists ---
|
|
791
|
-
$("ul, ol").each((_, list) => {
|
|
792
|
-
const $list = $(list);
|
|
793
|
-
// Only capture lists with structured content (multiple li with text)
|
|
794
|
-
const items: string[] = [];
|
|
795
|
-
$list.children("li").each((_, li) => {
|
|
796
|
-
const text = $(li).text().trim();
|
|
797
|
-
if (text) items.push(text);
|
|
798
|
-
});
|
|
799
|
-
if (items.length >= 2) {
|
|
800
|
-
results.push({ type: "list", data: items, element_count: items.length });
|
|
801
|
-
}
|
|
802
|
-
});
|
|
803
|
-
|
|
804
|
-
// --- Repeating card/element patterns ---
|
|
805
|
-
const cardResults = detectRepeatingPatterns($);
|
|
806
|
-
results.push(...cardResults);
|
|
807
|
-
|
|
808
|
-
// --- Single-record detail pages ---
|
|
809
|
-
const detailResults = detectDetailPatterns($);
|
|
810
|
-
results.push(...detailResults);
|
|
811
|
-
|
|
812
|
-
return results;
|
|
813
|
-
}
|
|
814
|
-
|
|
815
|
-
function parseTable($: cheerio.CheerioAPI, $table: cheerio.Cheerio<CheerioEl>): Record<string, string>[] {
|
|
816
|
-
const headers: string[] = [];
|
|
817
|
-
$table.find("thead th, thead td, tr:first-child th").each((_, th) => {
|
|
818
|
-
headers.push($(th).text().trim());
|
|
819
|
-
});
|
|
820
|
-
|
|
821
|
-
// If no headers found in thead, try first row
|
|
822
|
-
if (headers.length === 0) {
|
|
823
|
-
const firstRow = $table.find("tr").first();
|
|
824
|
-
firstRow.find("td, th").each((_, cell) => {
|
|
825
|
-
headers.push($(cell).text().trim());
|
|
826
|
-
});
|
|
827
|
-
}
|
|
828
|
-
|
|
829
|
-
if (headers.length === 0) return [];
|
|
830
|
-
|
|
831
|
-
const hasThead = $table.find("thead").length > 0;
|
|
832
|
-
// When thead exists, only iterate tbody rows; otherwise skip the first row (used as headers)
|
|
833
|
-
const dataRows = hasThead
|
|
834
|
-
? $table.find("tbody tr").toArray()
|
|
835
|
-
: $table.find("tr").toArray().slice(1);
|
|
836
|
-
|
|
837
|
-
const rows: Record<string, string>[] = [];
|
|
838
|
-
for (let i = 0; i < dataRows.length; i++) {
|
|
839
|
-
const row: Record<string, string> = {};
|
|
840
|
-
let hasData = false;
|
|
841
|
-
$(dataRows[i]).find("td, th").each((j, cell) => {
|
|
842
|
-
if (j < headers.length && headers[j]) {
|
|
843
|
-
const val = $(cell).text().trim();
|
|
844
|
-
if (val) {
|
|
845
|
-
row[headers[j]] = val;
|
|
846
|
-
hasData = true;
|
|
847
|
-
}
|
|
848
|
-
}
|
|
849
|
-
});
|
|
850
|
-
if (hasData) rows.push(row);
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
return rows;
|
|
854
|
-
}
|
|
855
|
-
|
|
856
|
-
function parseDL($: cheerio.CheerioAPI, $dl: cheerio.Cheerio<CheerioEl>): Record<string, string> {
|
|
857
|
-
const result: Record<string, string> = {};
|
|
858
|
-
let currentKey = "";
|
|
859
|
-
$dl.children("dt, dd").each((_, el) => {
|
|
860
|
-
const tag = (el as CheerioEl).tagName?.toLowerCase();
|
|
861
|
-
if (tag === "dt") {
|
|
862
|
-
currentKey = $(el).text().trim();
|
|
863
|
-
} else if (tag === "dd" && currentKey) {
|
|
864
|
-
result[currentKey] = $(el).text().trim();
|
|
865
|
-
currentKey = "";
|
|
866
|
-
}
|
|
867
|
-
});
|
|
868
|
-
return result;
|
|
869
|
-
}
|
|
870
|
-
|
|
871
|
-
function detectRepeatingPatterns($: cheerio.CheerioAPI): ExtractedStructure[] {
|
|
872
|
-
const results: ExtractedStructure[] = [];
|
|
873
|
-
const seen = new Set<string>();
|
|
874
|
-
|
|
875
|
-
for (const selector of CARD_SELECTORS) {
|
|
876
|
-
const elements = $(selector);
|
|
877
|
-
if (elements.length < 2) continue;
|
|
878
|
-
|
|
879
|
-
// Deduplicate by parent to avoid capturing the same set via multiple selectors
|
|
880
|
-
const parent = elements.first().parent();
|
|
881
|
-
const parentId = getElementSignature($, parent);
|
|
882
|
-
if (seen.has(parentId)) continue;
|
|
883
|
-
seen.add(parentId);
|
|
884
|
-
|
|
885
|
-
const items: Record<string, string>[] = [];
|
|
886
|
-
elements.each((_, el) => {
|
|
887
|
-
const item = extractCardFields($, $(el));
|
|
888
|
-
// Require at least 2 fields to be a meaningful card
|
|
889
|
-
if (Object.keys(item).length >= 2) items.push(item);
|
|
890
|
-
});
|
|
891
|
-
|
|
892
|
-
if (items.length >= 2) {
|
|
893
|
-
results.push({
|
|
894
|
-
type: "repeated-elements",
|
|
895
|
-
data: items,
|
|
896
|
-
element_count: items.length,
|
|
897
|
-
selector: buildReplaySelector($(elements[0])) ?? selector,
|
|
898
|
-
});
|
|
899
|
-
}
|
|
900
|
-
}
|
|
901
|
-
|
|
902
|
-
// Sibling-based detection: group child elements by identical class strings.
|
|
903
|
-
// Handles Tailwind/utility-class sites where class names are non-semantic
|
|
904
|
-
// (e.g. "h-full cursor-pointer overflow-hidden rounded-lg flex flex-col").
|
|
905
|
-
if (results.length === 0) {
|
|
906
|
-
const siblingGroups = detectSiblingPatterns($);
|
|
907
|
-
results.push(...siblingGroups);
|
|
908
|
-
}
|
|
909
|
-
|
|
910
|
-
return results;
|
|
911
|
-
}
|
|
912
|
-
|
|
913
|
-
function hasDetailFieldShape(fields: Record<string, string>): boolean {
|
|
914
|
-
if (!fields.title && !fields.name && !fields.term) return false;
|
|
915
|
-
return !!(
|
|
916
|
-
fields.description ||
|
|
917
|
-
fields.definition ||
|
|
918
|
-
fields.price ||
|
|
919
|
-
fields.rating ||
|
|
920
|
-
fields.author ||
|
|
921
|
-
fields.url ||
|
|
922
|
-
fields.link ||
|
|
923
|
-
fields.score ||
|
|
924
|
-
fields.image
|
|
925
|
-
);
|
|
926
|
-
}
|
|
927
|
-
|
|
928
|
-
function detectDetailPatterns($: cheerio.CheerioAPI): ExtractedStructure[] {
|
|
929
|
-
const results: ExtractedStructure[] = [];
|
|
930
|
-
const seen = new Set<string>();
|
|
931
|
-
|
|
932
|
-
for (const selector of [
|
|
933
|
-
"main",
|
|
934
|
-
"article",
|
|
935
|
-
"[role='main']",
|
|
936
|
-
"[class*='detail']",
|
|
937
|
-
"[class*='details']",
|
|
938
|
-
"[class*='product']",
|
|
939
|
-
"[class*='listing']",
|
|
940
|
-
"[class*='profile']",
|
|
941
|
-
"[class*='content']",
|
|
942
|
-
]) {
|
|
943
|
-
$(selector).each((_, el) => {
|
|
944
|
-
const $el = $(el);
|
|
945
|
-
const signature = `${selector}|${getElementSignature($, $el)}`;
|
|
946
|
-
if (seen.has(signature)) return;
|
|
947
|
-
seen.add(signature);
|
|
948
|
-
if ($el.text().trim().length < 20) return;
|
|
949
|
-
const fields = extractCardFields($, $el);
|
|
950
|
-
if (Object.keys(fields).length < 2) return;
|
|
951
|
-
if (!hasDetailFieldShape(fields)) return;
|
|
952
|
-
results.push({
|
|
953
|
-
type: "key-value",
|
|
954
|
-
data: fields,
|
|
955
|
-
element_count: 1,
|
|
956
|
-
selector: buildReplaySelector($el) ?? selector,
|
|
957
|
-
});
|
|
958
|
-
});
|
|
959
|
-
}
|
|
960
|
-
|
|
961
|
-
return results;
|
|
962
|
-
}
|
|
963
|
-
|
|
964
|
-
/**
|
|
965
|
-
* Detect repeating sibling elements that share the same full class string.
|
|
966
|
-
* Works for Tailwind/utility-class sites where standard selectors fail.
|
|
967
|
-
*/
|
|
968
|
-
function detectSiblingPatterns($: cheerio.CheerioAPI): ExtractedStructure[] {
|
|
969
|
-
const results: ExtractedStructure[] = [];
|
|
970
|
-
const seenParents = new Set<string>();
|
|
971
|
-
|
|
972
|
-
// Scan all elements that could be container parents
|
|
973
|
-
$("div, section, ul, ol, main").each((_, parent) => {
|
|
974
|
-
const $parent = $(parent);
|
|
975
|
-
const children = $parent.children();
|
|
976
|
-
if (children.length < 3) return;
|
|
977
|
-
|
|
978
|
-
// Group children by their full class string
|
|
979
|
-
const groups = new Map<string, CheerioEl[]>();
|
|
980
|
-
children.each((_, child) => {
|
|
981
|
-
const cls = $(child).attr("class") || "";
|
|
982
|
-
if (cls.length < 3) return; // skip classless or trivially-classed elements
|
|
983
|
-
const key = `${(child as any).tagName}|${cls}`;
|
|
984
|
-
const arr = groups.get(key) || [];
|
|
985
|
-
arr.push(child);
|
|
986
|
-
groups.set(key, arr);
|
|
987
|
-
});
|
|
988
|
-
|
|
989
|
-
for (const [key, elements] of groups) {
|
|
990
|
-
if (elements.length < 3) continue;
|
|
991
|
-
|
|
992
|
-
// Avoid processing the same parent+class group twice
|
|
993
|
-
const parentSig = getElementSignature($, $parent) + "|" + key;
|
|
994
|
-
if (seenParents.has(parentSig)) continue;
|
|
995
|
-
seenParents.add(parentSig);
|
|
996
|
-
|
|
997
|
-
const items: Record<string, string>[] = [];
|
|
998
|
-
for (const el of elements) {
|
|
999
|
-
const item = extractCardFields($, $(el));
|
|
1000
|
-
if (Object.keys(item).length >= 2) items.push(item);
|
|
1001
|
-
}
|
|
1002
|
-
|
|
1003
|
-
if (items.length >= 3) {
|
|
1004
|
-
results.push({
|
|
1005
|
-
type: "repeated-elements",
|
|
1006
|
-
data: items,
|
|
1007
|
-
element_count: items.length,
|
|
1008
|
-
selector: buildReplaySelector($(elements[0])),
|
|
1009
|
-
});
|
|
1010
|
-
}
|
|
1011
|
-
}
|
|
1012
|
-
});
|
|
1013
|
-
|
|
1014
|
-
return results;
|
|
1015
|
-
}
|
|
1016
|
-
|
|
1017
|
-
function getElementSignature($: cheerio.CheerioAPI, $el: cheerio.Cheerio<CheerioEl>): string {
|
|
1018
|
-
const tag = $el.prop("tagName") ?? "?";
|
|
1019
|
-
const cls = $el.attr("class") ?? "";
|
|
1020
|
-
const id = $el.attr("id") ?? "";
|
|
1021
|
-
return `${tag}#${id}.${cls}`;
|
|
1022
|
-
}
|
|
1023
|
-
|
|
1024
|
-
function extractCardFields($: cheerio.CheerioAPI, $el: cheerio.Cheerio<CheerioEl>): Record<string, string> {
|
|
1025
|
-
const fields: Record<string, string> = {};
|
|
1026
|
-
|
|
1027
|
-
// Extract text from headings (semantic tags + Bootstrap heading classes)
|
|
1028
|
-
$el.find("h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6, [class*='title'], [class*='header-text'], [class*='hearder']").each((i, h) => {
|
|
1029
|
-
const text = $(h).text().trim();
|
|
1030
|
-
if (text && text.length < 300) fields[i === 0 ? "title" : `heading_${i}`] = text;
|
|
1031
|
-
});
|
|
1032
|
-
if (!fields["message"] && fields["heading_1"] && fields["heading_1"].length > 10) {
|
|
1033
|
-
fields["message"] = fields["heading_1"];
|
|
1034
|
-
}
|
|
1035
|
-
|
|
1036
|
-
// Fallback title: strong/bold text or [class*='name']
|
|
1037
|
-
if (!fields["title"]) {
|
|
1038
|
-
const strong = $el.find("strong, b, [class*='name']").first();
|
|
1039
|
-
if (strong.length) {
|
|
1040
|
-
const text = strong.text().trim();
|
|
1041
|
-
if (text && text.length < 200) fields["title"] = text;
|
|
1042
|
-
}
|
|
1043
|
-
}
|
|
1044
|
-
|
|
1045
|
-
// Extract links
|
|
1046
|
-
const links: string[] = [];
|
|
1047
|
-
$el.find("a[href]").each((_, a) => {
|
|
1048
|
-
const href = $(a).attr("href");
|
|
1049
|
-
if (href && !href.startsWith("#") && !href.startsWith("javascript:")) {
|
|
1050
|
-
links.push(href);
|
|
1051
|
-
}
|
|
1052
|
-
});
|
|
1053
|
-
if (links.length > 0) {
|
|
1054
|
-
fields["link"] = links[0];
|
|
1055
|
-
fields["url"] = links[0];
|
|
1056
|
-
}
|
|
1057
|
-
|
|
1058
|
-
// Fallback title from link text
|
|
1059
|
-
if (!fields["title"] && links.length > 0) {
|
|
1060
|
-
const linkText = $el.find("a").first().text().trim();
|
|
1061
|
-
if (linkText && linkText.length > 2 && linkText.length < 200 && !/^(read|more|view|see|click)/i.test(linkText)) {
|
|
1062
|
-
fields["title"] = linkText;
|
|
1063
|
-
}
|
|
1064
|
-
}
|
|
1065
|
-
|
|
1066
|
-
// Extract images
|
|
1067
|
-
const img = $el.find("img[src]").first();
|
|
1068
|
-
const imgSrc = img.attr("src");
|
|
1069
|
-
if (imgSrc) fields["image"] = imgSrc;
|
|
1070
|
-
|
|
1071
|
-
// Extract description/paragraph text (skip price paragraphs)
|
|
1072
|
-
$el.find("p").each((_, p) => {
|
|
1073
|
-
if (fields["description"]) return;
|
|
1074
|
-
const $p = $(p);
|
|
1075
|
-
const cls = $p.attr("class") ?? "";
|
|
1076
|
-
if (/price|cost|amount|stock|availability/i.test(cls)) return;
|
|
1077
|
-
const text = $p.text().trim();
|
|
1078
|
-
if (text && text.length > 10) fields["description"] = text;
|
|
1079
|
-
});
|
|
1080
|
-
|
|
1081
|
-
// Generic summary/excerpt containers often hold the useful body text for docs/questions/cards.
|
|
1082
|
-
if (!fields["description"]) {
|
|
1083
|
-
$el.find("[class*='summary'], [class*='excerpt'], [class*='description'], [class*='desc'], [class*='snippet']").each((_, node) => {
|
|
1084
|
-
if (fields["description"]) return;
|
|
1085
|
-
const text = $(node).text().trim();
|
|
1086
|
-
if (text && text.length > 10 && text.length < 500) fields["description"] = text;
|
|
1087
|
-
});
|
|
1088
|
-
}
|
|
1089
|
-
|
|
1090
|
-
// Alerts / success pages often carry the user-facing payload in strong text or flash-like containers.
|
|
1091
|
-
if (!fields["message"]) {
|
|
1092
|
-
$el.find("[role='alert'], .flash, .alert, [class*='message'], [class*='flash'], [class*='alert'], p strong, p b").each((_, node) => {
|
|
1093
|
-
if (fields["message"]) return;
|
|
1094
|
-
const text = $(node).text().trim();
|
|
1095
|
-
if (text && text.length > 5 && text.length < 500 && text !== fields["title"]) {
|
|
1096
|
-
fields["message"] = text;
|
|
1097
|
-
}
|
|
1098
|
-
});
|
|
1099
|
-
}
|
|
1100
|
-
if (!fields["message"] && fields["description"] && /congratulations|successfully|logged in|logged out|welcome|error|invalid|warning|flash|alert/i.test(fields["description"])) {
|
|
1101
|
-
fields["message"] = fields["description"];
|
|
1102
|
-
}
|
|
1103
|
-
|
|
1104
|
-
// Extract price-like patterns — use the most specific (deepest) match
|
|
1105
|
-
const priceEl = $el.find(".price_color, [class*='price']:not(:has([class*='price'])), .price, .cost, .amount").first();
|
|
1106
|
-
if (priceEl.length > 0) {
|
|
1107
|
-
// Get only direct text content, not nested children
|
|
1108
|
-
const priceText = priceEl.contents().filter((_, node) => node.type === "text" || (node as any).tagName === "span")
|
|
1109
|
-
.text().trim();
|
|
1110
|
-
if (priceText) fields["price"] = priceText;
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
const scoreEl = $el.find("[class*='vote'], [class*='score'], [data-score]").first();
|
|
1114
|
-
if (scoreEl.length > 0) {
|
|
1115
|
-
const scoreText = scoreEl.text().trim() || scoreEl.attr("data-score")?.trim();
|
|
1116
|
-
if (scoreText && scoreText.length < 80) fields["score"] = scoreText;
|
|
1117
|
-
}
|
|
1118
|
-
|
|
1119
|
-
const answersEl = $el.find("[class*='answer'], [data-answercount]").first();
|
|
1120
|
-
if (answersEl.length > 0) {
|
|
1121
|
-
const answersText = answersEl.text().trim() || answersEl.attr("data-answercount")?.trim();
|
|
1122
|
-
if (answersText && answersText.length < 80) fields["answer_count"] = answersText;
|
|
1123
|
-
}
|
|
1124
|
-
|
|
1125
|
-
const ratingEl = $el.find("[class*='rating'], [aria-label*='rating'], [aria-label*='Rating'], [aria-valuenow], [aria-valuetext], [data-rating]").first();
|
|
1126
|
-
if (ratingEl.length > 0) {
|
|
1127
|
-
const ratingProbe = ratingEl.find("[aria-valuenow], [aria-valuetext], [aria-label*='rating'], [aria-label*='Rating']").first();
|
|
1128
|
-
const ratingText = ratingProbe.attr("aria-valuenow")?.trim()
|
|
1129
|
-
|| ratingProbe.attr("aria-valuetext")?.trim()
|
|
1130
|
-
|| ratingProbe.attr("aria-label")?.trim()
|
|
1131
|
-
|| ratingEl.attr("aria-valuenow")?.trim()
|
|
1132
|
-
|| ratingEl.attr("aria-valuetext")?.trim()
|
|
1133
|
-
|| ratingEl.attr("aria-label")?.trim()
|
|
1134
|
-
|| ratingEl.attr("data-rating")?.trim()
|
|
1135
|
-
|| ratingProbe.text().trim()
|
|
1136
|
-
|| ratingEl.text().trim();
|
|
1137
|
-
const numeric = ratingText?.match(/\b([0-5](?:\.\d)?)\b/)?.[1];
|
|
1138
|
-
if (numeric) fields["rating"] = numeric;
|
|
1139
|
-
else if (ratingText && ratingText.length < 80 && !/^rating$/i.test(ratingText)) fields["rating"] = ratingText;
|
|
1140
|
-
}
|
|
1141
|
-
|
|
1142
|
-
const authorEl = $el.find("[class*='author'], [class*='byline'], [class*='user'], [rel='author']").first();
|
|
1143
|
-
if (authorEl.length > 0) {
|
|
1144
|
-
const authorText = authorEl.text().trim();
|
|
1145
|
-
if (authorText && authorText.length < 120) fields["author"] = authorText;
|
|
1146
|
-
}
|
|
1147
|
-
|
|
1148
|
-
const partnerEl = $el.find("[class*='partnerName'], [class*='partnerNames'], [class*='partner']").first();
|
|
1149
|
-
if (partnerEl.length > 0) {
|
|
1150
|
-
const partnerText = partnerEl.text().trim();
|
|
1151
|
-
if (partnerText && partnerText.length < 160) fields["partner"] = partnerText;
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
const definitionEl = $el.find("dd, [class*='def'], [class*='meaning'], [class*='definition']").first();
|
|
1155
|
-
if (definitionEl.length > 0) {
|
|
1156
|
-
const definitionText = definitionEl.text().trim();
|
|
1157
|
-
if (definitionText && definitionText.length > 10 && definitionText.length < 600) fields["definition"] = definitionText;
|
|
1158
|
-
}
|
|
1159
|
-
|
|
1160
|
-
// Extract metadata spans (dates, citations, info text)
|
|
1161
|
-
$el.find("[class*='date'], [class*='info'], [class*='meta'], [class*='citation'], [class*='addinfo'], time").each((_, s) => {
|
|
1162
|
-
const text = $(s).text().trim();
|
|
1163
|
-
if (text && text.length > 3 && text.length < 200) {
|
|
1164
|
-
// Derive a key from the class name
|
|
1165
|
-
const cls = ($(s).attr("class") ?? "").toLowerCase();
|
|
1166
|
-
const key = cls.match(/(date|citation|info|meta|time|author|category)/)?.[1] ?? "info";
|
|
1167
|
-
if (!fields[key]) fields[key] = text;
|
|
1168
|
-
}
|
|
1169
|
-
});
|
|
1170
|
-
|
|
1171
|
-
// Fallback: capture the element's direct text if nothing else matched
|
|
1172
|
-
if (Object.keys(fields).length === 0) {
|
|
1173
|
-
const text = $el.text().trim();
|
|
1174
|
-
if (text && text.length < 500) fields["text"] = text;
|
|
1175
|
-
}
|
|
1176
|
-
|
|
1177
|
-
return fields;
|
|
1178
|
-
}
|
|
1179
|
-
|
|
1180
|
-
function scoreSemanticFit(structure: ExtractedStructure, intent: string): number {
|
|
1181
|
-
const assessment = assessIntentResult(structure.data, intent);
|
|
1182
|
-
if (assessment.verdict === "pass") return 140;
|
|
1183
|
-
if (assessment.verdict === "fail") return -140;
|
|
1184
|
-
return 0;
|
|
1185
|
-
}
|
|
1186
|
-
|
|
1187
|
-
function scoreSparseLinkList(structure: ExtractedStructure): number {
|
|
1188
|
-
if (structure.type !== "repeated-elements" || !Array.isArray(structure.data)) return 0;
|
|
1189
|
-
const items = structure.data as Array<Record<string, unknown>>;
|
|
1190
|
-
if (items.length < 4) return 0;
|
|
1191
|
-
const sparse = items.filter((item) => {
|
|
1192
|
-
const keys = Object.keys(item);
|
|
1193
|
-
if (keys.length > 2) return false;
|
|
1194
|
-
const title = typeof item.title === "string" ? item.title : typeof item.name === "string" ? item.name : "";
|
|
1195
|
-
const link = typeof item.link === "string" ? item.link : typeof item.url === "string" ? item.url : "";
|
|
1196
|
-
return !!title && !!link && title.length <= 32;
|
|
1197
|
-
}).length;
|
|
1198
|
-
return sparse / items.length >= 0.7 ? -80 : 0;
|
|
1199
|
-
}
|
|
1200
|
-
|
|
1201
|
-
function scoreFieldRichness(structure: ExtractedStructure): number {
|
|
1202
|
-
if (structure.type !== "repeated-elements" || !Array.isArray(structure.data)) return 0;
|
|
1203
|
-
const items = structure.data as Array<Record<string, unknown>>;
|
|
1204
|
-
if (items.length === 0) return 0;
|
|
1205
|
-
const avgFields = items.reduce((sum, item) => sum + Object.keys(item).length, 0) / items.length;
|
|
1206
|
-
if (avgFields >= 4) return 14;
|
|
1207
|
-
if (avgFields >= 3) return 8;
|
|
1208
|
-
return 0;
|
|
1209
|
-
}
|
|
1210
|
-
|
|
1211
|
-
// ---------------------------------------------------------------------------
|
|
1212
|
-
// extractFromDOM
|
|
1213
|
-
// ---------------------------------------------------------------------------
|
|
1214
|
-
|
|
1215
|
-
export interface ExtractionResult {
|
|
1216
|
-
data: unknown;
|
|
1217
|
-
extraction_method: string;
|
|
1218
|
-
confidence: number;
|
|
1219
|
-
selector?: string;
|
|
1220
|
-
}
|
|
1221
|
-
|
|
1222
|
-
function buildReplaySelector($el: cheerio.Cheerio<CheerioEl>): string | undefined {
|
|
1223
|
-
const tag = $el.get(0)?.tagName;
|
|
1224
|
-
if (!tag) return undefined;
|
|
1225
|
-
const id = ($el.attr("id") ?? "").trim();
|
|
1226
|
-
if (id) return `${tag}#${id.replace(/[^a-zA-Z0-9_-]/g, "")}`;
|
|
1227
|
-
const classes = (($el.attr("class") ?? "").trim())
|
|
1228
|
-
.split(/\s+/)
|
|
1229
|
-
.filter(Boolean)
|
|
1230
|
-
.slice(0, 4)
|
|
1231
|
-
.map((part) => part.replace(/[^a-zA-Z0-9_-]/g, ""))
|
|
1232
|
-
.filter(Boolean);
|
|
1233
|
-
return classes.length > 0 ? `${tag}.${classes.join(".")}` : tag;
|
|
1234
|
-
}
|
|
1235
|
-
|
|
1236
|
-
function extractUsingSelector(html: string, selector: string): ExtractedStructure | null {
|
|
1237
|
-
const $ = cheerio.load(cleanDOM(html));
|
|
1238
|
-
const elements = $(selector);
|
|
1239
|
-
if (elements.length < 1) return null;
|
|
1240
|
-
const items: Record<string, string>[] = [];
|
|
1241
|
-
elements.each((_, el) => {
|
|
1242
|
-
const fields = extractCardFields($, $(el));
|
|
1243
|
-
if (Object.keys(fields).length >= 2) items.push(fields);
|
|
1244
|
-
});
|
|
1245
|
-
if (items.length >= 2) {
|
|
1246
|
-
return { type: "repeated-elements", data: items, element_count: items.length, selector };
|
|
1247
|
-
}
|
|
1248
|
-
if (items.length === 1) {
|
|
1249
|
-
return { type: "key-value", data: items[0], element_count: 1, selector };
|
|
1250
|
-
}
|
|
1251
|
-
return null;
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
|
-
export function extractFromDOMWithHint(
|
|
1255
|
-
html: string,
|
|
1256
|
-
intent: string,
|
|
1257
|
-
hint?: { selector?: string },
|
|
1258
|
-
): ExtractionResult {
|
|
1259
|
-
if (hint?.selector) {
|
|
1260
|
-
const extracted = extractUsingSelector(html, hint.selector);
|
|
1261
|
-
if (extracted) {
|
|
1262
|
-
const assessment = assessIntentResult(extracted.data, intent);
|
|
1263
|
-
if (assessment.verdict === "pass") {
|
|
1264
|
-
return {
|
|
1265
|
-
data: extracted.data,
|
|
1266
|
-
extraction_method: extracted.type,
|
|
1267
|
-
confidence: 0.95,
|
|
1268
|
-
selector: hint.selector,
|
|
1269
|
-
};
|
|
1270
|
-
}
|
|
1271
|
-
}
|
|
1272
|
-
}
|
|
1273
|
-
return extractFromDOM(html, intent);
|
|
1274
|
-
}
|
|
1275
|
-
|
|
1276
|
-
/**
|
|
1277
|
-
* Main entry point: clean HTML, extract structured data, and return
|
|
1278
|
-
* the best match for the given intent.
|
|
1279
|
-
*/
|
|
1280
|
-
export function extractFromDOM(html: string, intent: string): ExtractionResult {
|
|
1281
|
-
// Cap HTML size to prevent cheerio from hanging on massive pages
|
|
1282
|
-
const MAX_HTML_SIZE = 300_000;
|
|
1283
|
-
let workingHtml = html;
|
|
1284
|
-
if (workingHtml.length > MAX_HTML_SIZE) {
|
|
1285
|
-
// Strip attribute bloat first (class/style/data-* attributes inflate HTML 2-3x)
|
|
1286
|
-
workingHtml = workingHtml
|
|
1287
|
-
.replace(/\s+class="[^"]*"/g, "")
|
|
1288
|
-
.replace(/\s+style="[^"]*"/g, "")
|
|
1289
|
-
.replace(/\s+data-[a-z][-a-z]*="[^"]*"/g, "");
|
|
1290
|
-
// If still too large, truncate keeping body content
|
|
1291
|
-
if (workingHtml.length > MAX_HTML_SIZE) {
|
|
1292
|
-
const bodyStart = workingHtml.indexOf("<body");
|
|
1293
|
-
if (bodyStart > 0) {
|
|
1294
|
-
workingHtml = workingHtml.substring(0, Math.max(MAX_HTML_SIZE, bodyStart + MAX_HTML_SIZE));
|
|
1295
|
-
} else {
|
|
1296
|
-
workingHtml = workingHtml.substring(0, MAX_HTML_SIZE);
|
|
1297
|
-
}
|
|
1298
|
-
}
|
|
1299
|
-
}
|
|
1300
|
-
|
|
1301
|
-
// Extract SPA-embedded data from raw HTML BEFORE cleanDOM strips scripts
|
|
1302
|
-
const spaStructures = extractSPAData(workingHtml);
|
|
1303
|
-
const flashStructures = extractFlashNoticeSpecial(workingHtml, intent);
|
|
1304
|
-
const cleaned = cleanDOM(workingHtml);
|
|
1305
|
-
const githubStructures = extractGitHubSpecial(workingHtml, intent);
|
|
1306
|
-
const linkedInStructures = extractLinkedInSpecial(workingHtml, intent);
|
|
1307
|
-
const packageSearchStructures = extractPackageSearchSpecial(workingHtml, intent);
|
|
1308
|
-
const xProfileStructures = extractXProfileSpecial(workingHtml, intent);
|
|
1309
|
-
const postStructures = extractPostSpecial(workingHtml, intent);
|
|
1310
|
-
const trendStructures = extractTrendSpecial(workingHtml, intent);
|
|
1311
|
-
const definitionStructures = extractDefinitionSpecial(workingHtml, intent);
|
|
1312
|
-
const courseStructures = extractCourseSearchSpecial(workingHtml, intent);
|
|
1313
|
-
const structures = [...flashStructures, ...githubStructures, ...linkedInStructures, ...packageSearchStructures, ...xProfileStructures, ...postStructures, ...trendStructures, ...definitionStructures, ...courseStructures, ...spaStructures, ...parseStructured(cleaned)]
|
|
1314
|
-
.map((structure) => normalizeStructureForIntent(structure, intent));
|
|
1315
|
-
|
|
1316
|
-
if (structures.length === 0) {
|
|
1317
|
-
return { data: null, extraction_method: "none", confidence: 0 };
|
|
1318
|
-
}
|
|
1319
|
-
|
|
1320
|
-
// Score each structure by relevance to intent
|
|
1321
|
-
const intentWords = intent.toLowerCase().split(/\s+/).filter(Boolean);
|
|
1322
|
-
const scored = structures.map((s) => ({
|
|
1323
|
-
structure: s,
|
|
1324
|
-
score: scoreRelevance(s, intentWords) + scoreSemanticFit(s, intent) + scoreSparseLinkList(s) + scoreFieldRichness(s),
|
|
1325
|
-
}));
|
|
1326
|
-
|
|
1327
|
-
scored.sort((a, b) => b.score - a.score);
|
|
1328
|
-
|
|
1329
|
-
const passing = scored.filter((candidate) => assessIntentResult(candidate.structure.data, intent).verdict === "pass");
|
|
1330
|
-
const bestPassing = (() => {
|
|
1331
|
-
if (passing.length === 0) return undefined;
|
|
1332
|
-
const bestPassingOverall = passing[0];
|
|
1333
|
-
const bestPassingSpa = passing.find((candidate) => candidate.structure.type.startsWith("spa-"));
|
|
1334
|
-
// Prefer cleaner SPA payloads when they're effectively tied with DOM-derived candidates.
|
|
1335
|
-
if (bestPassingSpa && bestPassingOverall && bestPassingSpa.score >= bestPassingOverall.score - 2) {
|
|
1336
|
-
return bestPassingSpa;
|
|
1337
|
-
}
|
|
1338
|
-
return bestPassingOverall;
|
|
1339
|
-
})();
|
|
1340
|
-
if (bestPassing) {
|
|
1341
|
-
return {
|
|
1342
|
-
data: bestPassing.structure.data,
|
|
1343
|
-
extraction_method: bestPassing.structure.type,
|
|
1344
|
-
confidence: computeConfidence(bestPassing.structure, bestPassing.score),
|
|
1345
|
-
selector: bestPassing.structure.selector,
|
|
1346
|
-
};
|
|
1347
|
-
}
|
|
1348
|
-
|
|
1349
|
-
const best = scored[0];
|
|
1350
|
-
if (isMessageLikeStructure(best.structure, intent)) {
|
|
1351
|
-
return {
|
|
1352
|
-
data: best.structure.data,
|
|
1353
|
-
extraction_method: best.structure.type,
|
|
1354
|
-
confidence: computeConfidence(best.structure, best.score),
|
|
1355
|
-
selector: best.structure.selector,
|
|
1356
|
-
};
|
|
1357
|
-
}
|
|
1358
|
-
|
|
1359
|
-
if (scored.length === 1) {
|
|
1360
|
-
return {
|
|
1361
|
-
data: best.structure.data,
|
|
1362
|
-
extraction_method: best.structure.type,
|
|
1363
|
-
confidence: computeConfidence(best.structure, best.score),
|
|
1364
|
-
selector: best.structure.selector,
|
|
1365
|
-
};
|
|
1366
|
-
}
|
|
1367
|
-
|
|
1368
|
-
const hasClearWinner = best.score > scored[1].score * 1.5;
|
|
1369
|
-
|
|
1370
|
-
if (hasClearWinner && best.score > 0) {
|
|
1371
|
-
return {
|
|
1372
|
-
data: best.structure.data,
|
|
1373
|
-
extraction_method: best.structure.type,
|
|
1374
|
-
confidence: computeConfidence(best.structure, best.score),
|
|
1375
|
-
selector: best.structure.selector,
|
|
1376
|
-
};
|
|
1377
|
-
}
|
|
1378
|
-
|
|
1379
|
-
// No clear winner — return all structures
|
|
1380
|
-
return {
|
|
1381
|
-
data: scored.map((s) => ({
|
|
1382
|
-
type: s.structure.type,
|
|
1383
|
-
data: s.structure.data,
|
|
1384
|
-
relevance_score: s.score,
|
|
1385
|
-
})),
|
|
1386
|
-
extraction_method: "multiple",
|
|
1387
|
-
confidence: computeConfidence(best.structure, best.score) * 0.7,
|
|
1388
|
-
selector: best.structure.selector,
|
|
1389
|
-
};
|
|
1390
|
-
}
|
|
1391
|
-
|
|
1392
|
-
function scoreRelevance(structure: ExtractedStructure, intentWords: string[]): number {
|
|
1393
|
-
const text = JSON.stringify(structure.data).toLowerCase();
|
|
1394
|
-
let score = 0;
|
|
1395
|
-
const intentSet = new Set(intentWords);
|
|
1396
|
-
|
|
1397
|
-
for (const word of intentWords) {
|
|
1398
|
-
if (word.length < 3) continue; // skip short words like "a", "to", etc.
|
|
1399
|
-
// Count occurrences of intent word in the data
|
|
1400
|
-
const regex = new RegExp(word, "gi");
|
|
1401
|
-
const matches = text.match(regex);
|
|
1402
|
-
if (matches) {
|
|
1403
|
-
score += matches.length;
|
|
1404
|
-
}
|
|
1405
|
-
}
|
|
1406
|
-
|
|
1407
|
-
// Bonus for highly structured data
|
|
1408
|
-
if (structure.type === "spa-nextjs") score += 5;
|
|
1409
|
-
if (structure.type.startsWith("spa-")) score += 3;
|
|
1410
|
-
if (structure.type === "json-ld") score += 3;
|
|
1411
|
-
if (structure.type === "itemlist") score += 3;
|
|
1412
|
-
if (structure.type === "table") score += 2;
|
|
1413
|
-
if (structure.type === "repeated-elements") score += 1;
|
|
1414
|
-
if (structure.type === "key-value") score += 1;
|
|
1415
|
-
|
|
1416
|
-
// GitHub/repo-aware shaping: prefer repo-shaped objects/lists over file tables.
|
|
1417
|
-
if (structure.type === "key-value" && structure.data && typeof structure.data === "object" && !Array.isArray(structure.data)) {
|
|
1418
|
-
const keys = Object.keys(structure.data as Record<string, unknown>);
|
|
1419
|
-
if (keys.includes("full_name")) score += 4;
|
|
1420
|
-
if (keys.includes("description")) score += 2;
|
|
1421
|
-
if (keys.includes("stars")) score += 2;
|
|
1422
|
-
if ((intentSet.has("repository") || intentSet.has("repo")) && keys.includes("full_name")) score += 6;
|
|
1423
|
-
if (intentSet.has("info")) score += 2;
|
|
1424
|
-
}
|
|
1425
|
-
|
|
1426
|
-
if (structure.type === "repeated-elements" && Array.isArray(structure.data)) {
|
|
1427
|
-
const items = structure.data as Array<Record<string, unknown>>;
|
|
1428
|
-
const repoShaped = items.filter((item) => typeof item?.full_name === "string" || typeof item?.url === "string");
|
|
1429
|
-
if (repoShaped.length >= 2) score += 8;
|
|
1430
|
-
if ((intentSet.has("search") || intentSet.has("trending")) && repoShaped.length >= 2) score += 8;
|
|
1431
|
-
const peopleShaped = items.filter((item) => typeof item?.name === "string" && (typeof item?.headline === "string" || typeof item?.public_identifier === "string"));
|
|
1432
|
-
if (peopleShaped.length >= 2) score += 8;
|
|
1433
|
-
if ((intentSet.has("people") || intentSet.has("person") || intentSet.has("profile")) && peopleShaped.length >= 2) score += 10;
|
|
1434
|
-
const postShaped = items.filter((item) =>
|
|
1435
|
-
(typeof item?.id === "string" || typeof item?.url === "string") &&
|
|
1436
|
-
(typeof item?.text === "string" || typeof item?.content === "string" || typeof item?.username === "string")
|
|
1437
|
-
);
|
|
1438
|
-
if (postShaped.length >= 1) score += 8;
|
|
1439
|
-
if ((intentSet.has("post") || intentSet.has("posts") || intentSet.has("status") || intentSet.has("statuses") || intentSet.has("tweet")) && postShaped.length >= 1) score += 10;
|
|
1440
|
-
const topicShaped = items.filter((item) =>
|
|
1441
|
-
(typeof item?.name === "string" || typeof item?.title === "string") &&
|
|
1442
|
-
typeof item?.url === "string"
|
|
1443
|
-
);
|
|
1444
|
-
if (topicShaped.length >= 2) score += 8;
|
|
1445
|
-
if ((intentSet.has("trend") || intentSet.has("trending") || intentSet.has("topic") || intentSet.has("topics") || intentSet.has("hashtag")) && topicShaped.length >= 2) score += 10;
|
|
1446
|
-
}
|
|
1447
|
-
|
|
1448
|
-
if (structure.type === "table" && Array.isArray(structure.data)) {
|
|
1449
|
-
const keys = new Set((structure.data as Array<Record<string, unknown>>).flatMap((row) => Object.keys(row)));
|
|
1450
|
-
if (keys.has("Last commit message") || keys.has("Last commit date")) score -= 8;
|
|
1451
|
-
if (keys.has("Name") && !intentSet.has("file") && !intentSet.has("commit")) score -= 4;
|
|
1452
|
-
}
|
|
1453
|
-
|
|
1454
|
-
// Bonus for more elements (richer data)
|
|
1455
|
-
score += Math.min(structure.element_count * 0.1, 2);
|
|
1456
|
-
|
|
1457
|
-
return score;
|
|
1458
|
-
}
|
|
1459
|
-
|
|
1460
|
-
function computeConfidence(structure: ExtractedStructure, relevanceScore: number): number {
|
|
1461
|
-
let confidence = 0;
|
|
1462
|
-
|
|
1463
|
-
// Base confidence from structure type
|
|
1464
|
-
switch (structure.type) {
|
|
1465
|
-
case "spa-nextjs":
|
|
1466
|
-
confidence = 0.9;
|
|
1467
|
-
break;
|
|
1468
|
-
case "spa-nuxt":
|
|
1469
|
-
case "spa-initial-state":
|
|
1470
|
-
case "spa-preloaded-state":
|
|
1471
|
-
confidence = 0.85;
|
|
1472
|
-
break;
|
|
1473
|
-
case "json-ld":
|
|
1474
|
-
confidence = 0.9;
|
|
1475
|
-
break;
|
|
1476
|
-
case "itemlist":
|
|
1477
|
-
confidence = 0.9;
|
|
1478
|
-
break;
|
|
1479
|
-
case "table":
|
|
1480
|
-
confidence = 0.8;
|
|
1481
|
-
break;
|
|
1482
|
-
case "repeated-elements":
|
|
1483
|
-
confidence = 0.7;
|
|
1484
|
-
break;
|
|
1485
|
-
case "key-value":
|
|
1486
|
-
confidence = 0.7;
|
|
1487
|
-
break;
|
|
1488
|
-
case "meta":
|
|
1489
|
-
confidence = 0.6;
|
|
1490
|
-
break;
|
|
1491
|
-
case "list":
|
|
1492
|
-
confidence = 0.5;
|
|
1493
|
-
break;
|
|
1494
|
-
default:
|
|
1495
|
-
confidence = 0.3;
|
|
1496
|
-
}
|
|
1497
|
-
|
|
1498
|
-
// Boost from element count (more data = more confidence)
|
|
1499
|
-
if (structure.element_count > 5) confidence += 0.05;
|
|
1500
|
-
if (structure.element_count > 10) confidence += 0.05;
|
|
1501
|
-
|
|
1502
|
-
// Boost from relevance score
|
|
1503
|
-
if (relevanceScore > 5) confidence += 0.05;
|
|
1504
|
-
if (relevanceScore > 10) confidence += 0.05;
|
|
1505
|
-
|
|
1506
|
-
return Math.min(confidence, 1);
|
|
1507
|
-
}
|