unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/cli.js +79 -33
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +73 -22
  4. package/dist/server.js +25994 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/manifest.json +1 -1
  7. package/runtime-src/agent-outcome.ts +0 -166
  8. package/runtime-src/analytics-session.ts +0 -55
  9. package/runtime-src/api/browse-index.ts +0 -343
  10. package/runtime-src/api/browse-session.ts +0 -572
  11. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  12. package/runtime-src/api/browse-submit.ts +0 -1184
  13. package/runtime-src/api/routes.ts +0 -1833
  14. package/runtime-src/auth/browser-cookies.ts +0 -423
  15. package/runtime-src/auth/index.ts +0 -535
  16. package/runtime-src/auth/runtime.ts +0 -116
  17. package/runtime-src/browser/index.ts +0 -659
  18. package/runtime-src/browser/types.ts +0 -41
  19. package/runtime-src/build-info.generated.ts +0 -6
  20. package/runtime-src/capture/index.ts +0 -1929
  21. package/runtime-src/capture/prefetch.ts +0 -95
  22. package/runtime-src/capture/rsc.ts +0 -45
  23. package/runtime-src/cli/shortcuts.ts +0 -273
  24. package/runtime-src/cli.ts +0 -1734
  25. package/runtime-src/client/graph-client.ts +0 -100
  26. package/runtime-src/client/index.ts +0 -1425
  27. package/runtime-src/debug-trace.ts +0 -18
  28. package/runtime-src/domain.ts +0 -38
  29. package/runtime-src/execution/index.ts +0 -3407
  30. package/runtime-src/execution/retry.ts +0 -46
  31. package/runtime-src/execution/robots.ts +0 -167
  32. package/runtime-src/execution/search-forms.ts +0 -188
  33. package/runtime-src/execution/token-resolver.ts +0 -135
  34. package/runtime-src/extraction/index.ts +0 -1507
  35. package/runtime-src/foundry/publish-bundle.ts +0 -392
  36. package/runtime-src/graph/agent-augment.ts +0 -315
  37. package/runtime-src/graph/index.ts +0 -1532
  38. package/runtime-src/graph/local-fixtures.ts +0 -393
  39. package/runtime-src/graph/local-harness.ts +0 -646
  40. package/runtime-src/graph/planner.ts +0 -411
  41. package/runtime-src/graph/session.ts +0 -294
  42. package/runtime-src/graph/trace-store.ts +0 -136
  43. package/runtime-src/impact-log.ts +0 -227
  44. package/runtime-src/index.ts +0 -24
  45. package/runtime-src/indexer/index.ts +0 -465
  46. package/runtime-src/intent-match.ts +0 -1515
  47. package/runtime-src/kuri/client.ts +0 -1839
  48. package/runtime-src/logger.ts +0 -30
  49. package/runtime-src/marketplace/index.ts +0 -111
  50. package/runtime-src/mcp.ts +0 -1911
  51. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  52. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  53. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  54. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  55. package/runtime-src/orchestrator/index.ts +0 -4480
  56. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  57. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  58. package/runtime-src/payments/cascade.ts +0 -137
  59. package/runtime-src/payments/index.ts +0 -270
  60. package/runtime-src/payments/lobster-pay.ts +0 -182
  61. package/runtime-src/payments/wallet.ts +0 -98
  62. package/runtime-src/publish/review-context.ts +0 -93
  63. package/runtime-src/publish/sanitize.ts +0 -197
  64. package/runtime-src/publish/schema-review.ts +0 -192
  65. package/runtime-src/publish-admission.ts +0 -388
  66. package/runtime-src/ratelimit/index.ts +0 -23
  67. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  68. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  69. package/runtime-src/reverse-engineer/index.ts +0 -1551
  70. package/runtime-src/reverse-engineer/token-sources.ts +0 -379
  71. package/runtime-src/router.ts +0 -17
  72. package/runtime-src/routing-telemetry.ts +0 -395
  73. package/runtime-src/runtime/browser-access.ts +0 -11
  74. package/runtime-src/runtime/browser-auth.ts +0 -12
  75. package/runtime-src/runtime/browser-host.ts +0 -48
  76. package/runtime-src/runtime/lifecycle.ts +0 -17
  77. package/runtime-src/runtime/local-server.ts +0 -311
  78. package/runtime-src/runtime/paths.ts +0 -99
  79. package/runtime-src/runtime/setup.ts +0 -251
  80. package/runtime-src/runtime/supervisor.ts +0 -69
  81. package/runtime-src/runtime/update-hints.ts +0 -351
  82. package/runtime-src/server.ts +0 -100
  83. package/runtime-src/session-logs.ts +0 -142
  84. package/runtime-src/settings.ts +0 -221
  85. package/runtime-src/single-binary.ts +0 -143
  86. package/runtime-src/site-policy.ts +0 -54
  87. package/runtime-src/stale-cleanup-runner.ts +0 -144
  88. package/runtime-src/stale-cleanup.ts +0 -133
  89. package/runtime-src/telemetry-attribution.ts +0 -120
  90. package/runtime-src/telemetry.ts +0 -253
  91. package/runtime-src/template-params.ts +0 -141
  92. package/runtime-src/transform/drift.ts +0 -60
  93. package/runtime-src/transform/index.ts +0 -277
  94. package/runtime-src/types/index.ts +0 -1
  95. package/runtime-src/types/skill.ts +0 -931
  96. package/runtime-src/vault/index.ts +0 -196
  97. package/runtime-src/verification/auth-gate.ts +0 -8
  98. package/runtime-src/verification/candidates.ts +0 -27
  99. package/runtime-src/verification/index.ts +0 -120
  100. package/runtime-src/verification/matrix.ts +0 -30
  101. package/runtime-src/version.ts +0 -148
  102. package/runtime-src/workflow/artifact.ts +0 -161
  103. package/runtime-src/workflow/compile.ts +0 -808
  104. package/runtime-src/workflow/publish.ts +0 -225
  105. package/runtime-src/workflow/runtime.ts +0 -213
@@ -1,1507 +0,0 @@
1
- import * as cheerio from "cheerio";
2
- import { assessIntentResult } from "../intent-match.js";
3
-
4
- // eslint-disable-next-line @typescript-eslint/no-explicit-any -- cheerio v1.x doesn't export Element directly
5
- type CheerioEl = any;
6
-
7
- // --- Tag / attribute removal sets ---
8
-
9
- const STRIP_TAGS = new Set(["script", "style", "noscript", "svg", "iframe"]);
10
- const CHROME_TAGS = new Set(["nav", "footer", "header"]);
11
-
12
- const AD_PATTERNS = /\b(ad|ads|advert|advertisement|tracking|tracker|cookie-banner|cookie-consent|cookie-notice|popup|modal-overlay|gdpr|consent|banner-promo)\b/i;
13
- const HIDDEN_ATTRS: Array<{ attr: string; value?: string }> = [
14
- { attr: "aria-hidden", value: "true" },
15
- { attr: "hidden" },
16
- ];
17
-
18
- // Selectors for "main content" regions — tried in priority order
19
- const CONTENT_SELECTORS = [
20
- "main",
21
- "article",
22
- "[role=\"main\"]",
23
- "#content",
24
- ".content",
25
- ];
26
-
27
- // Common repeating-element selectors for card detection
28
- const CARD_SELECTORS = [
29
- ".card", ".item", ".result", ".product", ".listing",
30
- ".entry", ".post", ".tile", ".row",
31
- "[class*='card']", "[class*='item']", "[class*='result']",
32
- "[class*='product']", "[class*='listing']",
33
- ".cds-ProductCard-card", ".cds-ProductCard", "[class*='ProductCard-card']", "[class*='ProductCard']",
34
- // Semantic HTML patterns — articles/sections as repeated items
35
- "article", "section > div > div",
36
- // Common e-commerce / catalog patterns
37
- "[class*='pod']", "[class*='grid-item']", "[class*='col-']",
38
- ];
39
-
40
- // ---------------------------------------------------------------------------
41
- // extractSPAData — parse SPA-embedded JSON before cleanDOM strips scripts
42
- // ---------------------------------------------------------------------------
43
-
44
- interface SPAExtraction extends ExtractedStructure {
45
- type: "spa-nextjs" | "spa-nuxt" | "spa-initial-state" | "spa-preloaded-state";
46
- }
47
-
48
- function extractFlashNoticeSpecial(html: string, intent: string): ExtractedStructure[] {
49
- if (!/\b(flash|message|messages|alert|success|error|warning)\b/i.test(intent)) return [];
50
- const $ = cheerio.load(html);
51
- const flash = $("#flash, .flash, .alert, [role='alert']").first();
52
- if (flash.length === 0) return [];
53
- const flashText = flash.text().replace(/\s+/g, " ").replace(/[×x]\s*$/, "").trim();
54
- if (!flashText || flashText.length < 4) return [];
55
- const title = $("main h1, main h2, article h1, article h2, h1, h2").first().text().trim();
56
- return [{
57
- type: "key-value",
58
- data: {
59
- ...(title ? { title } : {}),
60
- flash: flashText,
61
- message: flashText,
62
- },
63
- element_count: title ? 2 : 1,
64
- selector: buildReplaySelector(flash),
65
- }];
66
- }
67
-
68
- /**
69
- * Extract structured data embedded by SPA frameworks BEFORE cleanDOM strips scripts.
70
- * Must be called on raw HTML.
71
- */
72
- export function extractSPAData(html: string): SPAExtraction[] {
73
- const results: SPAExtraction[] = [];
74
-
75
- // --- Next.js: <script id="__NEXT_DATA__" type="application/json"> ---
76
- const nextDataMatch = html.match(/<script\s+id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/i);
77
- if (nextDataMatch) {
78
- try {
79
- const parsed = JSON.parse(nextDataMatch[1]);
80
- const pageProps = parsed?.props?.pageProps;
81
- if (pageProps && typeof pageProps === "object" && Object.keys(pageProps).length > 0) {
82
- results.push({
83
- type: "spa-nextjs",
84
- data: pageProps,
85
- element_count: countDataElements(pageProps),
86
- });
87
- }
88
- } catch { /* malformed __NEXT_DATA__ */ }
89
- }
90
-
91
- // --- Nuxt.js: window.__NUXT__={...} or <script>window.__NUXT__=... ---
92
- const nuxtMatch = html.match(/window\.__NUXT__\s*=\s*(\{[\s\S]*?\});?\s*(?:<\/script>|$)/i);
93
- if (nuxtMatch) {
94
- try {
95
- const parsed = JSON.parse(nuxtMatch[1]);
96
- const data = parsed?.data?.[0] ?? parsed?.state ?? parsed;
97
- if (data && typeof data === "object" && Object.keys(data).length > 0) {
98
- results.push({
99
- type: "spa-nuxt",
100
- data,
101
- element_count: countDataElements(data),
102
- });
103
- }
104
- } catch { /* malformed __NUXT__ — often not pure JSON, skip */ }
105
- }
106
-
107
- // --- Generic: window.__INITIAL_STATE__ ---
108
- const initialStateMatch = html.match(/window\.__INITIAL_STATE__\s*=\s*(\{[\s\S]*?\});?\s*(?:<\/script>|$)/i);
109
- if (initialStateMatch) {
110
- try {
111
- const parsed = JSON.parse(initialStateMatch[1]);
112
- if (parsed && typeof parsed === "object" && Object.keys(parsed).length > 0) {
113
- results.push({
114
- type: "spa-initial-state",
115
- data: parsed,
116
- element_count: countDataElements(parsed),
117
- });
118
- }
119
- } catch { /* malformed __INITIAL_STATE__ */ }
120
- }
121
-
122
- // --- Generic: window.__PRELOADED_STATE__ ---
123
- const preloadedMatch = html.match(/window\.__PRELOADED_STATE__\s*=\s*(\{[\s\S]*?\});?\s*(?:<\/script>|$)/i);
124
- if (preloadedMatch) {
125
- try {
126
- const parsed = JSON.parse(preloadedMatch[1]);
127
- if (parsed && typeof parsed === "object" && Object.keys(parsed).length > 0) {
128
- results.push({
129
- type: "spa-preloaded-state",
130
- data: parsed,
131
- element_count: countDataElements(parsed),
132
- });
133
- }
134
- } catch { /* malformed __PRELOADED_STATE__ */ }
135
- }
136
-
137
- return results;
138
- }
139
-
140
- /** Count meaningful data elements in a nested structure */
141
- function countDataElements(obj: unknown, depth = 0): number {
142
- if (depth > 5) return 0;
143
- if (Array.isArray(obj)) return obj.reduce((sum, item) => sum + Math.max(1, countDataElements(item, depth + 1)), 0);
144
- if (obj && typeof obj === "object") {
145
- const keys = Object.keys(obj);
146
- return keys.reduce((sum, k) => sum + countDataElements((obj as Record<string, unknown>)[k], depth + 1), 0);
147
- }
148
- return 1;
149
- }
150
-
151
- // ---------------------------------------------------------------------------
152
- // cleanDOM
153
- // ---------------------------------------------------------------------------
154
-
155
- /**
156
- * Strip noise from raw page HTML — remove scripts, styles, nav chrome,
157
- * ads, hidden elements. Prefer content inside main/article regions.
158
- */
159
- export function cleanDOM(html: string): string {
160
- const $ = cheerio.load(html);
161
-
162
- // 1. Remove script/style/svg/iframe/noscript tags entirely
163
- // Preserve JSON-LD scripts — they contain structured data
164
- for (const tag of STRIP_TAGS) {
165
- if (tag === "script") {
166
- $("script").not('[type="application/ld+json"]').remove();
167
- } else {
168
- $(tag).remove();
169
- }
170
- }
171
-
172
- // 2. Remove navigation chrome
173
- for (const tag of CHROME_TAGS) {
174
- $(tag).remove();
175
- }
176
-
177
- // 3. Remove ad/tracking elements by class/id
178
- $("*").each((_, el) => {
179
- const $el = $(el);
180
- const cls = $el.attr("class") ?? "";
181
- const id = $el.attr("id") ?? "";
182
- if (AD_PATTERNS.test(cls) || AD_PATTERNS.test(id)) {
183
- $el.remove();
184
- }
185
- });
186
-
187
- // 4. Remove hidden elements
188
- $("[style]").each((_, el) => {
189
- const $el = $(el);
190
- const style = ($el.attr("style") ?? "").replace(/\s/g, "");
191
- if (style.includes("display:none") || style.includes("visibility:hidden")) {
192
- $el.remove();
193
- }
194
- });
195
- for (const { attr, value } of HIDDEN_ATTRS) {
196
- const selector = value ? `[${attr}="${value}"]` : `[${attr}]`;
197
- $(selector).remove();
198
- }
199
-
200
- // 5. Prefer content region if available (but only if it's a single container,
201
- // not multiple repeating elements like <article> per product)
202
- for (const sel of CONTENT_SELECTORS) {
203
- const region = $(sel);
204
- if (region.length === 1 && region.text().trim().length > 100) {
205
- return region.html() ?? $.html();
206
- }
207
- }
208
-
209
- return $("body").html() ?? $.html();
210
- }
211
-
212
- // ---------------------------------------------------------------------------
213
- // parseStructured
214
- // ---------------------------------------------------------------------------
215
-
216
- interface ExtractedStructure {
217
- type: string;
218
- data: unknown;
219
- element_count: number;
220
- selector?: string;
221
- }
222
-
223
- function hasMessageLikeRecord(value: unknown): boolean {
224
- if (!value || typeof value !== "object" || Array.isArray(value)) return false;
225
- const record = value as Record<string, unknown>;
226
- return (
227
- (typeof record.title === "string" || typeof record.heading_1 === "string" || typeof record.heading === "string") &&
228
- (typeof record.message === "string" || typeof record.description === "string" || typeof record.flash === "string")
229
- );
230
- }
231
-
232
- function isMessageLikeStructure(structure: ExtractedStructure, intent: string): boolean {
233
- if (!/\b(message|messages|flash|alert|success|error|warning)\b/i.test(intent)) return false;
234
- if (Array.isArray(structure.data)) return structure.data.some((item) => hasMessageLikeRecord(item));
235
- return hasMessageLikeRecord(structure.data);
236
- }
237
-
238
- function pruneRowsForIntent(rows: Record<string, string>[], intent: string): Record<string, string>[] {
239
- const lower = intent.toLowerCase();
240
- const keep = (predicate: (row: Record<string, string>) => boolean): Record<string, string>[] => rows.filter(predicate);
241
-
242
- if (/\b(question|questions)\b/.test(lower)) {
243
- return keep((row) =>
244
- !!(row.title || row.name) &&
245
- !!(row.url || row.link) &&
246
- !!(row.score || row.answer_count || row.author || row.date || row.meta || row.description) &&
247
- String(row.title ?? row.name ?? "").trim().length > 12
248
- );
249
- }
250
-
251
- if (/\b(post|posts|tweet|tweets|status|statuses)\b/.test(lower)) {
252
- return keep((row) =>
253
- !!(row.title || row.text || row.description) &&
254
- !!(row.url || row.link) &&
255
- !!(row.author || row.score || row.date || row.meta || row.description || row.text)
256
- );
257
- }
258
-
259
- if (/\b(doc|docs|documentation)\b/.test(lower)) {
260
- return keep((row) =>
261
- !!(row.title || row.name) &&
262
- !!(row.url || row.link) &&
263
- !!(row.summary || row.description || row.slug || row.meta)
264
- );
265
- }
266
-
267
- if (/\b(paper|papers)\b/.test(lower)) {
268
- return keep((row) =>
269
- !!(row.title || row.name) &&
270
- !!(row.url || row.link) &&
271
- !!(row.summary || row.description || row.author || row.date || row.meta)
272
- );
273
- }
274
-
275
- if (/\b(definition|dictionary|meaning)\b/.test(lower)) {
276
- return keep((row) =>
277
- !!(row.term || row.title || row.name) &&
278
- !!(row.definition || row.description)
279
- );
280
- }
281
-
282
- if (/\b(recipe|recipes)\b/.test(lower)) {
283
- return keep((row) =>
284
- !!(row.title || row.name) &&
285
- !!(row.url || row.link) &&
286
- !!(row.rating || row.description || row.author || row.meta)
287
- );
288
- }
289
-
290
- if (/\b(course|courses)\b/.test(lower)) {
291
- return keep((row) =>
292
- !!(row.title || row.name) &&
293
- !!(row.url || row.link) &&
294
- !!(row.rating || row.description || row.author || row.meta)
295
- );
296
- }
297
-
298
- return rows;
299
- }
300
-
301
- function normalizeStructureForIntent(structure: ExtractedStructure, intent: string): ExtractedStructure {
302
- if (structure.type !== "repeated-elements" || !Array.isArray(structure.data)) return structure;
303
- const objectRows = (structure.data as unknown[]).filter((row): row is Record<string, string> => !!row && typeof row === "object" && !Array.isArray(row));
304
- if (objectRows.length === 0) return structure;
305
- const pruned = pruneRowsForIntent(objectRows, intent);
306
- if (pruned.length >= 1 && pruned.length < objectRows.length) {
307
- return {
308
- ...structure,
309
- data: pruned,
310
- element_count: pruned.length,
311
- };
312
- }
313
- return structure;
314
- }
315
-
316
- function normalizeGitHubPath(href: string | undefined): string | null {
317
- if (!href) return null;
318
- const clean = href.split("?")[0].replace(/\/+$/, "");
319
- const match = clean.match(/^\/([^/]+)\/([^/]+)$/);
320
- if (!match) return null;
321
- const owner = match[1];
322
- const repo = match[2];
323
- if (/^(features|topics|collections|marketplace|orgs|users|settings|login|signup|sponsors|pricing|search|notifications|explore|pulls|issues)$/.test(owner)) return null;
324
- return `${owner}/${repo}`;
325
- }
326
-
327
- function cleanText(value: string): string {
328
- return value.replace(/\s+/g, " ").trim();
329
- }
330
-
331
- function normalizeLinkedInProfilePath(href: string | undefined): string | null {
332
- if (!href) return null;
333
- const clean = href.split("?")[0].replace(/\/+$/, "");
334
- const match = clean.match(/\/in\/([^/]+)$/);
335
- return match ? match[1] : null;
336
- }
337
-
338
- function extractGitHubSpecial(html: string, intent: string): ExtractedStructure[] {
339
- if (
340
- !/github/i.test(html) &&
341
- !/href=["']\/[^/"']+\/[^/"']+["']/i.test(html) &&
342
- !/data-target="react-app\.embeddedData"/i.test(html)
343
- ) return [];
344
- const $ = cheerio.load(html);
345
- const results: ExtractedStructure[] = [];
346
- const intentLower = intent.toLowerCase();
347
-
348
- const embeddedDataMatch = html.match(/<script[^>]+data-target="react-app\.embeddedData"[^>]*>([\s\S]*?)<\/script>/i);
349
- if (embeddedDataMatch && intentLower.includes("search")) {
350
- try {
351
- const parsed = JSON.parse(embeddedDataMatch[1]);
352
- const embeddedResults = parsed?.payload?.results;
353
- if (Array.isArray(embeddedResults) && embeddedResults.length >= 2) {
354
- const repos = embeddedResults
355
- .map((item: Record<string, unknown>) => {
356
- const repo = item.repo as { repository?: { owner_login?: string; name?: string } } | undefined;
357
- const owner = repo?.repository?.owner_login;
358
- const name = repo?.repository?.name;
359
- if (!owner || !name) return null;
360
- const row: Record<string, string> = {
361
- full_name: `${owner}/${name}`,
362
- url: `https://github.com/${owner}/${name}`,
363
- };
364
- const description = String(item.hl_trunc_description ?? "").replace(/<[^>]+>/g, "").trim();
365
- const language = String(item.language ?? "").trim();
366
- const stars = item.followers != null ? String(item.followers) : "";
367
- if (description) row.description = description;
368
- if (language) row.language = language;
369
- if (stars) row.stargazers_count = stars;
370
- return row;
371
- })
372
- .filter((row): row is Record<string, string> => !!row);
373
- if (repos.length >= 2) {
374
- results.push({ type: "repeated-elements", data: repos.slice(0, 20), element_count: repos.length });
375
- }
376
- }
377
- } catch { /* malformed embedded data */ }
378
- }
379
-
380
- const repoNwo = $('meta[name="octolytics-dimension-repository_nwo"]').attr("content")?.trim();
381
- if (repoNwo && (intentLower.includes("repository") || intentLower.includes("repo"))) {
382
- const ogDesc = $('meta[property="og:description"]').attr("content")?.trim() || "";
383
- const stars = $("#repo-stars-counter-star").first().text().trim()
384
- || $('a[href$="/stargazers"]').first().text().replace(/\s+/g, " ").trim();
385
- const forks = $('a[href$="/forks"]').first().text().replace(/\s+/g, " ").trim();
386
- const about = $("h2").filter((_, el) => $(el).text().trim() === "About").first()
387
- .parent().text().replace(/\s+/g, " ").trim();
388
- const data: Record<string, string> = {
389
- full_name: repoNwo,
390
- description: ogDesc || $('meta[name="description"]').attr("content")?.trim() || "",
391
- url: $('meta[property="og:url"]').attr("content")?.trim() || `https://github.com/${repoNwo}`,
392
- };
393
- if (stars) data.stars = stars;
394
- if (forks) data.forks = forks;
395
- if (about && about.length > 20 && about.length < 500) data.about = about;
396
- results.push({ type: "key-value", data, element_count: Object.keys(data).length });
397
- }
398
-
399
- if ((/search-results-page/.test(html) || /\/search\?/.test(html) || /resultsrepositories/i.test(html) || /href=["']\/[^/"']+\/[^/"']+["']/i.test(html)) && intentLower.includes("search")) {
400
- const seen = new Set<string>();
401
- const repos: Record<string, string>[] = [];
402
- $(".search-title a[href], [data-testid='results-list'] a[href], .search-results-container a[href], a[href^='/']").each((_, el) => {
403
- const $a = $(el);
404
- const href = $a.attr("href");
405
- const fullName = normalizeGitHubPath(href);
406
- if (!fullName || seen.has(fullName)) return;
407
- const title = cleanText($a.text());
408
- if (!title || title.length > 120) return;
409
- const card = $a.closest("div, li");
410
- const cardText = cleanText(card.text());
411
- const desc = cleanText(card.find("p").first().text());
412
- const lang = cleanText(card.find("[itemprop='programmingLanguage']").first().text());
413
- if (!/star|fork|updated|results?|language|repository/i.test(cardText) && !desc && !lang) return;
414
- seen.add(fullName);
415
- const stars = cleanText(card.find("a[href$='/stargazers']").first().text());
416
- const row: Record<string, string> = {
417
- full_name: fullName,
418
- url: `https://github.com/${fullName}`,
419
- };
420
- if (desc) row.description = desc;
421
- if (lang) row.language = lang;
422
- if (stars) row.stars = stars;
423
- repos.push(row);
424
- });
425
- if (repos.length >= 2) {
426
- results.push({ type: "repeated-elements", data: repos.slice(0, 10), element_count: repos.length });
427
- }
428
- }
429
-
430
- if (/trending/i.test(intentLower) || /\/trending\b/.test(html)) {
431
- const seen = new Set<string>();
432
- const repos: Record<string, string>[] = [];
433
- $("article.Box-row, article, .Box-row").each((_, el) => {
434
- const $el = $(el);
435
- const repoLink = $el.find("h1 a[href], h2 a[href], a[href^='/']").filter((_, a) => !!normalizeGitHubPath($(a).attr("href"))).first();
436
- const fullName = normalizeGitHubPath(repoLink.attr("href"));
437
- if (!fullName || seen.has(fullName)) return;
438
- const desc = $el.find("p").first().text().replace(/\s+/g, " ").trim();
439
- const lang = $el.find('[itemprop="programmingLanguage"]').first().text().trim();
440
- const stars = $el.find('a[href$="/stargazers"]').first().text().replace(/\s+/g, " ").trim();
441
- seen.add(fullName);
442
- const row: Record<string, string> = {
443
- full_name: fullName,
444
- url: `https://github.com/${fullName}`,
445
- };
446
- if (desc) row.description = desc;
447
- if (lang) row.language = lang;
448
- if (stars) row.stars = stars;
449
- repos.push(row);
450
- });
451
- if (repos.length >= 2) {
452
- results.push({ type: "repeated-elements", data: repos.slice(0, 20), element_count: repos.length });
453
- }
454
- }
455
-
456
- return results;
457
- }
458
-
459
- function extractLinkedInSpecial(html: string, intent: string): ExtractedStructure[] {
460
- if (!/linkedin/i.test(html)) return [];
461
- const intentLower = intent.toLowerCase();
462
- if (!/(search|people|person|profile|member)/.test(intentLower)) return [];
463
- const $ = cheerio.load(html);
464
- const seen = new Set<string>();
465
- const people: Record<string, string>[] = [];
466
-
467
- $("a[href*='/in/']").each((_, el) => {
468
- const $a = $(el);
469
- const handle = normalizeLinkedInProfilePath($a.attr("href"));
470
- if (!handle || seen.has(handle)) return;
471
- const name = cleanText($a.text());
472
- if (!name || name.length < 3 || name.length > 120) return;
473
- const card = $a.closest("li, div");
474
- const cardText = cleanText(card.text());
475
- if (cardText.length < name.length + 5) return;
476
- const headline = cleanText(
477
- card.find("div, span, p")
478
- .map((_, node) => cleanText($(node).text()))
479
- .get()
480
- .find((text) =>
481
- !!text &&
482
- text !== name &&
483
- text.length >= 8 &&
484
- text.length <= 220 &&
485
- !/^(message|connect|follow|premium|linkedin|see more|show all)$/i.test(text)
486
- ) ?? ""
487
- );
488
- const row: Record<string, string> = {
489
- name,
490
- url: `https://www.linkedin.com/in/${handle}`,
491
- public_identifier: handle,
492
- };
493
- if (headline) row.headline = headline;
494
- people.push(row);
495
- seen.add(handle);
496
- });
497
-
498
- return people.length >= 2 ? [{ type: "repeated-elements", data: people.slice(0, 10), element_count: people.length }] : [];
499
- }
500
-
501
- function extractPackageSearchSpecial(html: string, intent: string): ExtractedStructure[] {
502
- const intentLower = intent.toLowerCase();
503
- if (!/\bsearch\b/.test(intentLower) || !/\b(package|packages|crate|crates)\b/.test(intentLower)) return [];
504
- if (!/package-snippet/i.test(html)) return [];
505
- const $ = cheerio.load(html);
506
- const rows: Record<string, string>[] = [];
507
- const seen = new Set<string>();
508
-
509
- $("a.package-snippet[href], .package-snippet").each((_, el) => {
510
- const $el = $(el);
511
- const name = cleanText($el.find(".package-snippet__name").first().text());
512
- if (!name || seen.has(name)) return;
513
- const version = cleanText($el.find(".package-snippet__version").first().text());
514
- const description = cleanText($el.find(".package-snippet__description").first().text());
515
- const href = $el.attr("href") ?? "";
516
- rows.push({
517
- name,
518
- ...(version ? { version } : {}),
519
- ...(description ? { description } : {}),
520
- url: href ? new URL(href, "https://pypi.org").toString() : `https://pypi.org/project/${encodeURIComponent(name)}/`,
521
- });
522
- seen.add(name);
523
- });
524
-
525
- return rows.length >= 2 ? [{ type: "repeated-elements", data: rows.slice(0, 20), element_count: rows.length }] : [];
526
- }
527
-
528
- function extractXProfileSpecial(html: string, intent: string): ExtractedStructure[] {
529
- const intentLower = intent.toLowerCase();
530
- if (!/(person|people|profile|profiles|user|users|member)/.test(intentLower)) return [];
531
- if (!/(twitter|x\.com|twitter:|og:)/i.test(html)) return [];
532
- const $ = cheerio.load(html);
533
- const title = cleanText($("title").first().text());
534
- const ogTitle = cleanText($('meta[property="og:title"]').attr("content") ?? $('meta[name="twitter:title"]').attr("content") ?? "");
535
- const description = cleanText($('meta[name="description"]').attr("content") ?? $('meta[property="og:description"]').attr("content") ?? "");
536
- const canonical = ($('link[rel="canonical"]').attr("href") ?? $('meta[property="og:url"]').attr("content") ?? "").trim();
537
-
538
- const source = ogTitle || title;
539
- const titleMatch = source.match(/^(.*?)\s*\(@?([A-Za-z0-9_]{1,30})\)/);
540
- const handleFromUrl = canonical.match(/https?:\/\/(?:www\.)?x\.com\/([A-Za-z0-9_]{1,30})(?:\/|$)/)?.[1]
541
- ?? canonical.match(/https?:\/\/(?:www\.)?twitter\.com\/([A-Za-z0-9_]{1,30})(?:\/|$)/)?.[1];
542
- const username = titleMatch?.[2] ?? handleFromUrl ?? "";
543
- const name = cleanText(titleMatch?.[1] ?? source.replace(/\s*\/\s*[XT]$/i, ""));
544
-
545
- if (!name || !username) return [];
546
-
547
- const row: Record<string, string> = {
548
- name,
549
- username,
550
- public_identifier: username,
551
- url: canonical || `https://x.com/${username}`,
552
- };
553
- if (description) row.description = description;
554
- return [{ type: "key-value", data: row, element_count: 1 }];
555
- }
556
-
557
- function extractPostSpecial(html: string, intent: string): ExtractedStructure[] {
558
- const intentLower = intent.toLowerCase();
559
- if (!/(post|posts|tweet|tweets|status|statuses)/.test(intentLower)) return [];
560
- const $ = cheerio.load(html);
561
- const seen = new Set<string>();
562
- const posts: Record<string, string>[] = [];
563
-
564
- $("article, [role='article'], li, div").each((_, el) => {
565
- const $el = $(el);
566
- const link = $el.find("a[href*='/status/'], a[href*='/statuses/'], a[href*='/posts/'], a[href*='/@'], a[href*='/s/']").first();
567
- const href = link.attr("href");
568
- if (!href || href.length > 300) return;
569
- const canonical = href.split("?")[0];
570
- if (seen.has(canonical)) return;
571
- const title = cleanText(link.text());
572
-
573
- const text = cleanText(
574
- $el.find("p, span, div")
575
- .map((__, node) => cleanText($(node).text()))
576
- .get()
577
- .filter((value) =>
578
- value.length >= 20 &&
579
- value.length <= 700 &&
580
- !/^(reply|repost|like|share|show more|show less|follow|message)$/i.test(value)
581
- )
582
- .sort((a, b) => b.length - a.length)[0] ?? ""
583
- );
584
-
585
- const mastodonMatch = canonical.match(/\/@([^/]+)\/(\d+)/);
586
- const statusMatch = canonical.match(/\/status\/(\d+)/);
587
- const lobstersMatch = canonical.match(/\/s\/([^/]+)/);
588
- const username = mastodonMatch?.[1]
589
- ?? canonical.match(/\/([^/@]+)\/status\/\d+/)?.[1]
590
- ?? cleanText($el.find("[class*='author'], [class*='byline'], .u-author").first().text())
591
- ?? "";
592
- const id = mastodonMatch?.[2] ?? statusMatch?.[1] ?? lobstersMatch?.[1] ?? canonical.split("/").pop() ?? "";
593
- const score = cleanText($el.find("[class*='score'], [class*='points']").first().text());
594
-
595
- if (!text && !username && !title) return;
596
-
597
- posts.push({
598
- ...(id ? { id } : {}),
599
- ...(username ? { username } : {}),
600
- url: canonical,
601
- ...(title ? { title } : {}),
602
- ...(text ? { text } : {}),
603
- ...(score ? { score } : {}),
604
- ...(username ? { author: username } : {}),
605
- });
606
- seen.add(canonical);
607
- });
608
-
609
- return posts.length >= 1 ? [{ type: "repeated-elements", data: posts.slice(0, 20), element_count: posts.length }] : [];
610
- }
611
-
612
- function extractDefinitionSpecial(html: string, intent: string): ExtractedStructure[] {
613
- const intentLower = intent.toLowerCase();
614
- if (!/(definition|dictionary|meaning)/.test(intentLower)) return [];
615
- const $ = cheerio.load(html);
616
- const root = $("main, article, [role='main'], .entry-body, .di-body").first();
617
- const scope = root.length > 0 ? root : $("body");
618
- const term = cleanText(scope.find("h1").first().text()) || cleanText($("h1").first().text());
619
- let definition = cleanText(
620
- scope.find("dd, [class*='def'], [class*='meaning'], [class*='definition']").first().text(),
621
- );
622
- let normalizedTerm = term;
623
- if ((!normalizedTerm || !definition || definition.length < 10)) {
624
- const ogTitle = cleanText($('meta[property="og:title"]').attr("content") ?? "");
625
- const metaDescription = cleanText($('meta[name="description"]').attr("content") ?? $('meta[itemprop="headline"]').attr("content") ?? "");
626
- const canonical = cleanText($('link[rel="canonical"]').attr("href") ?? "");
627
- if (!normalizedTerm) {
628
- normalizedTerm = ogTitle
629
- || canonical.split("/").filter(Boolean).pop()?.replace(/[-_]+/g, " ")
630
- || "";
631
- }
632
- if (!definition && metaDescription) {
633
- definition = metaDescription
634
- .replace(/^[A-Z0-9 _-]+\s+definition:\s*/i, "")
635
- .replace(/\s*Learn more\.?$/i, "")
636
- .replace(/&hellip;/g, "...")
637
- .trim();
638
- }
639
- }
640
- if (!normalizedTerm || !definition || definition.length < 10) return [];
641
- return [{
642
- type: "key-value",
643
- data: {
644
- term: normalizedTerm,
645
- title: normalizedTerm,
646
- definition,
647
- },
648
- element_count: 1,
649
- }];
650
- }
651
-
652
- function extractCourseSearchSpecial(html: string, intent: string): ExtractedStructure[] {
653
- if (!/\b(course|courses)\b/i.test(intent)) return [];
654
- if (!/ProductCard|CommonCard-titleLink|RatingStat|partnerNames/i.test(html)) return [];
655
- const $ = cheerio.load(html);
656
- const rows: Record<string, string>[] = [];
657
- const seen = new Set<string>();
658
-
659
- $(".cds-ProductCard-card, .cds-ProductCard, [class*='ProductCard-card'], [class*='ProductCard']").each((_, el) => {
660
- const $el = $(el);
661
- const fields = extractCardFields($, $el);
662
- const title = fields.title?.trim();
663
- const url = (fields.url ?? fields.link ?? "").trim();
664
- if (!title || !url || title === "All Results") return;
665
- const stable = `${title}|${url}`;
666
- if (seen.has(stable)) return;
667
- if (!fields.rating && !fields.partner && !fields.description) return;
668
- rows.push(fields);
669
- seen.add(stable);
670
- });
671
-
672
- return rows.length >= 2
673
- ? [{ type: "repeated-elements", data: rows.slice(0, 20), element_count: rows.length, selector: ".cds-ProductCard-card" }]
674
- : [];
675
- }
676
-
677
- function extractTrendSpecial(html: string, intent: string): ExtractedStructure[] {
678
- const intentLower = intent.toLowerCase();
679
- if (!/(trend|trending|topic|topics|hashtag|hashtags)/.test(intentLower)) return [];
680
- const $ = cheerio.load(html);
681
- const roots = $("main, [role='main'], section");
682
- const scope = roots.length > 0 ? roots.first() : $("body");
683
- const seen = new Set<string>();
684
- const topics: Record<string, string>[] = [];
685
-
686
- scope.find("a[href]").each((_, el) => {
687
- const $a = $(el);
688
- const href = ($a.attr("href") ?? "").trim();
689
- const name = cleanText($a.text());
690
- if (!href || !name || name.length > 80 || name.length < 2) return;
691
- if (/^(home|explore|notifications|messages|lists|profile|more|show more|settings|terms|privacy)$/i.test(name)) return;
692
- const nearby = cleanText($a.closest("div, li, article, section").text());
693
- const trendish = name.startsWith("#")
694
- || /hashtag|trend|trending|topic/i.test(nearby)
695
- || /search\?q=|explore|hashtag/i.test(href);
696
- if (!trendish) return;
697
- const key = `${name}|${href.split("?")[0]}`;
698
- if (seen.has(key)) return;
699
- topics.push({ name, url: href });
700
- seen.add(key);
701
- });
702
-
703
- return topics.length >= 2 ? [{ type: "repeated-elements", data: topics.slice(0, 20), element_count: topics.length }] : [];
704
- }
705
-
706
- /**
707
- * Heuristic extraction of structured data from HTML.
708
- * Returns an array of discovered data structures.
709
- */
710
- export function parseStructured(html: string): ExtractedStructure[] {
711
- const $ = cheerio.load(html);
712
- const results: ExtractedStructure[] = [];
713
-
714
- // --- JSON-LD ---
715
- $('script[type="application/ld+json"]').each((_, el) => {
716
- try {
717
- const raw = $(el).html();
718
- if (!raw) return;
719
- const parsed = JSON.parse(raw);
720
- results.push({ type: "json-ld", data: parsed, element_count: 1 });
721
- } catch { /* malformed JSON-LD */ }
722
- });
723
-
724
- // --- Meta tags (Open Graph + schema.org) ---
725
- const meta: Record<string, string> = {};
726
- $("meta[property], meta[name]").each((_, el) => {
727
- const $el = $(el);
728
- const key = $el.attr("property") ?? $el.attr("name") ?? "";
729
- const content = $el.attr("content") ?? "";
730
- if ((key.startsWith("og:") || key.startsWith("article:") ||
731
- key.startsWith("twitter:") || key.startsWith("schema:")) && content) {
732
- meta[key] = content;
733
- }
734
- });
735
- if (Object.keys(meta).length > 0) {
736
- results.push({ type: "meta", data: meta, element_count: Object.keys(meta).length });
737
- }
738
-
739
- // --- Itemlist tables (HN-style: tr.athing with story rows) ---
740
- $("table").each((_, table) => {
741
- const $table = $(table);
742
- const athings = $table.find("tr.athing");
743
- if (athings.length >= 3) {
744
- const items: Record<string, string>[] = [];
745
- athings.each((_, tr) => {
746
- const $tr = $(tr);
747
- const item: Record<string, string> = {};
748
- const titleLink = $tr.find("span.titleline > a, td.title > span > a, td.title a.storylink").first();
749
- if (titleLink.length) {
750
- item.title = titleLink.text().trim();
751
- item.link = titleLink.attr("href") || "";
752
- }
753
- const rank = $tr.find("span.rank").text().trim().replace(".", "");
754
- if (rank) item.rank = rank;
755
- const $sub = $tr.next("tr");
756
- const score = $sub.find("span.score").text().trim();
757
- if (score) item.score = score;
758
- const age = $sub.find("span.age").text().trim();
759
- if (age) item.age = age;
760
- const author = $sub.find("a.hnuser").text().trim();
761
- if (author) item.author = author;
762
- const commentsLink = $sub.find("a").last().text().trim();
763
- if (commentsLink && commentsLink.includes("comment")) item.comments = commentsLink;
764
- if (item.title) items.push(item);
765
- });
766
- if (items.length >= 3) {
767
- results.push({ type: "itemlist", data: items, element_count: items.length });
768
- $table.remove();
769
- return;
770
- }
771
- }
772
- });
773
-
774
- // --- Tables ---
775
- $("table").each((_, table) => {
776
- const rows = parseTable($, $(table));
777
- if (rows.length > 0) {
778
- results.push({ type: "table", data: rows, element_count: rows.length });
779
- }
780
- });
781
-
782
- // --- Definition lists (key-value pairs) ---
783
- $("dl").each((_, dl) => {
784
- const pairs = parseDL($, $(dl));
785
- if (Object.keys(pairs).length > 0) {
786
- results.push({ type: "key-value", data: pairs, element_count: Object.keys(pairs).length });
787
- }
788
- });
789
-
790
- // --- Ordered/unordered lists ---
791
- $("ul, ol").each((_, list) => {
792
- const $list = $(list);
793
- // Only capture lists with structured content (multiple li with text)
794
- const items: string[] = [];
795
- $list.children("li").each((_, li) => {
796
- const text = $(li).text().trim();
797
- if (text) items.push(text);
798
- });
799
- if (items.length >= 2) {
800
- results.push({ type: "list", data: items, element_count: items.length });
801
- }
802
- });
803
-
804
- // --- Repeating card/element patterns ---
805
- const cardResults = detectRepeatingPatterns($);
806
- results.push(...cardResults);
807
-
808
- // --- Single-record detail pages ---
809
- const detailResults = detectDetailPatterns($);
810
- results.push(...detailResults);
811
-
812
- return results;
813
- }
814
-
815
- function parseTable($: cheerio.CheerioAPI, $table: cheerio.Cheerio<CheerioEl>): Record<string, string>[] {
816
- const headers: string[] = [];
817
- $table.find("thead th, thead td, tr:first-child th").each((_, th) => {
818
- headers.push($(th).text().trim());
819
- });
820
-
821
- // If no headers found in thead, try first row
822
- if (headers.length === 0) {
823
- const firstRow = $table.find("tr").first();
824
- firstRow.find("td, th").each((_, cell) => {
825
- headers.push($(cell).text().trim());
826
- });
827
- }
828
-
829
- if (headers.length === 0) return [];
830
-
831
- const hasThead = $table.find("thead").length > 0;
832
- // When thead exists, only iterate tbody rows; otherwise skip the first row (used as headers)
833
- const dataRows = hasThead
834
- ? $table.find("tbody tr").toArray()
835
- : $table.find("tr").toArray().slice(1);
836
-
837
- const rows: Record<string, string>[] = [];
838
- for (let i = 0; i < dataRows.length; i++) {
839
- const row: Record<string, string> = {};
840
- let hasData = false;
841
- $(dataRows[i]).find("td, th").each((j, cell) => {
842
- if (j < headers.length && headers[j]) {
843
- const val = $(cell).text().trim();
844
- if (val) {
845
- row[headers[j]] = val;
846
- hasData = true;
847
- }
848
- }
849
- });
850
- if (hasData) rows.push(row);
851
- }
852
-
853
- return rows;
854
- }
855
-
856
- function parseDL($: cheerio.CheerioAPI, $dl: cheerio.Cheerio<CheerioEl>): Record<string, string> {
857
- const result: Record<string, string> = {};
858
- let currentKey = "";
859
- $dl.children("dt, dd").each((_, el) => {
860
- const tag = (el as CheerioEl).tagName?.toLowerCase();
861
- if (tag === "dt") {
862
- currentKey = $(el).text().trim();
863
- } else if (tag === "dd" && currentKey) {
864
- result[currentKey] = $(el).text().trim();
865
- currentKey = "";
866
- }
867
- });
868
- return result;
869
- }
870
-
871
- function detectRepeatingPatterns($: cheerio.CheerioAPI): ExtractedStructure[] {
872
- const results: ExtractedStructure[] = [];
873
- const seen = new Set<string>();
874
-
875
- for (const selector of CARD_SELECTORS) {
876
- const elements = $(selector);
877
- if (elements.length < 2) continue;
878
-
879
- // Deduplicate by parent to avoid capturing the same set via multiple selectors
880
- const parent = elements.first().parent();
881
- const parentId = getElementSignature($, parent);
882
- if (seen.has(parentId)) continue;
883
- seen.add(parentId);
884
-
885
- const items: Record<string, string>[] = [];
886
- elements.each((_, el) => {
887
- const item = extractCardFields($, $(el));
888
- // Require at least 2 fields to be a meaningful card
889
- if (Object.keys(item).length >= 2) items.push(item);
890
- });
891
-
892
- if (items.length >= 2) {
893
- results.push({
894
- type: "repeated-elements",
895
- data: items,
896
- element_count: items.length,
897
- selector: buildReplaySelector($(elements[0])) ?? selector,
898
- });
899
- }
900
- }
901
-
902
- // Sibling-based detection: group child elements by identical class strings.
903
- // Handles Tailwind/utility-class sites where class names are non-semantic
904
- // (e.g. "h-full cursor-pointer overflow-hidden rounded-lg flex flex-col").
905
- if (results.length === 0) {
906
- const siblingGroups = detectSiblingPatterns($);
907
- results.push(...siblingGroups);
908
- }
909
-
910
- return results;
911
- }
912
-
913
- function hasDetailFieldShape(fields: Record<string, string>): boolean {
914
- if (!fields.title && !fields.name && !fields.term) return false;
915
- return !!(
916
- fields.description ||
917
- fields.definition ||
918
- fields.price ||
919
- fields.rating ||
920
- fields.author ||
921
- fields.url ||
922
- fields.link ||
923
- fields.score ||
924
- fields.image
925
- );
926
- }
927
-
928
- function detectDetailPatterns($: cheerio.CheerioAPI): ExtractedStructure[] {
929
- const results: ExtractedStructure[] = [];
930
- const seen = new Set<string>();
931
-
932
- for (const selector of [
933
- "main",
934
- "article",
935
- "[role='main']",
936
- "[class*='detail']",
937
- "[class*='details']",
938
- "[class*='product']",
939
- "[class*='listing']",
940
- "[class*='profile']",
941
- "[class*='content']",
942
- ]) {
943
- $(selector).each((_, el) => {
944
- const $el = $(el);
945
- const signature = `${selector}|${getElementSignature($, $el)}`;
946
- if (seen.has(signature)) return;
947
- seen.add(signature);
948
- if ($el.text().trim().length < 20) return;
949
- const fields = extractCardFields($, $el);
950
- if (Object.keys(fields).length < 2) return;
951
- if (!hasDetailFieldShape(fields)) return;
952
- results.push({
953
- type: "key-value",
954
- data: fields,
955
- element_count: 1,
956
- selector: buildReplaySelector($el) ?? selector,
957
- });
958
- });
959
- }
960
-
961
- return results;
962
- }
963
-
964
- /**
965
- * Detect repeating sibling elements that share the same full class string.
966
- * Works for Tailwind/utility-class sites where standard selectors fail.
967
- */
968
- function detectSiblingPatterns($: cheerio.CheerioAPI): ExtractedStructure[] {
969
- const results: ExtractedStructure[] = [];
970
- const seenParents = new Set<string>();
971
-
972
- // Scan all elements that could be container parents
973
- $("div, section, ul, ol, main").each((_, parent) => {
974
- const $parent = $(parent);
975
- const children = $parent.children();
976
- if (children.length < 3) return;
977
-
978
- // Group children by their full class string
979
- const groups = new Map<string, CheerioEl[]>();
980
- children.each((_, child) => {
981
- const cls = $(child).attr("class") || "";
982
- if (cls.length < 3) return; // skip classless or trivially-classed elements
983
- const key = `${(child as any).tagName}|${cls}`;
984
- const arr = groups.get(key) || [];
985
- arr.push(child);
986
- groups.set(key, arr);
987
- });
988
-
989
- for (const [key, elements] of groups) {
990
- if (elements.length < 3) continue;
991
-
992
- // Avoid processing the same parent+class group twice
993
- const parentSig = getElementSignature($, $parent) + "|" + key;
994
- if (seenParents.has(parentSig)) continue;
995
- seenParents.add(parentSig);
996
-
997
- const items: Record<string, string>[] = [];
998
- for (const el of elements) {
999
- const item = extractCardFields($, $(el));
1000
- if (Object.keys(item).length >= 2) items.push(item);
1001
- }
1002
-
1003
- if (items.length >= 3) {
1004
- results.push({
1005
- type: "repeated-elements",
1006
- data: items,
1007
- element_count: items.length,
1008
- selector: buildReplaySelector($(elements[0])),
1009
- });
1010
- }
1011
- }
1012
- });
1013
-
1014
- return results;
1015
- }
1016
-
1017
- function getElementSignature($: cheerio.CheerioAPI, $el: cheerio.Cheerio<CheerioEl>): string {
1018
- const tag = $el.prop("tagName") ?? "?";
1019
- const cls = $el.attr("class") ?? "";
1020
- const id = $el.attr("id") ?? "";
1021
- return `${tag}#${id}.${cls}`;
1022
- }
1023
-
1024
- function extractCardFields($: cheerio.CheerioAPI, $el: cheerio.Cheerio<CheerioEl>): Record<string, string> {
1025
- const fields: Record<string, string> = {};
1026
-
1027
- // Extract text from headings (semantic tags + Bootstrap heading classes)
1028
- $el.find("h1, h2, h3, h4, h5, h6, .h1, .h2, .h3, .h4, .h5, .h6, [class*='title'], [class*='header-text'], [class*='hearder']").each((i, h) => {
1029
- const text = $(h).text().trim();
1030
- if (text && text.length < 300) fields[i === 0 ? "title" : `heading_${i}`] = text;
1031
- });
1032
- if (!fields["message"] && fields["heading_1"] && fields["heading_1"].length > 10) {
1033
- fields["message"] = fields["heading_1"];
1034
- }
1035
-
1036
- // Fallback title: strong/bold text or [class*='name']
1037
- if (!fields["title"]) {
1038
- const strong = $el.find("strong, b, [class*='name']").first();
1039
- if (strong.length) {
1040
- const text = strong.text().trim();
1041
- if (text && text.length < 200) fields["title"] = text;
1042
- }
1043
- }
1044
-
1045
- // Extract links
1046
- const links: string[] = [];
1047
- $el.find("a[href]").each((_, a) => {
1048
- const href = $(a).attr("href");
1049
- if (href && !href.startsWith("#") && !href.startsWith("javascript:")) {
1050
- links.push(href);
1051
- }
1052
- });
1053
- if (links.length > 0) {
1054
- fields["link"] = links[0];
1055
- fields["url"] = links[0];
1056
- }
1057
-
1058
- // Fallback title from link text
1059
- if (!fields["title"] && links.length > 0) {
1060
- const linkText = $el.find("a").first().text().trim();
1061
- if (linkText && linkText.length > 2 && linkText.length < 200 && !/^(read|more|view|see|click)/i.test(linkText)) {
1062
- fields["title"] = linkText;
1063
- }
1064
- }
1065
-
1066
- // Extract images
1067
- const img = $el.find("img[src]").first();
1068
- const imgSrc = img.attr("src");
1069
- if (imgSrc) fields["image"] = imgSrc;
1070
-
1071
- // Extract description/paragraph text (skip price paragraphs)
1072
- $el.find("p").each((_, p) => {
1073
- if (fields["description"]) return;
1074
- const $p = $(p);
1075
- const cls = $p.attr("class") ?? "";
1076
- if (/price|cost|amount|stock|availability/i.test(cls)) return;
1077
- const text = $p.text().trim();
1078
- if (text && text.length > 10) fields["description"] = text;
1079
- });
1080
-
1081
- // Generic summary/excerpt containers often hold the useful body text for docs/questions/cards.
1082
- if (!fields["description"]) {
1083
- $el.find("[class*='summary'], [class*='excerpt'], [class*='description'], [class*='desc'], [class*='snippet']").each((_, node) => {
1084
- if (fields["description"]) return;
1085
- const text = $(node).text().trim();
1086
- if (text && text.length > 10 && text.length < 500) fields["description"] = text;
1087
- });
1088
- }
1089
-
1090
- // Alerts / success pages often carry the user-facing payload in strong text or flash-like containers.
1091
- if (!fields["message"]) {
1092
- $el.find("[role='alert'], .flash, .alert, [class*='message'], [class*='flash'], [class*='alert'], p strong, p b").each((_, node) => {
1093
- if (fields["message"]) return;
1094
- const text = $(node).text().trim();
1095
- if (text && text.length > 5 && text.length < 500 && text !== fields["title"]) {
1096
- fields["message"] = text;
1097
- }
1098
- });
1099
- }
1100
- if (!fields["message"] && fields["description"] && /congratulations|successfully|logged in|logged out|welcome|error|invalid|warning|flash|alert/i.test(fields["description"])) {
1101
- fields["message"] = fields["description"];
1102
- }
1103
-
1104
- // Extract price-like patterns — use the most specific (deepest) match
1105
- const priceEl = $el.find(".price_color, [class*='price']:not(:has([class*='price'])), .price, .cost, .amount").first();
1106
- if (priceEl.length > 0) {
1107
- // Get only direct text content, not nested children
1108
- const priceText = priceEl.contents().filter((_, node) => node.type === "text" || (node as any).tagName === "span")
1109
- .text().trim();
1110
- if (priceText) fields["price"] = priceText;
1111
- }
1112
-
1113
- const scoreEl = $el.find("[class*='vote'], [class*='score'], [data-score]").first();
1114
- if (scoreEl.length > 0) {
1115
- const scoreText = scoreEl.text().trim() || scoreEl.attr("data-score")?.trim();
1116
- if (scoreText && scoreText.length < 80) fields["score"] = scoreText;
1117
- }
1118
-
1119
- const answersEl = $el.find("[class*='answer'], [data-answercount]").first();
1120
- if (answersEl.length > 0) {
1121
- const answersText = answersEl.text().trim() || answersEl.attr("data-answercount")?.trim();
1122
- if (answersText && answersText.length < 80) fields["answer_count"] = answersText;
1123
- }
1124
-
1125
- const ratingEl = $el.find("[class*='rating'], [aria-label*='rating'], [aria-label*='Rating'], [aria-valuenow], [aria-valuetext], [data-rating]").first();
1126
- if (ratingEl.length > 0) {
1127
- const ratingProbe = ratingEl.find("[aria-valuenow], [aria-valuetext], [aria-label*='rating'], [aria-label*='Rating']").first();
1128
- const ratingText = ratingProbe.attr("aria-valuenow")?.trim()
1129
- || ratingProbe.attr("aria-valuetext")?.trim()
1130
- || ratingProbe.attr("aria-label")?.trim()
1131
- || ratingEl.attr("aria-valuenow")?.trim()
1132
- || ratingEl.attr("aria-valuetext")?.trim()
1133
- || ratingEl.attr("aria-label")?.trim()
1134
- || ratingEl.attr("data-rating")?.trim()
1135
- || ratingProbe.text().trim()
1136
- || ratingEl.text().trim();
1137
- const numeric = ratingText?.match(/\b([0-5](?:\.\d)?)\b/)?.[1];
1138
- if (numeric) fields["rating"] = numeric;
1139
- else if (ratingText && ratingText.length < 80 && !/^rating$/i.test(ratingText)) fields["rating"] = ratingText;
1140
- }
1141
-
1142
- const authorEl = $el.find("[class*='author'], [class*='byline'], [class*='user'], [rel='author']").first();
1143
- if (authorEl.length > 0) {
1144
- const authorText = authorEl.text().trim();
1145
- if (authorText && authorText.length < 120) fields["author"] = authorText;
1146
- }
1147
-
1148
- const partnerEl = $el.find("[class*='partnerName'], [class*='partnerNames'], [class*='partner']").first();
1149
- if (partnerEl.length > 0) {
1150
- const partnerText = partnerEl.text().trim();
1151
- if (partnerText && partnerText.length < 160) fields["partner"] = partnerText;
1152
- }
1153
-
1154
- const definitionEl = $el.find("dd, [class*='def'], [class*='meaning'], [class*='definition']").first();
1155
- if (definitionEl.length > 0) {
1156
- const definitionText = definitionEl.text().trim();
1157
- if (definitionText && definitionText.length > 10 && definitionText.length < 600) fields["definition"] = definitionText;
1158
- }
1159
-
1160
- // Extract metadata spans (dates, citations, info text)
1161
- $el.find("[class*='date'], [class*='info'], [class*='meta'], [class*='citation'], [class*='addinfo'], time").each((_, s) => {
1162
- const text = $(s).text().trim();
1163
- if (text && text.length > 3 && text.length < 200) {
1164
- // Derive a key from the class name
1165
- const cls = ($(s).attr("class") ?? "").toLowerCase();
1166
- const key = cls.match(/(date|citation|info|meta|time|author|category)/)?.[1] ?? "info";
1167
- if (!fields[key]) fields[key] = text;
1168
- }
1169
- });
1170
-
1171
- // Fallback: capture the element's direct text if nothing else matched
1172
- if (Object.keys(fields).length === 0) {
1173
- const text = $el.text().trim();
1174
- if (text && text.length < 500) fields["text"] = text;
1175
- }
1176
-
1177
- return fields;
1178
- }
1179
-
1180
- function scoreSemanticFit(structure: ExtractedStructure, intent: string): number {
1181
- const assessment = assessIntentResult(structure.data, intent);
1182
- if (assessment.verdict === "pass") return 140;
1183
- if (assessment.verdict === "fail") return -140;
1184
- return 0;
1185
- }
1186
-
1187
- function scoreSparseLinkList(structure: ExtractedStructure): number {
1188
- if (structure.type !== "repeated-elements" || !Array.isArray(structure.data)) return 0;
1189
- const items = structure.data as Array<Record<string, unknown>>;
1190
- if (items.length < 4) return 0;
1191
- const sparse = items.filter((item) => {
1192
- const keys = Object.keys(item);
1193
- if (keys.length > 2) return false;
1194
- const title = typeof item.title === "string" ? item.title : typeof item.name === "string" ? item.name : "";
1195
- const link = typeof item.link === "string" ? item.link : typeof item.url === "string" ? item.url : "";
1196
- return !!title && !!link && title.length <= 32;
1197
- }).length;
1198
- return sparse / items.length >= 0.7 ? -80 : 0;
1199
- }
1200
-
1201
- function scoreFieldRichness(structure: ExtractedStructure): number {
1202
- if (structure.type !== "repeated-elements" || !Array.isArray(structure.data)) return 0;
1203
- const items = structure.data as Array<Record<string, unknown>>;
1204
- if (items.length === 0) return 0;
1205
- const avgFields = items.reduce((sum, item) => sum + Object.keys(item).length, 0) / items.length;
1206
- if (avgFields >= 4) return 14;
1207
- if (avgFields >= 3) return 8;
1208
- return 0;
1209
- }
1210
-
1211
- // ---------------------------------------------------------------------------
1212
- // extractFromDOM
1213
- // ---------------------------------------------------------------------------
1214
-
1215
- export interface ExtractionResult {
1216
- data: unknown;
1217
- extraction_method: string;
1218
- confidence: number;
1219
- selector?: string;
1220
- }
1221
-
1222
- function buildReplaySelector($el: cheerio.Cheerio<CheerioEl>): string | undefined {
1223
- const tag = $el.get(0)?.tagName;
1224
- if (!tag) return undefined;
1225
- const id = ($el.attr("id") ?? "").trim();
1226
- if (id) return `${tag}#${id.replace(/[^a-zA-Z0-9_-]/g, "")}`;
1227
- const classes = (($el.attr("class") ?? "").trim())
1228
- .split(/\s+/)
1229
- .filter(Boolean)
1230
- .slice(0, 4)
1231
- .map((part) => part.replace(/[^a-zA-Z0-9_-]/g, ""))
1232
- .filter(Boolean);
1233
- return classes.length > 0 ? `${tag}.${classes.join(".")}` : tag;
1234
- }
1235
-
1236
- function extractUsingSelector(html: string, selector: string): ExtractedStructure | null {
1237
- const $ = cheerio.load(cleanDOM(html));
1238
- const elements = $(selector);
1239
- if (elements.length < 1) return null;
1240
- const items: Record<string, string>[] = [];
1241
- elements.each((_, el) => {
1242
- const fields = extractCardFields($, $(el));
1243
- if (Object.keys(fields).length >= 2) items.push(fields);
1244
- });
1245
- if (items.length >= 2) {
1246
- return { type: "repeated-elements", data: items, element_count: items.length, selector };
1247
- }
1248
- if (items.length === 1) {
1249
- return { type: "key-value", data: items[0], element_count: 1, selector };
1250
- }
1251
- return null;
1252
- }
1253
-
1254
- export function extractFromDOMWithHint(
1255
- html: string,
1256
- intent: string,
1257
- hint?: { selector?: string },
1258
- ): ExtractionResult {
1259
- if (hint?.selector) {
1260
- const extracted = extractUsingSelector(html, hint.selector);
1261
- if (extracted) {
1262
- const assessment = assessIntentResult(extracted.data, intent);
1263
- if (assessment.verdict === "pass") {
1264
- return {
1265
- data: extracted.data,
1266
- extraction_method: extracted.type,
1267
- confidence: 0.95,
1268
- selector: hint.selector,
1269
- };
1270
- }
1271
- }
1272
- }
1273
- return extractFromDOM(html, intent);
1274
- }
1275
-
1276
- /**
1277
- * Main entry point: clean HTML, extract structured data, and return
1278
- * the best match for the given intent.
1279
- */
1280
- export function extractFromDOM(html: string, intent: string): ExtractionResult {
1281
- // Cap HTML size to prevent cheerio from hanging on massive pages
1282
- const MAX_HTML_SIZE = 300_000;
1283
- let workingHtml = html;
1284
- if (workingHtml.length > MAX_HTML_SIZE) {
1285
- // Strip attribute bloat first (class/style/data-* attributes inflate HTML 2-3x)
1286
- workingHtml = workingHtml
1287
- .replace(/\s+class="[^"]*"/g, "")
1288
- .replace(/\s+style="[^"]*"/g, "")
1289
- .replace(/\s+data-[a-z][-a-z]*="[^"]*"/g, "");
1290
- // If still too large, truncate keeping body content
1291
- if (workingHtml.length > MAX_HTML_SIZE) {
1292
- const bodyStart = workingHtml.indexOf("<body");
1293
- if (bodyStart > 0) {
1294
- workingHtml = workingHtml.substring(0, Math.max(MAX_HTML_SIZE, bodyStart + MAX_HTML_SIZE));
1295
- } else {
1296
- workingHtml = workingHtml.substring(0, MAX_HTML_SIZE);
1297
- }
1298
- }
1299
- }
1300
-
1301
- // Extract SPA-embedded data from raw HTML BEFORE cleanDOM strips scripts
1302
- const spaStructures = extractSPAData(workingHtml);
1303
- const flashStructures = extractFlashNoticeSpecial(workingHtml, intent);
1304
- const cleaned = cleanDOM(workingHtml);
1305
- const githubStructures = extractGitHubSpecial(workingHtml, intent);
1306
- const linkedInStructures = extractLinkedInSpecial(workingHtml, intent);
1307
- const packageSearchStructures = extractPackageSearchSpecial(workingHtml, intent);
1308
- const xProfileStructures = extractXProfileSpecial(workingHtml, intent);
1309
- const postStructures = extractPostSpecial(workingHtml, intent);
1310
- const trendStructures = extractTrendSpecial(workingHtml, intent);
1311
- const definitionStructures = extractDefinitionSpecial(workingHtml, intent);
1312
- const courseStructures = extractCourseSearchSpecial(workingHtml, intent);
1313
- const structures = [...flashStructures, ...githubStructures, ...linkedInStructures, ...packageSearchStructures, ...xProfileStructures, ...postStructures, ...trendStructures, ...definitionStructures, ...courseStructures, ...spaStructures, ...parseStructured(cleaned)]
1314
- .map((structure) => normalizeStructureForIntent(structure, intent));
1315
-
1316
- if (structures.length === 0) {
1317
- return { data: null, extraction_method: "none", confidence: 0 };
1318
- }
1319
-
1320
- // Score each structure by relevance to intent
1321
- const intentWords = intent.toLowerCase().split(/\s+/).filter(Boolean);
1322
- const scored = structures.map((s) => ({
1323
- structure: s,
1324
- score: scoreRelevance(s, intentWords) + scoreSemanticFit(s, intent) + scoreSparseLinkList(s) + scoreFieldRichness(s),
1325
- }));
1326
-
1327
- scored.sort((a, b) => b.score - a.score);
1328
-
1329
- const passing = scored.filter((candidate) => assessIntentResult(candidate.structure.data, intent).verdict === "pass");
1330
- const bestPassing = (() => {
1331
- if (passing.length === 0) return undefined;
1332
- const bestPassingOverall = passing[0];
1333
- const bestPassingSpa = passing.find((candidate) => candidate.structure.type.startsWith("spa-"));
1334
- // Prefer cleaner SPA payloads when they're effectively tied with DOM-derived candidates.
1335
- if (bestPassingSpa && bestPassingOverall && bestPassingSpa.score >= bestPassingOverall.score - 2) {
1336
- return bestPassingSpa;
1337
- }
1338
- return bestPassingOverall;
1339
- })();
1340
- if (bestPassing) {
1341
- return {
1342
- data: bestPassing.structure.data,
1343
- extraction_method: bestPassing.structure.type,
1344
- confidence: computeConfidence(bestPassing.structure, bestPassing.score),
1345
- selector: bestPassing.structure.selector,
1346
- };
1347
- }
1348
-
1349
- const best = scored[0];
1350
- if (isMessageLikeStructure(best.structure, intent)) {
1351
- return {
1352
- data: best.structure.data,
1353
- extraction_method: best.structure.type,
1354
- confidence: computeConfidence(best.structure, best.score),
1355
- selector: best.structure.selector,
1356
- };
1357
- }
1358
-
1359
- if (scored.length === 1) {
1360
- return {
1361
- data: best.structure.data,
1362
- extraction_method: best.structure.type,
1363
- confidence: computeConfidence(best.structure, best.score),
1364
- selector: best.structure.selector,
1365
- };
1366
- }
1367
-
1368
- const hasClearWinner = best.score > scored[1].score * 1.5;
1369
-
1370
- if (hasClearWinner && best.score > 0) {
1371
- return {
1372
- data: best.structure.data,
1373
- extraction_method: best.structure.type,
1374
- confidence: computeConfidence(best.structure, best.score),
1375
- selector: best.structure.selector,
1376
- };
1377
- }
1378
-
1379
- // No clear winner — return all structures
1380
- return {
1381
- data: scored.map((s) => ({
1382
- type: s.structure.type,
1383
- data: s.structure.data,
1384
- relevance_score: s.score,
1385
- })),
1386
- extraction_method: "multiple",
1387
- confidence: computeConfidence(best.structure, best.score) * 0.7,
1388
- selector: best.structure.selector,
1389
- };
1390
- }
1391
-
1392
- function scoreRelevance(structure: ExtractedStructure, intentWords: string[]): number {
1393
- const text = JSON.stringify(structure.data).toLowerCase();
1394
- let score = 0;
1395
- const intentSet = new Set(intentWords);
1396
-
1397
- for (const word of intentWords) {
1398
- if (word.length < 3) continue; // skip short words like "a", "to", etc.
1399
- // Count occurrences of intent word in the data
1400
- const regex = new RegExp(word, "gi");
1401
- const matches = text.match(regex);
1402
- if (matches) {
1403
- score += matches.length;
1404
- }
1405
- }
1406
-
1407
- // Bonus for highly structured data
1408
- if (structure.type === "spa-nextjs") score += 5;
1409
- if (structure.type.startsWith("spa-")) score += 3;
1410
- if (structure.type === "json-ld") score += 3;
1411
- if (structure.type === "itemlist") score += 3;
1412
- if (structure.type === "table") score += 2;
1413
- if (structure.type === "repeated-elements") score += 1;
1414
- if (structure.type === "key-value") score += 1;
1415
-
1416
- // GitHub/repo-aware shaping: prefer repo-shaped objects/lists over file tables.
1417
- if (structure.type === "key-value" && structure.data && typeof structure.data === "object" && !Array.isArray(structure.data)) {
1418
- const keys = Object.keys(structure.data as Record<string, unknown>);
1419
- if (keys.includes("full_name")) score += 4;
1420
- if (keys.includes("description")) score += 2;
1421
- if (keys.includes("stars")) score += 2;
1422
- if ((intentSet.has("repository") || intentSet.has("repo")) && keys.includes("full_name")) score += 6;
1423
- if (intentSet.has("info")) score += 2;
1424
- }
1425
-
1426
- if (structure.type === "repeated-elements" && Array.isArray(structure.data)) {
1427
- const items = structure.data as Array<Record<string, unknown>>;
1428
- const repoShaped = items.filter((item) => typeof item?.full_name === "string" || typeof item?.url === "string");
1429
- if (repoShaped.length >= 2) score += 8;
1430
- if ((intentSet.has("search") || intentSet.has("trending")) && repoShaped.length >= 2) score += 8;
1431
- const peopleShaped = items.filter((item) => typeof item?.name === "string" && (typeof item?.headline === "string" || typeof item?.public_identifier === "string"));
1432
- if (peopleShaped.length >= 2) score += 8;
1433
- if ((intentSet.has("people") || intentSet.has("person") || intentSet.has("profile")) && peopleShaped.length >= 2) score += 10;
1434
- const postShaped = items.filter((item) =>
1435
- (typeof item?.id === "string" || typeof item?.url === "string") &&
1436
- (typeof item?.text === "string" || typeof item?.content === "string" || typeof item?.username === "string")
1437
- );
1438
- if (postShaped.length >= 1) score += 8;
1439
- if ((intentSet.has("post") || intentSet.has("posts") || intentSet.has("status") || intentSet.has("statuses") || intentSet.has("tweet")) && postShaped.length >= 1) score += 10;
1440
- const topicShaped = items.filter((item) =>
1441
- (typeof item?.name === "string" || typeof item?.title === "string") &&
1442
- typeof item?.url === "string"
1443
- );
1444
- if (topicShaped.length >= 2) score += 8;
1445
- if ((intentSet.has("trend") || intentSet.has("trending") || intentSet.has("topic") || intentSet.has("topics") || intentSet.has("hashtag")) && topicShaped.length >= 2) score += 10;
1446
- }
1447
-
1448
- if (structure.type === "table" && Array.isArray(structure.data)) {
1449
- const keys = new Set((structure.data as Array<Record<string, unknown>>).flatMap((row) => Object.keys(row)));
1450
- if (keys.has("Last commit message") || keys.has("Last commit date")) score -= 8;
1451
- if (keys.has("Name") && !intentSet.has("file") && !intentSet.has("commit")) score -= 4;
1452
- }
1453
-
1454
- // Bonus for more elements (richer data)
1455
- score += Math.min(structure.element_count * 0.1, 2);
1456
-
1457
- return score;
1458
- }
1459
-
1460
- function computeConfidence(structure: ExtractedStructure, relevanceScore: number): number {
1461
- let confidence = 0;
1462
-
1463
- // Base confidence from structure type
1464
- switch (structure.type) {
1465
- case "spa-nextjs":
1466
- confidence = 0.9;
1467
- break;
1468
- case "spa-nuxt":
1469
- case "spa-initial-state":
1470
- case "spa-preloaded-state":
1471
- confidence = 0.85;
1472
- break;
1473
- case "json-ld":
1474
- confidence = 0.9;
1475
- break;
1476
- case "itemlist":
1477
- confidence = 0.9;
1478
- break;
1479
- case "table":
1480
- confidence = 0.8;
1481
- break;
1482
- case "repeated-elements":
1483
- confidence = 0.7;
1484
- break;
1485
- case "key-value":
1486
- confidence = 0.7;
1487
- break;
1488
- case "meta":
1489
- confidence = 0.6;
1490
- break;
1491
- case "list":
1492
- confidence = 0.5;
1493
- break;
1494
- default:
1495
- confidence = 0.3;
1496
- }
1497
-
1498
- // Boost from element count (more data = more confidence)
1499
- if (structure.element_count > 5) confidence += 0.05;
1500
- if (structure.element_count > 10) confidence += 0.05;
1501
-
1502
- // Boost from relevance score
1503
- if (relevanceScore > 5) confidence += 0.05;
1504
- if (relevanceScore > 10) confidence += 0.05;
1505
-
1506
- return Math.min(confidence, 1);
1507
- }