unbrowse 3.1.0 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/cli.js +455 -96
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +695 -46
  4. package/dist/server.js +25811 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/darwin-arm64/kuri +0 -0
  7. package/vendor/kuri/darwin-x64/kuri +0 -0
  8. package/vendor/kuri/linux-arm64/kuri +0 -0
  9. package/vendor/kuri/linux-x64/kuri +0 -0
  10. package/vendor/kuri/manifest.json +7 -10
  11. package/runtime-src/agent-outcome.ts +0 -166
  12. package/runtime-src/analytics-session.ts +0 -55
  13. package/runtime-src/api/browse-index.ts +0 -317
  14. package/runtime-src/api/browse-session.ts +0 -572
  15. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  16. package/runtime-src/api/browse-submit.ts +0 -1184
  17. package/runtime-src/api/routes.ts +0 -1823
  18. package/runtime-src/auth/browser-cookies.ts +0 -423
  19. package/runtime-src/auth/index.ts +0 -535
  20. package/runtime-src/auth/runtime.ts +0 -116
  21. package/runtime-src/browser/index.ts +0 -659
  22. package/runtime-src/browser/types.ts +0 -41
  23. package/runtime-src/build-info.generated.ts +0 -6
  24. package/runtime-src/capture/index.ts +0 -1794
  25. package/runtime-src/capture/prefetch.ts +0 -95
  26. package/runtime-src/capture/rsc.ts +0 -45
  27. package/runtime-src/cli/shortcuts.ts +0 -273
  28. package/runtime-src/cli.ts +0 -1572
  29. package/runtime-src/client/graph-client.ts +0 -100
  30. package/runtime-src/client/index.ts +0 -1425
  31. package/runtime-src/debug-trace.ts +0 -18
  32. package/runtime-src/domain.ts +0 -38
  33. package/runtime-src/execution/index.ts +0 -3397
  34. package/runtime-src/execution/retry.ts +0 -46
  35. package/runtime-src/execution/robots.ts +0 -167
  36. package/runtime-src/execution/search-forms.ts +0 -188
  37. package/runtime-src/extraction/index.ts +0 -1507
  38. package/runtime-src/foundry/publish-bundle.ts +0 -392
  39. package/runtime-src/graph/agent-augment.ts +0 -315
  40. package/runtime-src/graph/index.ts +0 -1524
  41. package/runtime-src/graph/local-fixtures.ts +0 -393
  42. package/runtime-src/graph/local-harness.ts +0 -646
  43. package/runtime-src/graph/planner.ts +0 -411
  44. package/runtime-src/graph/session.ts +0 -294
  45. package/runtime-src/graph/trace-store.ts +0 -136
  46. package/runtime-src/index.ts +0 -24
  47. package/runtime-src/indexer/index.ts +0 -465
  48. package/runtime-src/intent-match.ts +0 -1515
  49. package/runtime-src/kuri/client.ts +0 -1839
  50. package/runtime-src/logger.ts +0 -30
  51. package/runtime-src/marketplace/index.ts +0 -103
  52. package/runtime-src/mcp.ts +0 -1747
  53. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  54. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  55. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  56. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  57. package/runtime-src/orchestrator/index.ts +0 -4480
  58. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  59. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  60. package/runtime-src/payments/cascade.ts +0 -137
  61. package/runtime-src/payments/index.ts +0 -270
  62. package/runtime-src/payments/lobster-pay.ts +0 -182
  63. package/runtime-src/payments/wallet.ts +0 -98
  64. package/runtime-src/publish/review-context.ts +0 -93
  65. package/runtime-src/publish/sanitize.ts +0 -197
  66. package/runtime-src/publish/schema-review.ts +0 -192
  67. package/runtime-src/publish-admission.ts +0 -388
  68. package/runtime-src/ratelimit/index.ts +0 -23
  69. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  70. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  71. package/runtime-src/reverse-engineer/index.ts +0 -1551
  72. package/runtime-src/router.ts +0 -17
  73. package/runtime-src/routing-telemetry.ts +0 -395
  74. package/runtime-src/runtime/browser-access.ts +0 -11
  75. package/runtime-src/runtime/browser-auth.ts +0 -12
  76. package/runtime-src/runtime/browser-host.ts +0 -48
  77. package/runtime-src/runtime/lifecycle.ts +0 -17
  78. package/runtime-src/runtime/local-server.ts +0 -311
  79. package/runtime-src/runtime/paths.ts +0 -99
  80. package/runtime-src/runtime/setup.ts +0 -251
  81. package/runtime-src/runtime/supervisor.ts +0 -69
  82. package/runtime-src/runtime/update-hints.ts +0 -351
  83. package/runtime-src/server.ts +0 -100
  84. package/runtime-src/session-logs.ts +0 -142
  85. package/runtime-src/settings.ts +0 -221
  86. package/runtime-src/single-binary.ts +0 -143
  87. package/runtime-src/site-policy.ts +0 -54
  88. package/runtime-src/stale-cleanup-runner.ts +0 -144
  89. package/runtime-src/stale-cleanup.ts +0 -133
  90. package/runtime-src/telemetry-attribution.ts +0 -120
  91. package/runtime-src/telemetry.ts +0 -253
  92. package/runtime-src/template-params.ts +0 -141
  93. package/runtime-src/transform/drift.ts +0 -60
  94. package/runtime-src/transform/index.ts +0 -277
  95. package/runtime-src/types/index.ts +0 -1
  96. package/runtime-src/types/skill.ts +0 -912
  97. package/runtime-src/vault/index.ts +0 -196
  98. package/runtime-src/verification/auth-gate.ts +0 -8
  99. package/runtime-src/verification/candidates.ts +0 -27
  100. package/runtime-src/verification/index.ts +0 -120
  101. package/runtime-src/verification/matrix.ts +0 -30
  102. package/runtime-src/version.ts +0 -148
  103. package/runtime-src/workflow/artifact.ts +0 -161
  104. package/runtime-src/workflow/compile.ts +0 -808
  105. package/runtime-src/workflow/publish.ts +0 -225
  106. package/runtime-src/workflow/runtime.ts +0 -213
  107. package/vendor/kuri/win-x64/kuri.exe +0 -0
@@ -1,1515 +0,0 @@
1
- function isRecord(value: unknown): value is Record<string, unknown> {
2
- return !!value && typeof value === "object" && !Array.isArray(value);
3
- }
4
-
5
- function getPath(obj: unknown, path: string): unknown {
6
- if (!isRecord(obj)) return undefined;
7
- let cur: unknown = obj;
8
- for (const key of path.split(".")) {
9
- if (!isRecord(cur)) return undefined;
10
- cur = cur[key];
11
- }
12
- return cur;
13
- }
14
-
15
- function hasNonEmptyString(value: unknown): boolean {
16
- return typeof value === "string" && value.trim().length > 0;
17
- }
18
-
19
- function hasAnyPath(obj: unknown, paths: string[]): boolean {
20
- return paths.some((path) => {
21
- const value = getPath(obj, path);
22
- return hasNonEmptyString(value) || typeof value === "number";
23
- });
24
- }
25
-
26
- function countMatchingGroups(obj: unknown, groups: string[][]): number {
27
- let matches = 0;
28
- for (const group of groups) {
29
- if (hasAnyPath(obj, group)) matches++;
30
- }
31
- return matches;
32
- }
33
-
34
- function toStringArray(value: unknown): string[] | undefined {
35
- if (Array.isArray(value)) {
36
- const out = value
37
- .filter((item): item is string => hasNonEmptyString(item))
38
- .map((item) => String(item).trim());
39
- return out.length > 0 ? out : undefined;
40
- }
41
- if (hasNonEmptyString(value)) return [String(value).trim()];
42
- return undefined;
43
- }
44
-
45
- function firstNonEmptyString(...values: unknown[]): string | undefined {
46
- for (const value of values) {
47
- if (hasNonEmptyString(value)) return String(value).trim();
48
- }
49
- return undefined;
50
- }
51
-
52
- function normalizePackageSearchResults(data: unknown): Record<string, unknown>[] {
53
- const sourceRows = Array.isArray(data)
54
- ? data
55
- : isRecord(data) && Array.isArray(data.objects)
56
- ? data.objects
57
- : isRecord(data) && Array.isArray(data.packages)
58
- ? data.packages
59
- : [];
60
- const rows: Record<string, unknown>[] = [];
61
- const seen = new Set<string>();
62
-
63
- for (const item of sourceRows) {
64
- if (!isRecord(item)) continue;
65
- const pkg = isRecord(item.package) ? item.package : item;
66
- const name = getPath(pkg, "name");
67
- if (!hasNonEmptyString(name) || seen.has(String(name))) continue;
68
- const version = firstNonEmptyString(getPath(pkg, "version"));
69
- const description = firstNonEmptyString(getPath(pkg, "description"), getPath(pkg, "summary"));
70
- const url =
71
- firstNonEmptyString(
72
- getPath(pkg, "links.npm"),
73
- getPath(pkg, "project_url"),
74
- getPath(pkg, "package_url"),
75
- getPath(pkg, "url"),
76
- ) ?? `https://www.npmjs.com/package/${encodeURIComponent(String(name))}`;
77
- const keywords = toStringArray(getPath(pkg, "keywords"));
78
- rows.push({
79
- name: String(name),
80
- ...(version ? { version } : {}),
81
- ...(description ? { description } : {}),
82
- ...(keywords ? { keywords } : {}),
83
- url,
84
- });
85
- seen.add(String(name));
86
- }
87
-
88
- return rows;
89
- }
90
-
91
- function normalizeCratesPackageSearchResults(data: unknown): Record<string, unknown>[] {
92
- const sourceRows = isRecord(data) && Array.isArray(data.crates)
93
- ? data.crates
94
- : Array.isArray(data)
95
- ? data
96
- : [];
97
- const rows: Record<string, unknown>[] = [];
98
- const seen = new Set<string>();
99
-
100
- for (const item of sourceRows) {
101
- if (!isRecord(item)) continue;
102
- const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "id"));
103
- if (!name || seen.has(name)) continue;
104
- const description = firstNonEmptyString(getPath(item, "description"), getPath(item, "summary"));
105
- const version = firstNonEmptyString(getPath(item, "max_version"), getPath(item, "default_version"), getPath(item, "version"));
106
- rows.push({
107
- name,
108
- ...(description ? { description, summary: description } : {}),
109
- ...(version ? { version } : {}),
110
- ...(typeof getPath(item, "downloads") === "number" ? { downloads: getPath(item, "downloads") } : {}),
111
- url: `https://crates.io/crates/${encodeURIComponent(name)}`,
112
- });
113
- seen.add(name);
114
- }
115
-
116
- return rows;
117
- }
118
-
119
- function normalizeNpmPackageInfo(data: unknown): Record<string, unknown> | null {
120
- if (!isRecord(data) || !hasNonEmptyString(data.name)) return null;
121
- const distTags = isRecord(data["dist-tags"]) ? data["dist-tags"] : undefined;
122
- const versions = isRecord(data.versions) ? data.versions : undefined;
123
- const latestVersion = firstNonEmptyString(distTags?.latest, data.version);
124
- const latestRecord =
125
- latestVersion && versions && isRecord(versions[latestVersion])
126
- ? versions[latestVersion]
127
- : undefined;
128
- const description = firstNonEmptyString(
129
- getPath(latestRecord, "description"),
130
- getPath(data, "description"),
131
- );
132
- const homepage = firstNonEmptyString(
133
- getPath(latestRecord, "homepage"),
134
- getPath(data, "homepage"),
135
- getPath(latestRecord, "repository.url"),
136
- getPath(data, "repository.url"),
137
- );
138
- const dependencies = isRecord(getPath(latestRecord, "dependencies"))
139
- ? Object.keys(getPath(latestRecord, "dependencies") as Record<string, unknown>)
140
- : undefined;
141
- const keywords = toStringArray(getPath(latestRecord, "keywords") ?? getPath(data, "keywords"));
142
- const author = firstNonEmptyString(
143
- getPath(latestRecord, "author.name"),
144
- getPath(latestRecord, "author"),
145
- getPath(data, "author.name"),
146
- getPath(data, "author"),
147
- );
148
-
149
- return {
150
- name: String(data.name),
151
- ...(latestVersion ? { version: latestVersion } : {}),
152
- ...(description ? { description } : {}),
153
- ...(keywords ? { keywords } : {}),
154
- ...(dependencies && dependencies.length > 0 ? { dependencies } : {}),
155
- ...(author ? { author } : {}),
156
- url: homepage ?? firstNonEmptyString(getPath(data, "url"), getPath(data, "package_url"), getPath(data, "project_url"))
157
- ?? `https://www.npmjs.com/package/${encodeURIComponent(String(data.name))}`,
158
- };
159
- }
160
-
161
- function normalizeDocumentRows(data: unknown): Record<string, unknown>[] {
162
- const sourceRows = Array.isArray(data)
163
- ? data
164
- : isRecord(data) && Array.isArray(data.documents)
165
- ? data.documents
166
- : isRecord(data) && Array.isArray(data.results)
167
- ? data.results
168
- : [];
169
- const rows: Record<string, unknown>[] = [];
170
- const seen = new Set<string>();
171
-
172
- for (const item of sourceRows) {
173
- if (!isRecord(item)) continue;
174
- const title = firstNonEmptyString(getPath(item, "title"), getPath(item, "name"));
175
- const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "mdn_url"), getPath(item, "link"), getPath(item, "href"));
176
- if (!title || !url) continue;
177
- const id = `${title}|${url}`;
178
- if (seen.has(id)) continue;
179
- rows.push({
180
- title,
181
- url,
182
- ...(firstNonEmptyString(getPath(item, "summary"), getPath(item, "description"), getPath(item, "slug")) ? {
183
- summary: firstNonEmptyString(getPath(item, "summary"), getPath(item, "description"), getPath(item, "slug")),
184
- } : {}),
185
- ...(typeof getPath(item, "score") === "number" ? { score: getPath(item, "score") } : {}),
186
- ...(typeof getPath(item, "popularity") === "number" ? { popularity: getPath(item, "popularity") } : {}),
187
- ...(firstNonEmptyString(getPath(item, "locale")) ? { locale: firstNonEmptyString(getPath(item, "locale")) } : {}),
188
- });
189
- seen.add(id);
190
- }
191
-
192
- return rows;
193
- }
194
-
195
- function normalizePyPIPackageInfo(data: unknown): Record<string, unknown> | null {
196
- if (!isRecord(data)) return null;
197
- const info = isRecord(data.info) ? data.info : data;
198
- const name = getPath(info, "name");
199
- if (!hasNonEmptyString(name)) return null;
200
- const summary = firstNonEmptyString(getPath(info, "summary"), getPath(info, "description"));
201
- const version = firstNonEmptyString(getPath(info, "version"));
202
- const author = firstNonEmptyString(getPath(info, "author"));
203
- const requiresDist = Array.isArray(getPath(info, "requires_dist"))
204
- ? (getPath(info, "requires_dist") as unknown[]).filter((item) => hasNonEmptyString(item)).map(String)
205
- : undefined;
206
- const url = firstNonEmptyString(
207
- getPath(info, "project_url"),
208
- getPath(info, "package_url"),
209
- getPath(info, "home_page"),
210
- getPath(info, "project_urls.Homepage"),
211
- );
212
-
213
- return {
214
- name: String(name),
215
- ...(version ? { version } : {}),
216
- ...(summary ? { summary, description: summary } : {}),
217
- ...(author ? { author } : {}),
218
- ...(requiresDist && requiresDist.length > 0 ? { requires_dist: requiresDist } : {}),
219
- ...(url ? { url } : {}),
220
- };
221
- }
222
-
223
- function normalizePubDevPackageInfo(data: unknown): Record<string, unknown> | null {
224
- if (!isRecord(data)) return null;
225
- const latest = isRecord(data.latest) ? data.latest : undefined;
226
- const pubspec = isRecord(latest?.pubspec) ? latest?.pubspec : undefined;
227
- const name = firstNonEmptyString(getPath(data, "name"), getPath(pubspec, "name"));
228
- if (!name) return null;
229
- const version = firstNonEmptyString(getPath(latest, "version"), getPath(pubspec, "version"));
230
- const description = firstNonEmptyString(getPath(pubspec, "description"));
231
- const repository = firstNonEmptyString(getPath(pubspec, "repository"), getPath(pubspec, "homepage"));
232
- const topics = toStringArray(getPath(pubspec, "topics"));
233
- return {
234
- name,
235
- ...(version ? { version } : {}),
236
- ...(description ? { description, summary: description } : {}),
237
- ...(topics ? { keywords: topics } : {}),
238
- url: repository ?? `https://pub.dev/packages/${encodeURIComponent(name)}`,
239
- };
240
- }
241
-
242
- function normalizeRubyGemsPackageInfo(data: unknown): Record<string, unknown> | null {
243
- if (!isRecord(data)) return null;
244
- const name = firstNonEmptyString(getPath(data, "name"));
245
- if (!name) return null;
246
- const description = firstNonEmptyString(getPath(data, "info"));
247
- const version = firstNonEmptyString(getPath(data, "version"));
248
- const author = firstNonEmptyString(getPath(data, "authors"), getPath(data, "author"));
249
- const homepage = firstNonEmptyString(getPath(data, "homepage_uri"), getPath(data, "project_uri"), getPath(data, "gem_uri"));
250
- return {
251
- name,
252
- ...(version ? { version } : {}),
253
- ...(description ? { description, summary: description } : {}),
254
- ...(author ? { author } : {}),
255
- url: homepage ?? `https://rubygems.org/gems/${encodeURIComponent(name)}`,
256
- };
257
- }
258
-
259
- function normalizeDockerImageSearchResults(data: unknown): Record<string, unknown>[] {
260
- const sourceRows = isRecord(data) && Array.isArray(data.results)
261
- ? data.results
262
- : Array.isArray(data)
263
- ? data
264
- : [];
265
- const rows: Record<string, unknown>[] = [];
266
- const seen = new Set<string>();
267
-
268
- for (const item of sourceRows) {
269
- if (!isRecord(item)) continue;
270
- const repoName = firstNonEmptyString(getPath(item, "repo_name"), getPath(item, "name"));
271
- if (!repoName || seen.has(repoName)) continue;
272
- rows.push({
273
- repo_name: repoName,
274
- ...(firstNonEmptyString(getPath(item, "short_description"), getPath(item, "description")) ? {
275
- short_description: firstNonEmptyString(getPath(item, "short_description"), getPath(item, "description")),
276
- } : {}),
277
- ...(typeof getPath(item, "star_count") === "number" ? { star_count: getPath(item, "star_count") } : {}),
278
- ...(typeof getPath(item, "pull_count") === "number" ? { pull_count: getPath(item, "pull_count") } : {}),
279
- url: `https://hub.docker.com/r/${repoName}`,
280
- });
281
- seen.add(repoName);
282
- }
283
-
284
- return rows;
285
- }
286
-
287
- function normalizeDockerTagResults(data: unknown): Record<string, unknown>[] {
288
- const sourceRows = isRecord(data) && Array.isArray(data.results)
289
- ? data.results
290
- : Array.isArray(data)
291
- ? data
292
- : [];
293
- const rows: Record<string, unknown>[] = [];
294
- const seen = new Set<string>();
295
-
296
- for (const item of sourceRows) {
297
- if (!isRecord(item)) continue;
298
- const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "tag"));
299
- if (!name || seen.has(name)) continue;
300
- rows.push({
301
- name,
302
- ...(typeof getPath(item, "full_size") === "number" ? { full_size: getPath(item, "full_size") } : {}),
303
- ...(firstNonEmptyString(getPath(item, "last_updated"), getPath(item, "updated_at")) ? {
304
- last_updated: firstNonEmptyString(getPath(item, "last_updated"), getPath(item, "updated_at")),
305
- } : {}),
306
- ...(firstNonEmptyString(getPath(item, "digest")) ? { digest: firstNonEmptyString(getPath(item, "digest")) } : {}),
307
- });
308
- seen.add(name);
309
- }
310
-
311
- return rows;
312
- }
313
-
314
- function normalizeHackerNewsStories(data: unknown): Record<string, unknown>[] {
315
- const sourceRows = isRecord(data) && Array.isArray(data.hits)
316
- ? data.hits
317
- : Array.isArray(data)
318
- ? data
319
- : [];
320
- const rows: Record<string, unknown>[] = [];
321
- const seen = new Set<string>();
322
-
323
- for (const item of sourceRows) {
324
- if (!isRecord(item)) continue;
325
- const id = firstNonEmptyString(getPath(item, "objectID"), getPath(item, "id"), getPath(item, "story_id"));
326
- const title = firstNonEmptyString(getPath(item, "title"), getPath(item, "story_title"));
327
- const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "story_url"), getPath(item, "link"));
328
- const author = firstNonEmptyString(getPath(item, "author"));
329
- const meta = firstNonEmptyString(getPath(item, "meta"));
330
- if (!title) continue;
331
- const stableId = id ?? url ?? title;
332
- if (seen.has(stableId)) continue;
333
- const pointsMatch = meta?.match(/(^|\|)\s*([0-9,]+)\s+points?\b/i);
334
- const commentsMatch = meta?.match(/(^|\|)\s*([0-9,]+)\s+comments?\b/i);
335
-
336
- rows.push({
337
- ...(stableId ? { id: stableId } : {}),
338
- title,
339
- ...(url ? { url } : {}),
340
- ...(author ? { author } : {}),
341
- ...(typeof getPath(item, "points") === "number"
342
- ? { points: getPath(item, "points") }
343
- : pointsMatch ? { points: Number(pointsMatch[2]?.replace(/,/g, "")) } : {}),
344
- ...(typeof getPath(item, "num_comments") === "number"
345
- ? { num_comments: getPath(item, "num_comments") }
346
- : commentsMatch ? { num_comments: Number(commentsMatch[2]?.replace(/,/g, "")) } : {}),
347
- ...(meta ? { meta } : {}),
348
- });
349
- seen.add(stableId);
350
- }
351
-
352
- return rows;
353
- }
354
-
355
- function normalizeStackExchangeQuestions(data: unknown): Record<string, unknown>[] {
356
- const sourceRows = isRecord(data) && Array.isArray(data.items)
357
- ? data.items
358
- : Array.isArray(data)
359
- ? data
360
- : [];
361
- const rows: Record<string, unknown>[] = [];
362
- const seen = new Set<string>();
363
- for (const item of sourceRows) {
364
- if (!isRecord(item)) continue;
365
- const title = firstNonEmptyString(getPath(item, "title"));
366
- const url = firstNonEmptyString(getPath(item, "link"));
367
- if (!title || !url || seen.has(url)) continue;
368
- rows.push({
369
- title,
370
- url,
371
- ...(typeof getPath(item, "score") === "number" ? { score: getPath(item, "score") } : {}),
372
- ...(typeof getPath(item, "answer_count") === "number" ? { answer_count: getPath(item, "answer_count") } : {}),
373
- ...(firstNonEmptyString(getPath(item, "owner.display_name")) ? { author: firstNonEmptyString(getPath(item, "owner.display_name")) } : {}),
374
- ...(typeof getPath(item, "last_activity_date") === "number" ? { date: getPath(item, "last_activity_date") } : {}),
375
- });
376
- seen.add(url);
377
- }
378
- return rows;
379
- }
380
-
381
- function normalizeDevToPosts(data: unknown): Record<string, unknown>[] {
382
- const sourceRows = Array.isArray(data)
383
- ? data
384
- : isRecord(data) && Array.isArray(data.result)
385
- ? data.result
386
- : [];
387
- const rows: Record<string, unknown>[] = [];
388
- const seen = new Set<string>();
389
- const inferAuthor = (value: unknown): string | undefined => {
390
- const raw = hasNonEmptyString(value) ? String(value) : undefined;
391
- if (!raw) return undefined;
392
- try {
393
- const pathname = raw.startsWith("http") ? new URL(raw).pathname : raw;
394
- const segments = pathname.split("/").filter(Boolean);
395
- const candidate = segments[0];
396
- if (!candidate || candidate === "t" || candidate === "s" || candidate === "tag" || candidate === "tags" || candidate === "top" || candidate === "latest") {
397
- return undefined;
398
- }
399
- return candidate;
400
- } catch {
401
- return undefined;
402
- }
403
- };
404
- for (const item of sourceRows) {
405
- if (!isRecord(item)) continue;
406
- const title = firstNonEmptyString(getPath(item, "title"));
407
- const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "path"));
408
- if (!title || !url) continue;
409
- const looksLikeDevTo = (
410
- firstNonEmptyString(getPath(item, "type_of")) === "article" ||
411
- hasNonEmptyString(getPath(item, "user.name")) ||
412
- hasNonEmptyString(getPath(item, "user.username")) ||
413
- typeof getPath(item, "positive_reactions_count") === "number" ||
414
- typeof getPath(item, "comments_count") === "number" ||
415
- hasNonEmptyString(getPath(item, "published_at")) ||
416
- hasNonEmptyString(getPath(item, "readable_publish_date")) ||
417
- url.startsWith("https://dev.to/") ||
418
- url.startsWith("http://dev.to/") ||
419
- (url.startsWith("/") && !!inferAuthor(url))
420
- );
421
- if (!looksLikeDevTo) continue;
422
- const canonicalUrl = url.startsWith("http") ? url : `https://dev.to${url}`;
423
- if (seen.has(canonicalUrl)) continue;
424
- const author = firstNonEmptyString(
425
- getPath(item, "user.name"),
426
- getPath(item, "user.username"),
427
- inferAuthor(getPath(item, "path")),
428
- inferAuthor(canonicalUrl),
429
- );
430
- rows.push({
431
- ...(firstNonEmptyString(getPath(item, "id")) ? { id: firstNonEmptyString(getPath(item, "id")) } : {}),
432
- title,
433
- url: canonicalUrl,
434
- ...(firstNonEmptyString(getPath(item, "description")) ? { description: firstNonEmptyString(getPath(item, "description")) } : {}),
435
- ...(author ? { author } : {}),
436
- ...(typeof getPath(item, "positive_reactions_count") === "number" ? { likes: getPath(item, "positive_reactions_count") } : {}),
437
- ...(typeof getPath(item, "comments_count") === "number" ? { comments: getPath(item, "comments_count") } : {}),
438
- ...(firstNonEmptyString(getPath(item, "published_at"), getPath(item, "readable_publish_date")) ? { date: firstNonEmptyString(getPath(item, "published_at"), getPath(item, "readable_publish_date")) } : {}),
439
- });
440
- seen.add(canonicalUrl);
441
- }
442
- return rows;
443
- }
444
-
445
- function normalizeLobstersPosts(data: unknown): Record<string, unknown>[] {
446
- const sourceRows = Array.isArray(data)
447
- ? data
448
- : isRecord(data) && Array.isArray(data.results)
449
- ? data.results
450
- : [];
451
- const rows: Record<string, unknown>[] = [];
452
- const seen = new Set<string>();
453
- for (const item of sourceRows) {
454
- if (!isRecord(item)) continue;
455
- const title = firstNonEmptyString(getPath(item, "title"));
456
- const rawUrl = firstNonEmptyString(getPath(item, "url"), getPath(item, "link"), getPath(item, "href"));
457
- const text = firstNonEmptyString(getPath(item, "text"), getPath(item, "body"));
458
- if (!title || !rawUrl || /^\d+\s+comments?$/i.test(title)) continue;
459
- const scoreMatch = text?.match(/^\s*(\d{1,4})\s+/)?.[1];
460
- const score = typeof getPath(item, "score") === "number"
461
- ? getPath(item, "score")
462
- : typeof getPath(item, "points") === "number"
463
- ? getPath(item, "points")
464
- : scoreMatch ? Number(scoreMatch) : undefined;
465
- const url = rawUrl.startsWith("http") ? rawUrl : `https://lobste.rs${rawUrl.startsWith("/") ? "" : "/"}${rawUrl}`;
466
- const stable = `${title}|${url}`;
467
- if (seen.has(stable)) continue;
468
- rows.push({
469
- title,
470
- url,
471
- ...(typeof score === "number" ? { score } : {}),
472
- ...(firstNonEmptyString(getPath(item, "author"), getPath(item, "username")) ? { author: firstNonEmptyString(getPath(item, "author"), getPath(item, "username")) } : {}),
473
- });
474
- seen.add(stable);
475
- }
476
- return rows;
477
- }
478
-
479
- function normalizeHuggingFaceModels(data: unknown): Record<string, unknown>[] {
480
- const sourceRows = Array.isArray(data)
481
- ? data
482
- : isRecord(data) && Array.isArray(data.models)
483
- ? data.models
484
- : [];
485
- const rows: Record<string, unknown>[] = [];
486
- const seen = new Set<string>();
487
-
488
- for (const item of sourceRows) {
489
- if (!isRecord(item)) continue;
490
- const id = firstNonEmptyString(getPath(item, "id"), getPath(item, "modelId"), getPath(item, "name"), getPath(item, "title"));
491
- if (!id || seen.has(id)) continue;
492
- const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "title"), getPath(item, "modelId"), getPath(item, "id"));
493
- rows.push({
494
- id,
495
- ...(name ? { name } : {}),
496
- ...(typeof getPath(item, "downloads") === "number" ? { downloads: getPath(item, "downloads") } : {}),
497
- ...(typeof getPath(item, "likes") === "number" ? { likes: getPath(item, "likes") } : {}),
498
- ...(firstNonEmptyString(getPath(item, "pipeline_tag"), getPath(item, "task")) ? {
499
- pipeline_tag: firstNonEmptyString(getPath(item, "pipeline_tag"), getPath(item, "task")),
500
- } : {}),
501
- url: firstNonEmptyString(getPath(item, "url"), getPath(item, "link"))
502
- ?? `https://huggingface.co/${encodeURI(id)}`,
503
- });
504
- seen.add(id);
505
- }
506
-
507
- return rows;
508
- }
509
-
510
- function normalizeEmailRows(data: unknown): Record<string, unknown>[] {
511
- const sourceRows = Array.isArray(data)
512
- ? data
513
- : isRecord(data) && Array.isArray(data.results)
514
- ? data.results
515
- : isRecord(data) && Array.isArray(data.emails)
516
- ? data.emails
517
- : isRecord(data) && Array.isArray(data.messages)
518
- ? data.messages
519
- : [];
520
- const rows: Record<string, unknown>[] = [];
521
- const seen = new Set<string>();
522
-
523
- for (const item of sourceRows) {
524
- if (!isRecord(item)) continue;
525
- const thread = isRecord(getPath(item, "thread")) ? getPath(item, "thread") as Record<string, unknown> : item;
526
- const message = isRecord(getPath(item, "matchedEmail")) ? getPath(item, "matchedEmail") as Record<string, unknown> : item;
527
- const id = firstNonEmptyString(
528
- getPath(message, "id"),
529
- getPath(thread, "doc_id"),
530
- getPath(item, "id"),
531
- getPath(item, "doc_id"),
532
- );
533
- if (!id || seen.has(id)) continue;
534
- const subject = firstNonEmptyString(getPath(thread, "subject"), getPath(item, "subject"), getPath(message, "subject"));
535
- const from = firstNonEmptyString(
536
- getPath(message, "sender"),
537
- getPath(thread, "latest_sender_name"),
538
- getPath(item, "from"),
539
- getPath(item, "sender"),
540
- );
541
- const date = firstNonEmptyString(getPath(thread, "formatted_date"), getPath(item, "date"), getPath(message, "date"));
542
- const preview = firstNonEmptyString(
543
- getPath(thread, "preview"),
544
- getPath(message, "content_markdown"),
545
- getPath(item, "preview"),
546
- getPath(item, "snippet"),
547
- );
548
- if (!subject && !from) continue;
549
- rows.push({
550
- id,
551
- ...(subject ? { subject } : {}),
552
- ...(from ? { from } : {}),
553
- ...(date ? { date } : {}),
554
- ...(preview ? { preview } : {}),
555
- });
556
- seen.add(id);
557
- }
558
-
559
- return rows;
560
- }
561
-
562
- function normalizeProductRows(data: unknown): Record<string, unknown>[] {
563
- const rows: Record<string, unknown>[] = [];
564
- const seen = new Set<string>();
565
- const blockedTitles = new Set(["results", "more results", "related searches", "need help?"]);
566
-
567
- collectNestedObjects(data, (obj) => {
568
- const headingKeys = Object.keys(obj).filter((key) => /^heading_\d+$/i.test(key));
569
- const title = firstNonEmptyString(
570
- getPath(obj, "title"),
571
- getPath(obj, "name"),
572
- getPath(obj, "productName"),
573
- getPath(obj, "productTitle"),
574
- getPath(obj, "title.text"),
575
- );
576
- const url = firstNonEmptyString(
577
- getPath(obj, "url"),
578
- getPath(obj, "link"),
579
- getPath(obj, "href"),
580
- getPath(obj, "canonicalUrl"),
581
- getPath(obj, "productUrl"),
582
- getPath(obj, "productPageUrl"),
583
- );
584
- const id = firstNonEmptyString(
585
- getPath(obj, "id"),
586
- getPath(obj, "usItemId"),
587
- getPath(obj, "itemId"),
588
- getPath(obj, "productId"),
589
- getPath(obj, "sku"),
590
- );
591
- const priceString = firstNonEmptyString(
592
- getPath(obj, "price"),
593
- getPath(obj, "priceString"),
594
- getPath(obj, "price.currentPrice.priceString"),
595
- getPath(obj, "priceInfo.currentPrice.priceString"),
596
- getPath(obj, "secondaryOfferPrice.currentPrice.priceString"),
597
- );
598
- const priceNumber = getPath(obj, "price.currentPrice.price")
599
- ?? getPath(obj, "priceInfo.currentPrice.price")
600
- ?? getPath(obj, "currentPrice.price")
601
- ?? getPath(obj, "salePrice");
602
- const rating = getPath(obj, "averageRating")
603
- ?? getPath(obj, "rating")
604
- ?? getPath(obj, "rating.average")
605
- ?? getPath(obj, "customerRating")
606
- ?? getPath(obj, "reviews.averageRating");
607
- const reviewCount = getPath(obj, "numberOfReviews")
608
- ?? getPath(obj, "reviewCount")
609
- ?? getPath(obj, "reviews.count")
610
- ?? getPath(obj, "ratingsTotal");
611
- const image = firstNonEmptyString(
612
- getPath(obj, "image"),
613
- getPath(obj, "imageUrl"),
614
- getPath(obj, "thumbnail"),
615
- getPath(obj, "primaryImageUrl"),
616
- );
617
- const brand = firstNonEmptyString(getPath(obj, "brand"), getPath(obj, "brandName"), getPath(obj, "seller"));
618
-
619
- if (!title && !id) return;
620
- if (title && blockedTitles.has(title.trim().toLowerCase())) return;
621
- if (headingKeys.length >= 6) return;
622
- if (!url && !id) return;
623
- if (
624
- priceString == null &&
625
- typeof priceNumber !== "number" &&
626
- rating == null &&
627
- reviewCount == null &&
628
- !image &&
629
- !brand
630
- ) {
631
- return;
632
- }
633
- if (url && /(amazon-adsystem|doubleclick|googlesyndication|pubmatic|taboola|outbrain|aax-|googleadservices|adservice)\./i.test(url)) {
634
- return;
635
- }
636
-
637
- const stable = String(url ?? id ?? title);
638
- if (seen.has(stable)) return;
639
- rows.push({
640
- ...(id ? { id } : {}),
641
- ...(title ? { title } : {}),
642
- ...(url ? { url } : {}),
643
- ...(priceString ? { price: priceString } : typeof priceNumber === "number" ? { price: priceNumber } : {}),
644
- ...(typeof rating === "number" || hasNonEmptyString(rating) ? { rating } : {}),
645
- ...(typeof reviewCount === "number" || hasNonEmptyString(reviewCount) ? { review_count: reviewCount } : {}),
646
- ...(image ? { image } : {}),
647
- ...(brand ? { brand } : {}),
648
- });
649
- seen.add(stable);
650
- });
651
-
652
- return rows;
653
- }
654
-
655
- function normalizeStockQuote(data: unknown): Record<string, unknown> | null {
656
- let best: Record<string, unknown> | null = null;
657
-
658
- collectNestedObjects(data, (obj) => {
659
- const symbol = firstNonEmptyString(
660
- getPath(obj, "symbol"),
661
- getPath(obj, "ticker"),
662
- getPath(obj, "quote.symbol"),
663
- );
664
- const price = getPath(obj, "regularMarketPrice.raw")
665
- ?? getPath(obj, "regularMarketPrice")
666
- ?? getPath(obj, "postMarketPrice.raw")
667
- ?? getPath(obj, "price.regularMarketPrice.raw")
668
- ?? getPath(obj, "price.regularMarketPrice")
669
- ?? getPath(obj, "price.currentPrice.raw")
670
- ?? getPath(obj, "price.currentPrice");
671
- if (!symbol || typeof price !== "number") return;
672
- const name = firstNonEmptyString(
673
- getPath(obj, "shortName"),
674
- getPath(obj, "longName"),
675
- getPath(obj, "displayName"),
676
- getPath(obj, "price.shortName"),
677
- getPath(obj, "price.longName"),
678
- );
679
- const currency = firstNonEmptyString(getPath(obj, "currency"), getPath(obj, "financialCurrency"), getPath(obj, "price.currency"));
680
- const marketState = firstNonEmptyString(getPath(obj, "marketState"), getPath(obj, "price.marketState"));
681
- const changePercent = getPath(obj, "regularMarketChangePercent.raw")
682
- ?? getPath(obj, "regularMarketChangePercent")
683
- ?? getPath(obj, "price.regularMarketChangePercent.raw")
684
- ?? getPath(obj, "price.regularMarketChangePercent");
685
- const marketCap = getPath(obj, "marketCap.raw")
686
- ?? getPath(obj, "marketCap")
687
- ?? getPath(obj, "price.marketCap.raw")
688
- ?? getPath(obj, "price.marketCap");
689
- best = {
690
- symbol,
691
- ...(name ? { name } : {}),
692
- price,
693
- ...(currency ? { currency } : {}),
694
- ...(typeof changePercent === "number" ? { change_percent: changePercent } : {}),
695
- ...(typeof marketCap === "number" ? { market_cap: marketCap } : {}),
696
- ...(marketState ? { market_state: marketState } : {}),
697
- };
698
- });
699
-
700
- return best;
701
- }
702
-
703
- function normalizeChannelRows(data: unknown): Record<string, unknown>[] {
704
- const rows: Record<string, unknown>[] = [];
705
- const seen = new Set<string>();
706
-
707
- collectNestedObjects(data, (obj) => {
708
- const name = firstNonEmptyString(
709
- getPath(obj, "name"),
710
- getPath(obj, "title"),
711
- getPath(obj, "channel_name"),
712
- getPath(obj, "workspace_name"),
713
- getPath(obj, "team_name"),
714
- );
715
- const id = firstNonEmptyString(
716
- getPath(obj, "id"),
717
- getPath(obj, "channel_id"),
718
- getPath(obj, "guild_id"),
719
- getPath(obj, "server_id"),
720
- getPath(obj, "workspace_id"),
721
- getPath(obj, "team_id"),
722
- );
723
- const url = firstNonEmptyString(
724
- getPath(obj, "url"),
725
- getPath(obj, "channel_url"),
726
- getPath(obj, "server_url"),
727
- getPath(obj, "workspace_url"),
728
- );
729
- if (!name || (!id && !url)) return;
730
- const stable = String(id ?? url ?? name);
731
- if (seen.has(stable)) return;
732
- rows.push({
733
- ...(id ? { id } : {}),
734
- name,
735
- ...(url ? { url } : {}),
736
- ...(firstNonEmptyString(getPath(obj, "topic"), getPath(obj, "purpose"), getPath(obj, "description")) ? {
737
- description: firstNonEmptyString(getPath(obj, "topic"), getPath(obj, "purpose"), getPath(obj, "description")),
738
- } : {}),
739
- ...(typeof getPath(obj, "member_count") === "number" ? { member_count: getPath(obj, "member_count") } : {}),
740
- ...(firstNonEmptyString(getPath(obj, "type"), getPath(obj, "channel_type"), getPath(obj, "kind")) ? {
741
- type: firstNonEmptyString(getPath(obj, "type"), getPath(obj, "channel_type"), getPath(obj, "kind")),
742
- } : {}),
743
- });
744
- seen.add(stable);
745
- });
746
-
747
- return rows;
748
- }
749
-
750
- function collectNestedObjects(value: unknown, visit: (obj: Record<string, unknown>) => void): void {
751
- if (Array.isArray(value)) {
752
- for (const item of value) collectNestedObjects(item, visit);
753
- return;
754
- }
755
- if (!isRecord(value)) return;
756
- visit(value);
757
- for (const child of Object.values(value)) collectNestedObjects(child, visit);
758
- }
759
-
760
- function normalizeXTweets(data: unknown): Record<string, unknown>[] {
761
- const rows: Record<string, unknown>[] = [];
762
- const seen = new Set<string>();
763
- collectNestedObjects(data, (obj) => {
764
- const result = getPath(obj, "tweet_results.result");
765
- if (!isRecord(result)) return;
766
- const restId = getPath(result, "rest_id");
767
- const text = getPath(result, "legacy.full_text");
768
- const screenName = getPath(result, "core.user_results.result.core.screen_name")
769
- ?? getPath(result, "core.user_results.result.legacy.screen_name");
770
- if (!hasNonEmptyString(restId) || !hasNonEmptyString(text) || !hasNonEmptyString(screenName)) return;
771
- if (seen.has(String(restId))) return;
772
- rows.push({
773
- id: String(restId),
774
- username: String(screenName),
775
- text: String(text),
776
- url: `https://x.com/${encodeURIComponent(String(screenName))}/status/${encodeURIComponent(String(restId))}`,
777
- });
778
- seen.add(String(restId));
779
- });
780
- return rows;
781
- }
782
-
783
- function normalizeXUsers(data: unknown): Record<string, unknown>[] {
784
- const rows: Record<string, unknown>[] = [];
785
- const seen = new Set<string>();
786
- collectNestedObjects(data, (obj) => {
787
- const result = getPath(obj, "user_results.result");
788
- const target = isRecord(result) ? result : obj;
789
- const screenName = getPath(target, "core.screen_name") ?? getPath(target, "legacy.screen_name");
790
- const name = getPath(target, "core.name") ?? getPath(target, "legacy.name");
791
- const description = getPath(target, "legacy.description");
792
- const followers = getPath(target, "legacy.followers_count");
793
- if (!hasNonEmptyString(screenName) || !hasNonEmptyString(name)) return;
794
- if (seen.has(String(screenName))) return;
795
- rows.push({
796
- username: String(screenName),
797
- public_identifier: String(screenName),
798
- name: String(name),
799
- ...(hasNonEmptyString(description) ? { description: String(description) } : {}),
800
- ...(typeof followers === "number" ? { followers_count: followers } : {}),
801
- url: `https://x.com/${encodeURIComponent(String(screenName))}`,
802
- });
803
- seen.add(String(screenName));
804
- });
805
- return rows;
806
- }
807
-
808
- function joinNameParts(...values: unknown[]): string | undefined {
809
- const parts = values.filter((value) => hasNonEmptyString(value)).map((value) => String(value).trim());
810
- return parts.length > 0 ? parts.join(" ") : undefined;
811
- }
812
-
813
- function absoluteLinkedInUrl(value: string | undefined): string | undefined {
814
- if (!value) return undefined;
815
- return value.startsWith("http") ? value : `https://www.linkedin.com${value.startsWith("/") ? "" : "/"}${value}`;
816
- }
817
-
818
- function normalizeLinkedInFeedPosts(data: unknown): Record<string, unknown>[] {
819
- if (!isRecord(data)) return [];
820
-
821
- const included = Array.isArray(getPath(data, "included"))
822
- ? getPath(data, "included") as unknown[]
823
- : Array.isArray(getPath(data, "data.included"))
824
- ? getPath(data, "data.included") as unknown[]
825
- : [];
826
- if (included.length === 0) return [];
827
-
828
- const elementRefs = Array.isArray(getPath(data, "data.data.feedDashMainFeedByMainFeed.*elements"))
829
- ? getPath(data, "data.data.feedDashMainFeedByMainFeed.*elements") as unknown[]
830
- : Array.isArray(getPath(data, "data.feedDashMainFeedByMainFeed.*elements"))
831
- ? getPath(data, "data.feedDashMainFeedByMainFeed.*elements") as unknown[]
832
- : Array.isArray(getPath(data, "feedDashMainFeedByMainFeed.*elements"))
833
- ? getPath(data, "feedDashMainFeedByMainFeed.*elements") as unknown[]
834
- : [];
835
- if (elementRefs.length === 0) return [];
836
-
837
- const entityIndex = new Map<string, Record<string, unknown>>();
838
- for (const item of included) {
839
- if (!isRecord(item)) continue;
840
- const urn = getPath(item, "entityUrn");
841
- if (!hasNonEmptyString(urn)) continue;
842
- entityIndex.set(String(urn), item);
843
- }
844
-
845
- const rows: Record<string, unknown>[] = [];
846
- const seen = new Set<string>();
847
-
848
- for (const ref of elementRefs) {
849
- if (!hasNonEmptyString(ref) || seen.has(String(ref))) continue;
850
- const update = entityIndex.get(String(ref));
851
- if (!update) continue;
852
-
853
- const actorProfileUrn = firstNonEmptyString(getPath(update, "actor.*profileUrn"), getPath(update, "actor.entityUrn"));
854
- const actorProfile = actorProfileUrn ? entityIndex.get(actorProfileUrn) : undefined;
855
- const author = firstNonEmptyString(
856
- getPath(update, "actor.name.text"),
857
- getPath(actorProfile, "name"),
858
- joinNameParts(getPath(actorProfile, "firstName"), getPath(actorProfile, "lastName")),
859
- );
860
- const username = firstNonEmptyString(getPath(actorProfile, "publicIdentifier"));
861
- const content = firstNonEmptyString(
862
- getPath(update, "commentary.text.text"),
863
- getPath(update, "commentary.accessibilityText"),
864
- getPath(update, "content.text"),
865
- getPath(update, "content.title"),
866
- getPath(update, "header.text"),
867
- getPath(update, "header.title"),
868
- );
869
- const permalink = firstNonEmptyString(
870
- getPath(update, "permalink"),
871
- getPath(update, "socialContent.shareUrl"),
872
- getPath(update, "url"),
873
- );
874
- const url = absoluteLinkedInUrl(permalink) ?? `https://www.linkedin.com/feed/update/${encodeURIComponent(String(ref))}/`;
875
-
876
- if (!content && !username && !author) continue;
877
-
878
- rows.push({
879
- id: String(ref),
880
- url,
881
- ...(content ? { content } : {}),
882
- ...(author ? { author } : {}),
883
- ...(username ? { username, public_identifier: username } : {}),
884
- ...(typeof getPath(update, "createdAt") === "number" ? { created_at: getPath(update, "createdAt") } : {}),
885
- });
886
- seen.add(String(ref));
887
- }
888
-
889
- return rows;
890
- }
891
-
892
- function normalizeGenericPeopleRows(data: unknown): Record<string, unknown>[] {
893
- const sourceRows = Array.isArray(data)
894
- ? data
895
- : isRecord(data) && Array.isArray(data.results)
896
- ? data.results
897
- : [];
898
- const rows: Record<string, unknown>[] = [];
899
- const seen = new Set<string>();
900
-
901
- for (const item of sourceRows) {
902
- if (!isRecord(item)) continue;
903
- if (
904
- hasAnyPath(item, ["full_name", "repository.name", "path_with_namespace", "stargazers_count", "stars", "star_count", "forks_count", "version", "price"])
905
- ) {
906
- continue;
907
- }
908
- const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "title"));
909
- const headline = firstNonEmptyString(
910
- getPath(item, "headline"),
911
- getPath(item, "description"),
912
- getPath(item, "subtitle"),
913
- getPath(item, "bio"),
914
- );
915
- const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "link"));
916
- const publicIdentifier = firstNonEmptyString(getPath(item, "public_identifier"), getPath(item, "username"));
917
- if (!name || name.includes("/") || name.includes("://")) continue;
918
- if (!headline && !url && !publicIdentifier) continue;
919
- const id = publicIdentifier ?? url ?? name;
920
- if (seen.has(id)) continue;
921
- rows.push({
922
- name,
923
- ...(headline ? { headline } : {}),
924
- ...(url ? { url } : {}),
925
- ...(publicIdentifier ? { public_identifier: publicIdentifier } : {}),
926
- });
927
- seen.add(id);
928
- }
929
-
930
- return rows;
931
- }
932
-
933
- function normalizeRedditPosts(data: unknown): Record<string, unknown>[] {
934
- const rows: Record<string, unknown>[] = [];
935
- const seen = new Set<string>();
936
- collectNestedObjects(data, (obj) => {
937
- const kind = getPath(obj, "kind");
938
- const payload = isRecord(getPath(obj, "data")) ? getPath(obj, "data") : obj;
939
- if (!isRecord(payload)) return;
940
- const title = getPath(payload, "title");
941
- const author = getPath(payload, "author");
942
- const permalink = getPath(payload, "permalink");
943
- if (!hasNonEmptyString(title) || !hasNonEmptyString(author) || !hasNonEmptyString(permalink)) return;
944
- if (hasNonEmptyString(kind) && kind !== "t3") return;
945
- const id = getPath(payload, "id") ?? getPath(payload, "name") ?? permalink;
946
- if (!hasNonEmptyString(id) || seen.has(String(id))) return;
947
- rows.push({
948
- id: String(id),
949
- title: String(title),
950
- author: String(author),
951
- ...(typeof getPath(payload, "score") === "number" ? { score: getPath(payload, "score") } : {}),
952
- ...(typeof getPath(payload, "num_comments") === "number" ? { num_comments: getPath(payload, "num_comments") } : {}),
953
- ...(hasNonEmptyString(getPath(payload, "subreddit")) ? { subreddit: String(getPath(payload, "subreddit")) } : {}),
954
- permalink: String(permalink),
955
- url: `https://www.reddit.com${String(permalink)}`,
956
- });
957
- seen.add(String(id));
958
- });
959
- return rows;
960
- }
961
-
962
- function normalizeRedditComments(data: unknown): Record<string, unknown>[] {
963
- const rows: Record<string, unknown>[] = [];
964
- const seen = new Set<string>();
965
- collectNestedObjects(data, (obj) => {
966
- const kind = getPath(obj, "kind");
967
- const payload = isRecord(getPath(obj, "data")) ? getPath(obj, "data") : obj;
968
- if (!isRecord(payload)) return;
969
- const body = getPath(payload, "body");
970
- const author = getPath(payload, "author");
971
- const permalink = getPath(payload, "permalink");
972
- if (!hasNonEmptyString(body) || !hasNonEmptyString(author) || !hasNonEmptyString(permalink)) return;
973
- if (hasNonEmptyString(kind) && kind !== "t1") return;
974
- const id = getPath(payload, "id") ?? getPath(payload, "name") ?? permalink;
975
- if (!hasNonEmptyString(id) || seen.has(String(id))) return;
976
- rows.push({
977
- id: String(id),
978
- author: String(author),
979
- body: String(body),
980
- permalink: String(permalink),
981
- url: `https://www.reddit.com${String(permalink)}`,
982
- });
983
- seen.add(String(id));
984
- });
985
- return rows;
986
- }
987
-
988
- function normalizeCompanies(data: unknown): Record<string, unknown>[] {
989
- const rows: Record<string, unknown>[] = [];
990
- const seen = new Set<string>();
991
- collectNestedObjects(data, (obj) => {
992
- const name = getPath(obj, "name") ?? getPath(obj, "title");
993
- const publicIdentifier =
994
- getPath(obj, "public_identifier") ??
995
- getPath(obj, "vanityName") ??
996
- getPath(obj, "universalName");
997
- const description =
998
- getPath(obj, "description") ??
999
- getPath(obj, "tagline") ??
1000
- getPath(obj, "headline");
1001
- const industry = getPath(obj, "industry") ?? getPath(obj, "primaryIndustry");
1002
- const employeeCount =
1003
- getPath(obj, "employee_count") ??
1004
- getPath(obj, "staffCount") ??
1005
- getPath(obj, "employeeCount");
1006
- const followerCount =
1007
- getPath(obj, "follower_count") ??
1008
- getPath(obj, "followerCount");
1009
- const website = getPath(obj, "website") ?? getPath(obj, "websiteUrl");
1010
- const url =
1011
- getPath(obj, "url") ??
1012
- (hasNonEmptyString(publicIdentifier)
1013
- ? `https://www.linkedin.com/company/${encodeURIComponent(String(publicIdentifier))}/`
1014
- : undefined);
1015
- if (!hasNonEmptyString(name)) return;
1016
- if (
1017
- !hasNonEmptyString(description) &&
1018
- !hasNonEmptyString(industry) &&
1019
- typeof employeeCount !== "number" &&
1020
- typeof followerCount !== "number" &&
1021
- !hasNonEmptyString(website) &&
1022
- !hasNonEmptyString(url) &&
1023
- !hasNonEmptyString(publicIdentifier)
1024
- ) {
1025
- return;
1026
- }
1027
- const id = String(publicIdentifier ?? url ?? name);
1028
- if (seen.has(id)) return;
1029
- rows.push({
1030
- name: String(name),
1031
- ...(hasNonEmptyString(publicIdentifier) ? { public_identifier: String(publicIdentifier) } : {}),
1032
- ...(hasNonEmptyString(description) ? { description: String(description) } : {}),
1033
- ...(hasNonEmptyString(industry) ? { industry: String(industry) } : {}),
1034
- ...(typeof employeeCount === "number" ? { employee_count: employeeCount } : {}),
1035
- ...(typeof followerCount === "number" ? { follower_count: followerCount } : {}),
1036
- ...(hasNonEmptyString(website) ? { website: String(website) } : {}),
1037
- ...(hasNonEmptyString(url) ? { url: String(url) } : {}),
1038
- });
1039
- seen.add(id);
1040
- });
1041
- return rows;
1042
- }
1043
-
1044
- function unwrapCarrier(data: unknown): unknown {
1045
- if (!isRecord(data)) return data;
1046
- const keys = Object.keys(data);
1047
- const isCarrierOnly = keys.every((key) => key === "data" || key === "_extraction");
1048
- if (isCarrierOnly && "data" in data && ("_extraction" in data || Array.isArray(data.data) || isRecord(data.data))) {
1049
- return unwrapCarrier(data.data);
1050
- }
1051
- return data;
1052
- }
1053
-
1054
- export function projectIntentData(data: unknown, intent?: string): unknown {
1055
- const unwrapped = unwrapCarrier(data);
1056
- if (!intent) return unwrapped;
1057
- const lower = intent.toLowerCase();
1058
-
1059
- if (/\b(stock|stocks|ticker|tickers|quote|quotes)\b/.test(lower)) {
1060
- const normalizedQuote = normalizeStockQuote(unwrapped);
1061
- if (normalizedQuote) return normalizedQuote;
1062
- }
1063
-
1064
- if (/\b(product|products|item|items)\b/.test(lower)) {
1065
- const normalizedProducts = normalizeProductRows(unwrapped);
1066
- if (normalizedProducts.length > 0) return normalizedProducts;
1067
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).products)) {
1068
- return (unwrapped as Record<string, unknown>).products;
1069
- }
1070
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).items)) {
1071
- return (unwrapped as Record<string, unknown>).items;
1072
- }
1073
- }
1074
-
1075
- if (/\b(channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(lower)) {
1076
- const normalizedChannels = normalizeChannelRows(unwrapped);
1077
- if (normalizedChannels.length > 0) return normalizedChannels;
1078
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).channels)) {
1079
- return (unwrapped as Record<string, unknown>).channels;
1080
- }
1081
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).guilds)) {
1082
- return (unwrapped as Record<string, unknown>).guilds;
1083
- }
1084
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).workspaces)) {
1085
- return (unwrapped as Record<string, unknown>).workspaces;
1086
- }
1087
- }
1088
-
1089
- if (/\b(package|packages)\b/.test(lower)) {
1090
- if (/\bsearch\b/.test(lower)) {
1091
- const normalizedCratesSearch = normalizeCratesPackageSearchResults(unwrapped);
1092
- if (normalizedCratesSearch.length > 0) return normalizedCratesSearch;
1093
- const normalizedPackageSearch = normalizePackageSearchResults(unwrapped);
1094
- if (normalizedPackageSearch.length > 0) return normalizedPackageSearch;
1095
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).packages)) {
1096
- return (unwrapped as Record<string, unknown>).packages;
1097
- }
1098
- } else {
1099
- const normalizedNpmPackage = normalizeNpmPackageInfo(unwrapped);
1100
- if (normalizedNpmPackage) return normalizedNpmPackage;
1101
- const normalizedPyPIPackage = normalizePyPIPackageInfo(unwrapped);
1102
- if (normalizedPyPIPackage) return normalizedPyPIPackage;
1103
- const normalizedPubDevPackage = normalizePubDevPackageInfo(unwrapped);
1104
- if (normalizedPubDevPackage) return normalizedPubDevPackage;
1105
- const normalizedRubyGemsPackage = normalizeRubyGemsPackageInfo(unwrapped);
1106
- if (normalizedRubyGemsPackage) return normalizedRubyGemsPackage;
1107
- if (isRecord(unwrapped) && isRecord((unwrapped as Record<string, unknown>).package)) {
1108
- return (unwrapped as Record<string, unknown>).package;
1109
- }
1110
- }
1111
- }
1112
-
1113
- if (/\b(doc|docs|documentation)\b/.test(lower)) {
1114
- const normalizedDocuments = normalizeDocumentRows(unwrapped);
1115
- if (normalizedDocuments.length > 0) return normalizedDocuments;
1116
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).documents)) {
1117
- return (unwrapped as Record<string, unknown>).documents;
1118
- }
1119
- }
1120
-
1121
- if (/\bsearch images\b/.test(lower)) {
1122
- const normalizedImageSearch = normalizeDockerImageSearchResults(unwrapped);
1123
- if (normalizedImageSearch.length > 0) return normalizedImageSearch;
1124
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).images)) {
1125
- return (unwrapped as Record<string, unknown>).images;
1126
- }
1127
- }
1128
-
1129
- if (/\b(get )?image tags\b/.test(lower)) {
1130
- const normalizedImageTags = normalizeDockerTagResults(unwrapped);
1131
- if (normalizedImageTags.length > 0) return normalizedImageTags;
1132
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).tags)) {
1133
- return (unwrapped as Record<string, unknown>).tags;
1134
- }
1135
- }
1136
-
1137
- if (/\b(comment|comments)\b/.test(lower)) {
1138
- const normalizedRedditComments = normalizeRedditComments(unwrapped);
1139
- if (normalizedRedditComments.length > 0) return normalizedRedditComments;
1140
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).comments)) {
1141
- return (unwrapped as Record<string, unknown>).comments;
1142
- }
1143
- }
1144
-
1145
- if (/\b(reddit|subreddit)\b/.test(lower)) {
1146
- const normalizedRedditPosts = normalizeRedditPosts(unwrapped);
1147
- if (normalizedRedditPosts.length > 0) return normalizedRedditPosts;
1148
- }
1149
-
1150
- if (/\b(company|companies|organization|organisations|business)\b/.test(lower)) {
1151
- const normalizedCompanies = normalizeCompanies(unwrapped);
1152
- if (normalizedCompanies.length === 1) return normalizedCompanies[0];
1153
- if (normalizedCompanies.length > 1) return normalizedCompanies;
1154
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).companies)) {
1155
- return (unwrapped as Record<string, unknown>).companies;
1156
- }
1157
- }
1158
-
1159
- if (!isRecord(unwrapped) && !Array.isArray(unwrapped)) return unwrapped;
1160
-
1161
- if (/\b(post|posts|tweet|tweets|status|statuses)\b/.test(lower)) {
1162
- const normalizedDevPosts = normalizeDevToPosts(unwrapped);
1163
- if (normalizedDevPosts.length > 0) return normalizedDevPosts;
1164
- const normalizedLobsters = normalizeLobstersPosts(unwrapped);
1165
- if (normalizedLobsters.length > 0) return normalizedLobsters;
1166
- const normalizedLinkedInFeed = normalizeLinkedInFeedPosts(unwrapped);
1167
- if (normalizedLinkedInFeed.length > 0) return normalizedLinkedInFeed;
1168
- const normalizedRedditPosts = normalizeRedditPosts(unwrapped);
1169
- if (normalizedRedditPosts.length > 0) return normalizedRedditPosts;
1170
- const normalizedXTweets = normalizeXTweets(unwrapped);
1171
- if (normalizedXTweets.length > 0) return normalizedXTweets;
1172
- if (Array.isArray(unwrapped)) return unwrapped;
1173
- if (Array.isArray(unwrapped.statuses)) return unwrapped.statuses;
1174
- if (Array.isArray(unwrapped.posts)) return unwrapped.posts;
1175
- if (Array.isArray(unwrapped.tweets)) return unwrapped.tweets;
1176
- }
1177
-
1178
- if (/\b(person|people|user|users|profile|profiles|member|members)\b/.test(lower)) {
1179
- if (Array.isArray(unwrapped.people)) return unwrapped.people;
1180
- if (Array.isArray(unwrapped.users)) return unwrapped.users;
1181
- if (Array.isArray(unwrapped.accounts)) return unwrapped.accounts;
1182
- if (Array.isArray(unwrapped.elements)) return unwrapped.elements;
1183
- if (Array.isArray(unwrapped.included)) return unwrapped.included;
1184
- const normalizedXUsers = normalizeXUsers(unwrapped);
1185
- if (normalizedXUsers.length > 0) return normalizedXUsers;
1186
- const normalizedGenericPeople = normalizeGenericPeopleRows(unwrapped);
1187
- if (normalizedGenericPeople.length > 0) return normalizedGenericPeople;
1188
- }
1189
-
1190
- if (/\b(topic|topics|trend|trending|hashtag|hashtags)\b/.test(lower)) {
1191
- const storyItems = getPath(unwrapped, "story_topic.stories.items");
1192
- if (Array.isArray(storyItems)) {
1193
- const normalized = storyItems
1194
- .map((item) => {
1195
- const core = getPath(item, "trend_results.result.core");
1196
- const name = getPath(core, "name");
1197
- const restId = getPath(item, "trend_results.result.rest_id");
1198
- if (!hasNonEmptyString(name)) return null;
1199
- return {
1200
- name,
1201
- ...(hasNonEmptyString(getPath(core, "category")) ? { category: getPath(core, "category") } : {}),
1202
- ...(hasNonEmptyString(getPath(item, "trend_results.result.post_count")) ? { post_count: getPath(item, "trend_results.result.post_count") } : {}),
1203
- ...(hasNonEmptyString(restId)
1204
- ? { url: `https://x.com/search?q=${encodeURIComponent(String(name))}&src=trend_click&trend_id=${encodeURIComponent(String(restId))}` }
1205
- : { url: `https://x.com/search?q=${encodeURIComponent(String(name))}` }),
1206
- };
1207
- })
1208
- .filter((item): item is Record<string, unknown> => !!item);
1209
- if (normalized.length > 0) return normalized;
1210
- }
1211
- if (Array.isArray(unwrapped.topics)) return unwrapped.topics;
1212
- if (Array.isArray(unwrapped.trends)) return unwrapped.trends;
1213
- if (Array.isArray(unwrapped.hashtags)) return unwrapped.hashtags;
1214
- }
1215
-
1216
- if (/\b(model|models)\b/.test(lower)) {
1217
- const normalizedModels = normalizeHuggingFaceModels(unwrapped);
1218
- if (normalizedModels.length > 0) return normalizedModels;
1219
- if (Array.isArray(unwrapped.models)) return unwrapped.models;
1220
- }
1221
-
1222
- if (/\b(news|story|stories|article|articles|hacker news)\b/.test(lower)) {
1223
- if (Array.isArray(unwrapped)) return unwrapped;
1224
- const normalizedStories = normalizeHackerNewsStories(unwrapped);
1225
- if (normalizedStories.length > 0) return normalizedStories;
1226
- if (Array.isArray(unwrapped.hits)) return unwrapped.hits;
1227
- if (Array.isArray(unwrapped.stories)) return unwrapped.stories;
1228
- if (Array.isArray(unwrapped.articles)) return unwrapped.articles;
1229
- }
1230
-
1231
- if (/\b(question|questions)\b/.test(lower)) {
1232
- if (Array.isArray(unwrapped)) return unwrapped;
1233
- const normalizedQuestions = normalizeStackExchangeQuestions(unwrapped);
1234
- if (normalizedQuestions.length > 0) return normalizedQuestions;
1235
- if (Array.isArray(unwrapped.items)) return unwrapped.items;
1236
- }
1237
-
1238
- if (/\b(email|emails|mail|inbox)\b/.test(lower)) {
1239
- const normalizedEmails = normalizeEmailRows(unwrapped);
1240
- if (normalizedEmails.length > 0) return normalizedEmails;
1241
- if (Array.isArray(unwrapped.emails)) return unwrapped.emails;
1242
- if (Array.isArray(unwrapped.messages)) return unwrapped.messages;
1243
- if (Array.isArray(unwrapped.results)) return unwrapped.results;
1244
- }
1245
-
1246
- if (/\b(repo|repository|repositories|project|projects)\b/.test(lower)) {
1247
- if (Array.isArray(unwrapped.repositories)) return unwrapped.repositories;
1248
- if (Array.isArray(unwrapped.items)) return unwrapped.items;
1249
- if (Array.isArray(unwrapped.projects)) return unwrapped.projects;
1250
- }
1251
-
1252
- return unwrapped;
1253
- }
1254
-
1255
- function classifyRows(rows: unknown[], intent: string): { verdict: "pass" | "fail" | "skip"; reason: string } {
1256
- if (rows.length === 0) return { verdict: "fail", reason: "empty_array" };
1257
- if (rows.every((row) => !isRecord(row))) return { verdict: "fail", reason: "primitive_rows" };
1258
-
1259
- const objects = rows.filter(isRecord);
1260
- if (objects.length === 0) return { verdict: "fail", reason: "primitive_rows" };
1261
-
1262
- const lower = intent.toLowerCase();
1263
-
1264
- if (/\b(repo|repository|repositories|project|projects)\b/.test(lower)) {
1265
- const matching = objects.filter((row) =>
1266
- hasAnyPath(row, ["full_name", "name", "repository.name", "path_with_namespace"]) &&
1267
- hasAnyPath(row, ["url", "web_url", "description", "stargazers_count", "stars", "star_count", "forks_count", "owner.login", "owner"])
1268
- );
1269
- return matching.length >= 1 ? { verdict: "pass", reason: "repository_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1270
- }
1271
-
1272
- if (/\b(package|packages)\b/.test(lower)) {
1273
- const matching = objects.filter((row) =>
1274
- hasAnyPath(row, ["name", "package.name"]) &&
1275
- hasAnyPath(row, ["version", "description", "summary", "url", "keywords", "requires_dist", "dependencies"])
1276
- );
1277
- return matching.length >= 1 ? { verdict: "pass", reason: "package_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1278
- }
1279
-
1280
- if (/\b(model|models)\b/.test(lower)) {
1281
- const matching = objects.filter((row) =>
1282
- hasAnyPath(row, ["id", "modelId", "name"]) &&
1283
- hasAnyPath(row, ["downloads", "likes", "pipeline_tag", "url"])
1284
- );
1285
- return matching.length >= 1 ? { verdict: "pass", reason: "model_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1286
- }
1287
-
1288
- if (/\b(news|story|stories|article|articles|hacker news)\b/.test(lower)) {
1289
- const matching = objects.filter((row) =>
1290
- hasAnyPath(row, ["objectID", "id", "url", "link"]) &&
1291
- hasAnyPath(row, ["title", "story_title", "author", "points", "num_comments", "meta"])
1292
- );
1293
- return matching.length >= 1 ? { verdict: "pass", reason: "story_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1294
- }
1295
-
1296
- if (/\bsearch images\b/.test(lower)) {
1297
- const matching = objects.filter((row) =>
1298
- hasAnyPath(row, ["repo_name", "name", "full_name"]) &&
1299
- hasAnyPath(row, ["short_description", "description", "star_count", "pull_count", "url"])
1300
- );
1301
- return matching.length >= 1 ? { verdict: "pass", reason: "image_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1302
- }
1303
-
1304
- if (/\b(get )?image tags\b/.test(lower)) {
1305
- const matching = objects.filter((row) =>
1306
- hasAnyPath(row, ["name", "tag"]) &&
1307
- hasAnyPath(row, ["last_updated", "updated_at", "full_size", "digest"])
1308
- );
1309
- return matching.length >= 1 ? { verdict: "pass", reason: "tag_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1310
- }
1311
-
1312
- if (/\b(company|companies|organization|organisations|business)\b/.test(lower)) {
1313
- const matching = objects.filter((row) =>
1314
- hasAnyPath(row, ["name", "title", "public_identifier"]) &&
1315
- hasAnyPath(row, ["description", "industry", "url", "website", "employee_count", "follower_count"])
1316
- );
1317
- return matching.length >= 1 ? { verdict: "pass", reason: "company_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1318
- }
1319
-
1320
- if (/\b(person|people|profile|profiles|member|members|user|users)\b/.test(lower)) {
1321
- const matching = objects.filter((row) =>
1322
- hasAnyPath(row, ["name", "title", "public_identifier", "username"]) &&
1323
- hasAnyPath(row, ["headline", "url", "public_identifier", "username"])
1324
- );
1325
- const minRows = /\b(profile|profiles|user|users|person)\b/.test(lower) && !/\bsearch\b/.test(lower) ? 1 : 2;
1326
- return matching.length >= minRows ? { verdict: "pass", reason: "people_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1327
- }
1328
-
1329
- if (/\b(comment|comments)\b/.test(lower)) {
1330
- const matching = objects.filter((row) =>
1331
- hasAnyPath(row, ["id", "url", "permalink"]) &&
1332
- hasAnyPath(row, ["author", "body", "text"])
1333
- );
1334
- return matching.length >= 1 ? { verdict: "pass", reason: "comment_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1335
- }
1336
-
1337
- if (/\b(email|emails|mail|inbox)\b/.test(lower)) {
1338
- const matching = objects.filter((row) =>
1339
- hasAnyPath(row, ["id", "thread.doc_id", "matchedEmail.id"]) &&
1340
- hasAnyPath(row, ["subject", "thread.subject", "from", "sender", "thread.latest_sender_name"]) &&
1341
- hasAnyPath(row, ["date", "thread.formatted_date", "preview", "thread.preview", "matchedEmail.content_markdown"])
1342
- );
1343
- return matching.length >= 1 ? { verdict: "pass", reason: "email_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1344
- }
1345
-
1346
- if (/\b(post|posts|tweet|tweets|status|statuses)\b/.test(lower)) {
1347
- const matching = objects.filter((row) =>
1348
- hasAnyPath(row, ["id", "url", "uri", "permalink", "_id", "entityUrn", "rest_id"]) &&
1349
- countMatchingGroups(row, [
1350
- ["content", "text", "title", "body", "message", "description", "summary", "commentary.text.text"],
1351
- ["account.username", "account.acct", "username", "author", "user.name", "from.name", "actor.name", "actor.name.text"],
1352
- ["score", "points", "likes", "favorite_count", "num_comments", "reply_count", "reactions", "repost_count"],
1353
- ["date", "created_at", "published_at", "timestamp", "meta", "created_time", "createdAt", "updated_at", "time"],
1354
- ]) >= 2
1355
- );
1356
- return matching.length >= 1 ? { verdict: "pass", reason: "post_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1357
- }
1358
-
1359
- if (/\b(reddit|subreddit)\b/.test(lower)) {
1360
- const matching = objects.filter((row) =>
1361
- hasAnyPath(row, ["id", "url", "permalink"]) &&
1362
- hasAnyPath(row, ["title", "subreddit", "author"]) &&
1363
- hasAnyPath(row, ["num_comments", "score", "permalink"])
1364
- );
1365
- return matching.length >= 1 ? { verdict: "pass", reason: "reddit_post_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1366
- }
1367
-
1368
- if (/\b(topic|topics|trend|trending|hashtag|hashtags)\b/.test(lower)) {
1369
- const matching = objects.filter((row) =>
1370
- hasAnyPath(row, ["name", "title", "query"]) &&
1371
- hasAnyPath(row, ["url", "name", "query"])
1372
- );
1373
- return matching.length >= 2 ? { verdict: "pass", reason: "topic_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1374
- }
1375
-
1376
- if (/\b(doc|docs|documentation)\b/.test(lower)) {
1377
- const matching = objects.filter((row) =>
1378
- hasAnyPath(row, ["title", "name", "slug"]) &&
1379
- hasAnyPath(row, ["url", "link", "href", "mdn_url"]) &&
1380
- hasAnyPath(row, ["summary", "description", "slug", "popularity", "score"])
1381
- );
1382
- return matching.length >= 1 ? { verdict: "pass", reason: "document_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1383
- }
1384
-
1385
- if (/\b(paper|papers)\b/.test(lower)) {
1386
- const matching = objects.filter((row) =>
1387
- hasAnyPath(row, ["title", "name"]) &&
1388
- hasAnyPath(row, ["url", "link", "href"]) &&
1389
- hasAnyPath(row, ["summary", "description", "author", "authors", "date", "published", "meta"])
1390
- );
1391
- return matching.length >= 1 ? { verdict: "pass", reason: "paper_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1392
- }
1393
-
1394
- if (/\b(question|questions)\b/.test(lower)) {
1395
- const matching = objects.filter((row) =>
1396
- hasAnyPath(row, ["title", "name"]) &&
1397
- hasAnyPath(row, ["url", "link", "href", "permalink"]) &&
1398
- countMatchingGroups(row, [
1399
- ["votes", "score", "points"],
1400
- ["answers", "answer_count", "num_answers", "num_comments"],
1401
- ["author", "date", "created_at", "meta"],
1402
- ]) >= 2
1403
- );
1404
- return matching.length >= 2 ? { verdict: "pass", reason: "question_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1405
- }
1406
-
1407
- if (/\b(recipe|recipes)\b/.test(lower)) {
1408
- const matching = objects.filter((row) =>
1409
- hasAnyPath(row, ["title", "name"]) &&
1410
- hasAnyPath(row, ["url", "link", "href"]) &&
1411
- hasAnyPath(row, ["rating", "review_count", "author", "description", "summary", "cook_time", "total_time"])
1412
- );
1413
- return matching.length >= 1 ? { verdict: "pass", reason: "recipe_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1414
- }
1415
-
1416
- if (/\b(course|courses)\b/.test(lower)) {
1417
- const matching = objects.filter((row) =>
1418
- hasAnyPath(row, ["title", "name"]) &&
1419
- hasAnyPath(row, ["url", "link", "href"]) &&
1420
- hasAnyPath(row, ["rating", "partner", "provider", "instructor", "description", "summary", "duration", "level"])
1421
- );
1422
- return matching.length >= 1 ? { verdict: "pass", reason: "course_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1423
- }
1424
-
1425
- if (/\b(definition|dictionary|meaning)\b/.test(lower)) {
1426
- const matching = objects.filter((row) =>
1427
- hasAnyPath(row, ["term", "title", "name", "word"]) &&
1428
- hasAnyPath(row, ["definition", "summary", "body", "description"])
1429
- );
1430
- return matching.length >= 1 ? { verdict: "pass", reason: "definition_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1431
- }
1432
-
1433
- if (/\b(stock|stocks|ticker|tickers|quote|quotes)\b/.test(lower)) {
1434
- const matching = objects.filter((row) =>
1435
- hasAnyPath(row, ["symbol", "ticker"]) &&
1436
- hasAnyPath(row, ["price", "regularMarketPrice", "current_price"]) &&
1437
- hasAnyPath(row, ["name", "currency", "change_percent", "market_cap", "market_state"])
1438
- );
1439
- return matching.length >= 1 ? { verdict: "pass", reason: "quote_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1440
- }
1441
-
1442
- if (/\b(product|products|item|items)\b/.test(lower)) {
1443
- const matching = objects.filter((row) =>
1444
- hasAnyPath(row, ["title", "name", "product_name"]) &&
1445
- (
1446
- hasAnyPath(row, ["url", "id", "product_id", "sku"]) ||
1447
- hasAnyPath(row, ["description", "summary"])
1448
- ) &&
1449
- hasAnyPath(row, ["price", "rating", "review_count", "brand", "image"]) &&
1450
- !/^results?$/i.test(String(getPath(row, "title") ?? getPath(row, "name") ?? "")) &&
1451
- !Object.keys(row).some((key) => /^heading_\d+$/i.test(key)) &&
1452
- !/(amazon-adsystem|doubleclick|googlesyndication|pubmatic|taboola|outbrain|aax-|googleadservices|adservice)/i.test(
1453
- String(getPath(row, "url") ?? ""),
1454
- )
1455
- );
1456
- return matching.length >= 1 ? { verdict: "pass", reason: "product_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1457
- }
1458
-
1459
- if (/\b(channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(lower)) {
1460
- const matching = objects.filter((row) =>
1461
- hasAnyPath(row, ["name", "title"]) &&
1462
- hasAnyPath(row, ["id", "url", "channel_id", "guild_id", "workspace_id"]) &&
1463
- hasAnyPath(row, ["description", "type", "member_count", "url"])
1464
- );
1465
- return matching.length >= 1 ? { verdict: "pass", reason: "channel_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1466
- }
1467
-
1468
- return { verdict: "skip", reason: "unclassified_array" };
1469
- }
1470
-
1471
- export function assessIntentResult(data: unknown, intent?: string): {
1472
- verdict: "pass" | "fail" | "skip";
1473
- reason: string;
1474
- projected: unknown;
1475
- } {
1476
- const projected = projectIntentData(data, intent);
1477
-
1478
- if (projected == null) return { verdict: "fail", reason: "no_data", projected };
1479
- if (typeof projected === "string") {
1480
- const trimmed = projected.trim().toLowerCase();
1481
- if (!trimmed) return { verdict: "fail", reason: "empty_text", projected };
1482
- if (trimmed.startsWith("<!doctype") || trimmed.startsWith("<html") || trimmed.startsWith("<body")) {
1483
- return { verdict: "fail", reason: "html_payload", projected };
1484
- }
1485
- return { verdict: "skip", reason: "plain_text", projected };
1486
- }
1487
-
1488
- if (Array.isArray(projected)) {
1489
- const classified = classifyRows(projected, intent ?? "");
1490
- return { ...classified, projected };
1491
- }
1492
-
1493
- if (isRecord(projected)) {
1494
- if ("error" in projected) return { verdict: "fail", reason: String(projected.error || "error_payload"), projected };
1495
- if ("available_endpoints" in projected || "available_operations" in projected) {
1496
- return { verdict: "fail", reason: "deferral_payload", projected };
1497
- }
1498
- if ("learned_skill_id" in projected && !("data" in projected)) {
1499
- return { verdict: "fail", reason: "learned_skill_only", projected };
1500
- }
1501
- const lower = (intent ?? "").toLowerCase();
1502
- if (("message" in projected || "flash" in projected) && !("data" in projected)) {
1503
- if (/\b(message|messages|flash|alert|success|error|warning)\b/.test(lower)) {
1504
- return { verdict: "skip", reason: "message_record", projected };
1505
- }
1506
- return { verdict: "fail", reason: "message_only", projected };
1507
- }
1508
- if (/\b(company|companies|organization|organisations|business|person|people|profile|profiles|member|members|user|users|repo|repository|repositories|project|projects|package|packages|doc|docs|documentation|question|questions|recipe|recipes|course|courses|definition|dictionary|meaning|product|products|item|items|stock|stocks|ticker|tickers|quote|quotes|channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(lower)) {
1509
- const classified = classifyRows([projected], intent ?? "");
1510
- return { ...classified, projected };
1511
- }
1512
- }
1513
-
1514
- return { verdict: "skip", reason: "unclassified_payload", projected };
1515
- }