unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/cli.js +79 -33
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +73 -22
  4. package/dist/server.js +25994 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/manifest.json +1 -1
  7. package/runtime-src/agent-outcome.ts +0 -166
  8. package/runtime-src/analytics-session.ts +0 -55
  9. package/runtime-src/api/browse-index.ts +0 -343
  10. package/runtime-src/api/browse-session.ts +0 -572
  11. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  12. package/runtime-src/api/browse-submit.ts +0 -1184
  13. package/runtime-src/api/routes.ts +0 -1833
  14. package/runtime-src/auth/browser-cookies.ts +0 -423
  15. package/runtime-src/auth/index.ts +0 -535
  16. package/runtime-src/auth/runtime.ts +0 -116
  17. package/runtime-src/browser/index.ts +0 -659
  18. package/runtime-src/browser/types.ts +0 -41
  19. package/runtime-src/build-info.generated.ts +0 -6
  20. package/runtime-src/capture/index.ts +0 -1929
  21. package/runtime-src/capture/prefetch.ts +0 -95
  22. package/runtime-src/capture/rsc.ts +0 -45
  23. package/runtime-src/cli/shortcuts.ts +0 -273
  24. package/runtime-src/cli.ts +0 -1734
  25. package/runtime-src/client/graph-client.ts +0 -100
  26. package/runtime-src/client/index.ts +0 -1425
  27. package/runtime-src/debug-trace.ts +0 -18
  28. package/runtime-src/domain.ts +0 -38
  29. package/runtime-src/execution/index.ts +0 -3407
  30. package/runtime-src/execution/retry.ts +0 -46
  31. package/runtime-src/execution/robots.ts +0 -167
  32. package/runtime-src/execution/search-forms.ts +0 -188
  33. package/runtime-src/execution/token-resolver.ts +0 -135
  34. package/runtime-src/extraction/index.ts +0 -1507
  35. package/runtime-src/foundry/publish-bundle.ts +0 -392
  36. package/runtime-src/graph/agent-augment.ts +0 -315
  37. package/runtime-src/graph/index.ts +0 -1532
  38. package/runtime-src/graph/local-fixtures.ts +0 -393
  39. package/runtime-src/graph/local-harness.ts +0 -646
  40. package/runtime-src/graph/planner.ts +0 -411
  41. package/runtime-src/graph/session.ts +0 -294
  42. package/runtime-src/graph/trace-store.ts +0 -136
  43. package/runtime-src/impact-log.ts +0 -227
  44. package/runtime-src/index.ts +0 -24
  45. package/runtime-src/indexer/index.ts +0 -465
  46. package/runtime-src/intent-match.ts +0 -1515
  47. package/runtime-src/kuri/client.ts +0 -1839
  48. package/runtime-src/logger.ts +0 -30
  49. package/runtime-src/marketplace/index.ts +0 -111
  50. package/runtime-src/mcp.ts +0 -1911
  51. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  52. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  53. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  54. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  55. package/runtime-src/orchestrator/index.ts +0 -4480
  56. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  57. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  58. package/runtime-src/payments/cascade.ts +0 -137
  59. package/runtime-src/payments/index.ts +0 -270
  60. package/runtime-src/payments/lobster-pay.ts +0 -182
  61. package/runtime-src/payments/wallet.ts +0 -98
  62. package/runtime-src/publish/review-context.ts +0 -93
  63. package/runtime-src/publish/sanitize.ts +0 -197
  64. package/runtime-src/publish/schema-review.ts +0 -192
  65. package/runtime-src/publish-admission.ts +0 -388
  66. package/runtime-src/ratelimit/index.ts +0 -23
  67. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  68. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  69. package/runtime-src/reverse-engineer/index.ts +0 -1551
  70. package/runtime-src/reverse-engineer/token-sources.ts +0 -379
  71. package/runtime-src/router.ts +0 -17
  72. package/runtime-src/routing-telemetry.ts +0 -395
  73. package/runtime-src/runtime/browser-access.ts +0 -11
  74. package/runtime-src/runtime/browser-auth.ts +0 -12
  75. package/runtime-src/runtime/browser-host.ts +0 -48
  76. package/runtime-src/runtime/lifecycle.ts +0 -17
  77. package/runtime-src/runtime/local-server.ts +0 -311
  78. package/runtime-src/runtime/paths.ts +0 -99
  79. package/runtime-src/runtime/setup.ts +0 -251
  80. package/runtime-src/runtime/supervisor.ts +0 -69
  81. package/runtime-src/runtime/update-hints.ts +0 -351
  82. package/runtime-src/server.ts +0 -100
  83. package/runtime-src/session-logs.ts +0 -142
  84. package/runtime-src/settings.ts +0 -221
  85. package/runtime-src/single-binary.ts +0 -143
  86. package/runtime-src/site-policy.ts +0 -54
  87. package/runtime-src/stale-cleanup-runner.ts +0 -144
  88. package/runtime-src/stale-cleanup.ts +0 -133
  89. package/runtime-src/telemetry-attribution.ts +0 -120
  90. package/runtime-src/telemetry.ts +0 -253
  91. package/runtime-src/template-params.ts +0 -141
  92. package/runtime-src/transform/drift.ts +0 -60
  93. package/runtime-src/transform/index.ts +0 -277
  94. package/runtime-src/types/index.ts +0 -1
  95. package/runtime-src/types/skill.ts +0 -931
  96. package/runtime-src/vault/index.ts +0 -196
  97. package/runtime-src/verification/auth-gate.ts +0 -8
  98. package/runtime-src/verification/candidates.ts +0 -27
  99. package/runtime-src/verification/index.ts +0 -120
  100. package/runtime-src/verification/matrix.ts +0 -30
  101. package/runtime-src/version.ts +0 -148
  102. package/runtime-src/workflow/artifact.ts +0 -161
  103. package/runtime-src/workflow/compile.ts +0 -808
  104. package/runtime-src/workflow/publish.ts +0 -225
  105. package/runtime-src/workflow/runtime.ts +0 -213
@@ -1,1515 +0,0 @@
1
- function isRecord(value: unknown): value is Record<string, unknown> {
2
- return !!value && typeof value === "object" && !Array.isArray(value);
3
- }
4
-
5
- function getPath(obj: unknown, path: string): unknown {
6
- if (!isRecord(obj)) return undefined;
7
- let cur: unknown = obj;
8
- for (const key of path.split(".")) {
9
- if (!isRecord(cur)) return undefined;
10
- cur = cur[key];
11
- }
12
- return cur;
13
- }
14
-
15
- function hasNonEmptyString(value: unknown): boolean {
16
- return typeof value === "string" && value.trim().length > 0;
17
- }
18
-
19
- function hasAnyPath(obj: unknown, paths: string[]): boolean {
20
- return paths.some((path) => {
21
- const value = getPath(obj, path);
22
- return hasNonEmptyString(value) || typeof value === "number";
23
- });
24
- }
25
-
26
- function countMatchingGroups(obj: unknown, groups: string[][]): number {
27
- let matches = 0;
28
- for (const group of groups) {
29
- if (hasAnyPath(obj, group)) matches++;
30
- }
31
- return matches;
32
- }
33
-
34
- function toStringArray(value: unknown): string[] | undefined {
35
- if (Array.isArray(value)) {
36
- const out = value
37
- .filter((item): item is string => hasNonEmptyString(item))
38
- .map((item) => String(item).trim());
39
- return out.length > 0 ? out : undefined;
40
- }
41
- if (hasNonEmptyString(value)) return [String(value).trim()];
42
- return undefined;
43
- }
44
-
45
- function firstNonEmptyString(...values: unknown[]): string | undefined {
46
- for (const value of values) {
47
- if (hasNonEmptyString(value)) return String(value).trim();
48
- }
49
- return undefined;
50
- }
51
-
52
- function normalizePackageSearchResults(data: unknown): Record<string, unknown>[] {
53
- const sourceRows = Array.isArray(data)
54
- ? data
55
- : isRecord(data) && Array.isArray(data.objects)
56
- ? data.objects
57
- : isRecord(data) && Array.isArray(data.packages)
58
- ? data.packages
59
- : [];
60
- const rows: Record<string, unknown>[] = [];
61
- const seen = new Set<string>();
62
-
63
- for (const item of sourceRows) {
64
- if (!isRecord(item)) continue;
65
- const pkg = isRecord(item.package) ? item.package : item;
66
- const name = getPath(pkg, "name");
67
- if (!hasNonEmptyString(name) || seen.has(String(name))) continue;
68
- const version = firstNonEmptyString(getPath(pkg, "version"));
69
- const description = firstNonEmptyString(getPath(pkg, "description"), getPath(pkg, "summary"));
70
- const url =
71
- firstNonEmptyString(
72
- getPath(pkg, "links.npm"),
73
- getPath(pkg, "project_url"),
74
- getPath(pkg, "package_url"),
75
- getPath(pkg, "url"),
76
- ) ?? `https://www.npmjs.com/package/${encodeURIComponent(String(name))}`;
77
- const keywords = toStringArray(getPath(pkg, "keywords"));
78
- rows.push({
79
- name: String(name),
80
- ...(version ? { version } : {}),
81
- ...(description ? { description } : {}),
82
- ...(keywords ? { keywords } : {}),
83
- url,
84
- });
85
- seen.add(String(name));
86
- }
87
-
88
- return rows;
89
- }
90
-
91
- function normalizeCratesPackageSearchResults(data: unknown): Record<string, unknown>[] {
92
- const sourceRows = isRecord(data) && Array.isArray(data.crates)
93
- ? data.crates
94
- : Array.isArray(data)
95
- ? data
96
- : [];
97
- const rows: Record<string, unknown>[] = [];
98
- const seen = new Set<string>();
99
-
100
- for (const item of sourceRows) {
101
- if (!isRecord(item)) continue;
102
- const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "id"));
103
- if (!name || seen.has(name)) continue;
104
- const description = firstNonEmptyString(getPath(item, "description"), getPath(item, "summary"));
105
- const version = firstNonEmptyString(getPath(item, "max_version"), getPath(item, "default_version"), getPath(item, "version"));
106
- rows.push({
107
- name,
108
- ...(description ? { description, summary: description } : {}),
109
- ...(version ? { version } : {}),
110
- ...(typeof getPath(item, "downloads") === "number" ? { downloads: getPath(item, "downloads") } : {}),
111
- url: `https://crates.io/crates/${encodeURIComponent(name)}`,
112
- });
113
- seen.add(name);
114
- }
115
-
116
- return rows;
117
- }
118
-
119
- function normalizeNpmPackageInfo(data: unknown): Record<string, unknown> | null {
120
- if (!isRecord(data) || !hasNonEmptyString(data.name)) return null;
121
- const distTags = isRecord(data["dist-tags"]) ? data["dist-tags"] : undefined;
122
- const versions = isRecord(data.versions) ? data.versions : undefined;
123
- const latestVersion = firstNonEmptyString(distTags?.latest, data.version);
124
- const latestRecord =
125
- latestVersion && versions && isRecord(versions[latestVersion])
126
- ? versions[latestVersion]
127
- : undefined;
128
- const description = firstNonEmptyString(
129
- getPath(latestRecord, "description"),
130
- getPath(data, "description"),
131
- );
132
- const homepage = firstNonEmptyString(
133
- getPath(latestRecord, "homepage"),
134
- getPath(data, "homepage"),
135
- getPath(latestRecord, "repository.url"),
136
- getPath(data, "repository.url"),
137
- );
138
- const dependencies = isRecord(getPath(latestRecord, "dependencies"))
139
- ? Object.keys(getPath(latestRecord, "dependencies") as Record<string, unknown>)
140
- : undefined;
141
- const keywords = toStringArray(getPath(latestRecord, "keywords") ?? getPath(data, "keywords"));
142
- const author = firstNonEmptyString(
143
- getPath(latestRecord, "author.name"),
144
- getPath(latestRecord, "author"),
145
- getPath(data, "author.name"),
146
- getPath(data, "author"),
147
- );
148
-
149
- return {
150
- name: String(data.name),
151
- ...(latestVersion ? { version: latestVersion } : {}),
152
- ...(description ? { description } : {}),
153
- ...(keywords ? { keywords } : {}),
154
- ...(dependencies && dependencies.length > 0 ? { dependencies } : {}),
155
- ...(author ? { author } : {}),
156
- url: homepage ?? firstNonEmptyString(getPath(data, "url"), getPath(data, "package_url"), getPath(data, "project_url"))
157
- ?? `https://www.npmjs.com/package/${encodeURIComponent(String(data.name))}`,
158
- };
159
- }
160
-
161
- function normalizeDocumentRows(data: unknown): Record<string, unknown>[] {
162
- const sourceRows = Array.isArray(data)
163
- ? data
164
- : isRecord(data) && Array.isArray(data.documents)
165
- ? data.documents
166
- : isRecord(data) && Array.isArray(data.results)
167
- ? data.results
168
- : [];
169
- const rows: Record<string, unknown>[] = [];
170
- const seen = new Set<string>();
171
-
172
- for (const item of sourceRows) {
173
- if (!isRecord(item)) continue;
174
- const title = firstNonEmptyString(getPath(item, "title"), getPath(item, "name"));
175
- const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "mdn_url"), getPath(item, "link"), getPath(item, "href"));
176
- if (!title || !url) continue;
177
- const id = `${title}|${url}`;
178
- if (seen.has(id)) continue;
179
- rows.push({
180
- title,
181
- url,
182
- ...(firstNonEmptyString(getPath(item, "summary"), getPath(item, "description"), getPath(item, "slug")) ? {
183
- summary: firstNonEmptyString(getPath(item, "summary"), getPath(item, "description"), getPath(item, "slug")),
184
- } : {}),
185
- ...(typeof getPath(item, "score") === "number" ? { score: getPath(item, "score") } : {}),
186
- ...(typeof getPath(item, "popularity") === "number" ? { popularity: getPath(item, "popularity") } : {}),
187
- ...(firstNonEmptyString(getPath(item, "locale")) ? { locale: firstNonEmptyString(getPath(item, "locale")) } : {}),
188
- });
189
- seen.add(id);
190
- }
191
-
192
- return rows;
193
- }
194
-
195
- function normalizePyPIPackageInfo(data: unknown): Record<string, unknown> | null {
196
- if (!isRecord(data)) return null;
197
- const info = isRecord(data.info) ? data.info : data;
198
- const name = getPath(info, "name");
199
- if (!hasNonEmptyString(name)) return null;
200
- const summary = firstNonEmptyString(getPath(info, "summary"), getPath(info, "description"));
201
- const version = firstNonEmptyString(getPath(info, "version"));
202
- const author = firstNonEmptyString(getPath(info, "author"));
203
- const requiresDist = Array.isArray(getPath(info, "requires_dist"))
204
- ? (getPath(info, "requires_dist") as unknown[]).filter((item) => hasNonEmptyString(item)).map(String)
205
- : undefined;
206
- const url = firstNonEmptyString(
207
- getPath(info, "project_url"),
208
- getPath(info, "package_url"),
209
- getPath(info, "home_page"),
210
- getPath(info, "project_urls.Homepage"),
211
- );
212
-
213
- return {
214
- name: String(name),
215
- ...(version ? { version } : {}),
216
- ...(summary ? { summary, description: summary } : {}),
217
- ...(author ? { author } : {}),
218
- ...(requiresDist && requiresDist.length > 0 ? { requires_dist: requiresDist } : {}),
219
- ...(url ? { url } : {}),
220
- };
221
- }
222
-
223
- function normalizePubDevPackageInfo(data: unknown): Record<string, unknown> | null {
224
- if (!isRecord(data)) return null;
225
- const latest = isRecord(data.latest) ? data.latest : undefined;
226
- const pubspec = isRecord(latest?.pubspec) ? latest?.pubspec : undefined;
227
- const name = firstNonEmptyString(getPath(data, "name"), getPath(pubspec, "name"));
228
- if (!name) return null;
229
- const version = firstNonEmptyString(getPath(latest, "version"), getPath(pubspec, "version"));
230
- const description = firstNonEmptyString(getPath(pubspec, "description"));
231
- const repository = firstNonEmptyString(getPath(pubspec, "repository"), getPath(pubspec, "homepage"));
232
- const topics = toStringArray(getPath(pubspec, "topics"));
233
- return {
234
- name,
235
- ...(version ? { version } : {}),
236
- ...(description ? { description, summary: description } : {}),
237
- ...(topics ? { keywords: topics } : {}),
238
- url: repository ?? `https://pub.dev/packages/${encodeURIComponent(name)}`,
239
- };
240
- }
241
-
242
- function normalizeRubyGemsPackageInfo(data: unknown): Record<string, unknown> | null {
243
- if (!isRecord(data)) return null;
244
- const name = firstNonEmptyString(getPath(data, "name"));
245
- if (!name) return null;
246
- const description = firstNonEmptyString(getPath(data, "info"));
247
- const version = firstNonEmptyString(getPath(data, "version"));
248
- const author = firstNonEmptyString(getPath(data, "authors"), getPath(data, "author"));
249
- const homepage = firstNonEmptyString(getPath(data, "homepage_uri"), getPath(data, "project_uri"), getPath(data, "gem_uri"));
250
- return {
251
- name,
252
- ...(version ? { version } : {}),
253
- ...(description ? { description, summary: description } : {}),
254
- ...(author ? { author } : {}),
255
- url: homepage ?? `https://rubygems.org/gems/${encodeURIComponent(name)}`,
256
- };
257
- }
258
-
259
- function normalizeDockerImageSearchResults(data: unknown): Record<string, unknown>[] {
260
- const sourceRows = isRecord(data) && Array.isArray(data.results)
261
- ? data.results
262
- : Array.isArray(data)
263
- ? data
264
- : [];
265
- const rows: Record<string, unknown>[] = [];
266
- const seen = new Set<string>();
267
-
268
- for (const item of sourceRows) {
269
- if (!isRecord(item)) continue;
270
- const repoName = firstNonEmptyString(getPath(item, "repo_name"), getPath(item, "name"));
271
- if (!repoName || seen.has(repoName)) continue;
272
- rows.push({
273
- repo_name: repoName,
274
- ...(firstNonEmptyString(getPath(item, "short_description"), getPath(item, "description")) ? {
275
- short_description: firstNonEmptyString(getPath(item, "short_description"), getPath(item, "description")),
276
- } : {}),
277
- ...(typeof getPath(item, "star_count") === "number" ? { star_count: getPath(item, "star_count") } : {}),
278
- ...(typeof getPath(item, "pull_count") === "number" ? { pull_count: getPath(item, "pull_count") } : {}),
279
- url: `https://hub.docker.com/r/${repoName}`,
280
- });
281
- seen.add(repoName);
282
- }
283
-
284
- return rows;
285
- }
286
-
287
- function normalizeDockerTagResults(data: unknown): Record<string, unknown>[] {
288
- const sourceRows = isRecord(data) && Array.isArray(data.results)
289
- ? data.results
290
- : Array.isArray(data)
291
- ? data
292
- : [];
293
- const rows: Record<string, unknown>[] = [];
294
- const seen = new Set<string>();
295
-
296
- for (const item of sourceRows) {
297
- if (!isRecord(item)) continue;
298
- const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "tag"));
299
- if (!name || seen.has(name)) continue;
300
- rows.push({
301
- name,
302
- ...(typeof getPath(item, "full_size") === "number" ? { full_size: getPath(item, "full_size") } : {}),
303
- ...(firstNonEmptyString(getPath(item, "last_updated"), getPath(item, "updated_at")) ? {
304
- last_updated: firstNonEmptyString(getPath(item, "last_updated"), getPath(item, "updated_at")),
305
- } : {}),
306
- ...(firstNonEmptyString(getPath(item, "digest")) ? { digest: firstNonEmptyString(getPath(item, "digest")) } : {}),
307
- });
308
- seen.add(name);
309
- }
310
-
311
- return rows;
312
- }
313
-
314
- function normalizeHackerNewsStories(data: unknown): Record<string, unknown>[] {
315
- const sourceRows = isRecord(data) && Array.isArray(data.hits)
316
- ? data.hits
317
- : Array.isArray(data)
318
- ? data
319
- : [];
320
- const rows: Record<string, unknown>[] = [];
321
- const seen = new Set<string>();
322
-
323
- for (const item of sourceRows) {
324
- if (!isRecord(item)) continue;
325
- const id = firstNonEmptyString(getPath(item, "objectID"), getPath(item, "id"), getPath(item, "story_id"));
326
- const title = firstNonEmptyString(getPath(item, "title"), getPath(item, "story_title"));
327
- const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "story_url"), getPath(item, "link"));
328
- const author = firstNonEmptyString(getPath(item, "author"));
329
- const meta = firstNonEmptyString(getPath(item, "meta"));
330
- if (!title) continue;
331
- const stableId = id ?? url ?? title;
332
- if (seen.has(stableId)) continue;
333
- const pointsMatch = meta?.match(/(^|\|)\s*([0-9,]+)\s+points?\b/i);
334
- const commentsMatch = meta?.match(/(^|\|)\s*([0-9,]+)\s+comments?\b/i);
335
-
336
- rows.push({
337
- ...(stableId ? { id: stableId } : {}),
338
- title,
339
- ...(url ? { url } : {}),
340
- ...(author ? { author } : {}),
341
- ...(typeof getPath(item, "points") === "number"
342
- ? { points: getPath(item, "points") }
343
- : pointsMatch ? { points: Number(pointsMatch[2]?.replace(/,/g, "")) } : {}),
344
- ...(typeof getPath(item, "num_comments") === "number"
345
- ? { num_comments: getPath(item, "num_comments") }
346
- : commentsMatch ? { num_comments: Number(commentsMatch[2]?.replace(/,/g, "")) } : {}),
347
- ...(meta ? { meta } : {}),
348
- });
349
- seen.add(stableId);
350
- }
351
-
352
- return rows;
353
- }
354
-
355
- function normalizeStackExchangeQuestions(data: unknown): Record<string, unknown>[] {
356
- const sourceRows = isRecord(data) && Array.isArray(data.items)
357
- ? data.items
358
- : Array.isArray(data)
359
- ? data
360
- : [];
361
- const rows: Record<string, unknown>[] = [];
362
- const seen = new Set<string>();
363
- for (const item of sourceRows) {
364
- if (!isRecord(item)) continue;
365
- const title = firstNonEmptyString(getPath(item, "title"));
366
- const url = firstNonEmptyString(getPath(item, "link"));
367
- if (!title || !url || seen.has(url)) continue;
368
- rows.push({
369
- title,
370
- url,
371
- ...(typeof getPath(item, "score") === "number" ? { score: getPath(item, "score") } : {}),
372
- ...(typeof getPath(item, "answer_count") === "number" ? { answer_count: getPath(item, "answer_count") } : {}),
373
- ...(firstNonEmptyString(getPath(item, "owner.display_name")) ? { author: firstNonEmptyString(getPath(item, "owner.display_name")) } : {}),
374
- ...(typeof getPath(item, "last_activity_date") === "number" ? { date: getPath(item, "last_activity_date") } : {}),
375
- });
376
- seen.add(url);
377
- }
378
- return rows;
379
- }
380
-
381
- function normalizeDevToPosts(data: unknown): Record<string, unknown>[] {
382
- const sourceRows = Array.isArray(data)
383
- ? data
384
- : isRecord(data) && Array.isArray(data.result)
385
- ? data.result
386
- : [];
387
- const rows: Record<string, unknown>[] = [];
388
- const seen = new Set<string>();
389
- const inferAuthor = (value: unknown): string | undefined => {
390
- const raw = hasNonEmptyString(value) ? String(value) : undefined;
391
- if (!raw) return undefined;
392
- try {
393
- const pathname = raw.startsWith("http") ? new URL(raw).pathname : raw;
394
- const segments = pathname.split("/").filter(Boolean);
395
- const candidate = segments[0];
396
- if (!candidate || candidate === "t" || candidate === "s" || candidate === "tag" || candidate === "tags" || candidate === "top" || candidate === "latest") {
397
- return undefined;
398
- }
399
- return candidate;
400
- } catch {
401
- return undefined;
402
- }
403
- };
404
- for (const item of sourceRows) {
405
- if (!isRecord(item)) continue;
406
- const title = firstNonEmptyString(getPath(item, "title"));
407
- const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "path"));
408
- if (!title || !url) continue;
409
- const looksLikeDevTo = (
410
- firstNonEmptyString(getPath(item, "type_of")) === "article" ||
411
- hasNonEmptyString(getPath(item, "user.name")) ||
412
- hasNonEmptyString(getPath(item, "user.username")) ||
413
- typeof getPath(item, "positive_reactions_count") === "number" ||
414
- typeof getPath(item, "comments_count") === "number" ||
415
- hasNonEmptyString(getPath(item, "published_at")) ||
416
- hasNonEmptyString(getPath(item, "readable_publish_date")) ||
417
- url.startsWith("https://dev.to/") ||
418
- url.startsWith("http://dev.to/") ||
419
- (url.startsWith("/") && !!inferAuthor(url))
420
- );
421
- if (!looksLikeDevTo) continue;
422
- const canonicalUrl = url.startsWith("http") ? url : `https://dev.to${url}`;
423
- if (seen.has(canonicalUrl)) continue;
424
- const author = firstNonEmptyString(
425
- getPath(item, "user.name"),
426
- getPath(item, "user.username"),
427
- inferAuthor(getPath(item, "path")),
428
- inferAuthor(canonicalUrl),
429
- );
430
- rows.push({
431
- ...(firstNonEmptyString(getPath(item, "id")) ? { id: firstNonEmptyString(getPath(item, "id")) } : {}),
432
- title,
433
- url: canonicalUrl,
434
- ...(firstNonEmptyString(getPath(item, "description")) ? { description: firstNonEmptyString(getPath(item, "description")) } : {}),
435
- ...(author ? { author } : {}),
436
- ...(typeof getPath(item, "positive_reactions_count") === "number" ? { likes: getPath(item, "positive_reactions_count") } : {}),
437
- ...(typeof getPath(item, "comments_count") === "number" ? { comments: getPath(item, "comments_count") } : {}),
438
- ...(firstNonEmptyString(getPath(item, "published_at"), getPath(item, "readable_publish_date")) ? { date: firstNonEmptyString(getPath(item, "published_at"), getPath(item, "readable_publish_date")) } : {}),
439
- });
440
- seen.add(canonicalUrl);
441
- }
442
- return rows;
443
- }
444
-
445
- function normalizeLobstersPosts(data: unknown): Record<string, unknown>[] {
446
- const sourceRows = Array.isArray(data)
447
- ? data
448
- : isRecord(data) && Array.isArray(data.results)
449
- ? data.results
450
- : [];
451
- const rows: Record<string, unknown>[] = [];
452
- const seen = new Set<string>();
453
- for (const item of sourceRows) {
454
- if (!isRecord(item)) continue;
455
- const title = firstNonEmptyString(getPath(item, "title"));
456
- const rawUrl = firstNonEmptyString(getPath(item, "url"), getPath(item, "link"), getPath(item, "href"));
457
- const text = firstNonEmptyString(getPath(item, "text"), getPath(item, "body"));
458
- if (!title || !rawUrl || /^\d+\s+comments?$/i.test(title)) continue;
459
- const scoreMatch = text?.match(/^\s*(\d{1,4})\s+/)?.[1];
460
- const score = typeof getPath(item, "score") === "number"
461
- ? getPath(item, "score")
462
- : typeof getPath(item, "points") === "number"
463
- ? getPath(item, "points")
464
- : scoreMatch ? Number(scoreMatch) : undefined;
465
- const url = rawUrl.startsWith("http") ? rawUrl : `https://lobste.rs${rawUrl.startsWith("/") ? "" : "/"}${rawUrl}`;
466
- const stable = `${title}|${url}`;
467
- if (seen.has(stable)) continue;
468
- rows.push({
469
- title,
470
- url,
471
- ...(typeof score === "number" ? { score } : {}),
472
- ...(firstNonEmptyString(getPath(item, "author"), getPath(item, "username")) ? { author: firstNonEmptyString(getPath(item, "author"), getPath(item, "username")) } : {}),
473
- });
474
- seen.add(stable);
475
- }
476
- return rows;
477
- }
478
-
479
- function normalizeHuggingFaceModels(data: unknown): Record<string, unknown>[] {
480
- const sourceRows = Array.isArray(data)
481
- ? data
482
- : isRecord(data) && Array.isArray(data.models)
483
- ? data.models
484
- : [];
485
- const rows: Record<string, unknown>[] = [];
486
- const seen = new Set<string>();
487
-
488
- for (const item of sourceRows) {
489
- if (!isRecord(item)) continue;
490
- const id = firstNonEmptyString(getPath(item, "id"), getPath(item, "modelId"), getPath(item, "name"), getPath(item, "title"));
491
- if (!id || seen.has(id)) continue;
492
- const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "title"), getPath(item, "modelId"), getPath(item, "id"));
493
- rows.push({
494
- id,
495
- ...(name ? { name } : {}),
496
- ...(typeof getPath(item, "downloads") === "number" ? { downloads: getPath(item, "downloads") } : {}),
497
- ...(typeof getPath(item, "likes") === "number" ? { likes: getPath(item, "likes") } : {}),
498
- ...(firstNonEmptyString(getPath(item, "pipeline_tag"), getPath(item, "task")) ? {
499
- pipeline_tag: firstNonEmptyString(getPath(item, "pipeline_tag"), getPath(item, "task")),
500
- } : {}),
501
- url: firstNonEmptyString(getPath(item, "url"), getPath(item, "link"))
502
- ?? `https://huggingface.co/${encodeURI(id)}`,
503
- });
504
- seen.add(id);
505
- }
506
-
507
- return rows;
508
- }
509
-
510
- function normalizeEmailRows(data: unknown): Record<string, unknown>[] {
511
- const sourceRows = Array.isArray(data)
512
- ? data
513
- : isRecord(data) && Array.isArray(data.results)
514
- ? data.results
515
- : isRecord(data) && Array.isArray(data.emails)
516
- ? data.emails
517
- : isRecord(data) && Array.isArray(data.messages)
518
- ? data.messages
519
- : [];
520
- const rows: Record<string, unknown>[] = [];
521
- const seen = new Set<string>();
522
-
523
- for (const item of sourceRows) {
524
- if (!isRecord(item)) continue;
525
- const thread = isRecord(getPath(item, "thread")) ? getPath(item, "thread") as Record<string, unknown> : item;
526
- const message = isRecord(getPath(item, "matchedEmail")) ? getPath(item, "matchedEmail") as Record<string, unknown> : item;
527
- const id = firstNonEmptyString(
528
- getPath(message, "id"),
529
- getPath(thread, "doc_id"),
530
- getPath(item, "id"),
531
- getPath(item, "doc_id"),
532
- );
533
- if (!id || seen.has(id)) continue;
534
- const subject = firstNonEmptyString(getPath(thread, "subject"), getPath(item, "subject"), getPath(message, "subject"));
535
- const from = firstNonEmptyString(
536
- getPath(message, "sender"),
537
- getPath(thread, "latest_sender_name"),
538
- getPath(item, "from"),
539
- getPath(item, "sender"),
540
- );
541
- const date = firstNonEmptyString(getPath(thread, "formatted_date"), getPath(item, "date"), getPath(message, "date"));
542
- const preview = firstNonEmptyString(
543
- getPath(thread, "preview"),
544
- getPath(message, "content_markdown"),
545
- getPath(item, "preview"),
546
- getPath(item, "snippet"),
547
- );
548
- if (!subject && !from) continue;
549
- rows.push({
550
- id,
551
- ...(subject ? { subject } : {}),
552
- ...(from ? { from } : {}),
553
- ...(date ? { date } : {}),
554
- ...(preview ? { preview } : {}),
555
- });
556
- seen.add(id);
557
- }
558
-
559
- return rows;
560
- }
561
-
562
- function normalizeProductRows(data: unknown): Record<string, unknown>[] {
563
- const rows: Record<string, unknown>[] = [];
564
- const seen = new Set<string>();
565
- const blockedTitles = new Set(["results", "more results", "related searches", "need help?"]);
566
-
567
- collectNestedObjects(data, (obj) => {
568
- const headingKeys = Object.keys(obj).filter((key) => /^heading_\d+$/i.test(key));
569
- const title = firstNonEmptyString(
570
- getPath(obj, "title"),
571
- getPath(obj, "name"),
572
- getPath(obj, "productName"),
573
- getPath(obj, "productTitle"),
574
- getPath(obj, "title.text"),
575
- );
576
- const url = firstNonEmptyString(
577
- getPath(obj, "url"),
578
- getPath(obj, "link"),
579
- getPath(obj, "href"),
580
- getPath(obj, "canonicalUrl"),
581
- getPath(obj, "productUrl"),
582
- getPath(obj, "productPageUrl"),
583
- );
584
- const id = firstNonEmptyString(
585
- getPath(obj, "id"),
586
- getPath(obj, "usItemId"),
587
- getPath(obj, "itemId"),
588
- getPath(obj, "productId"),
589
- getPath(obj, "sku"),
590
- );
591
- const priceString = firstNonEmptyString(
592
- getPath(obj, "price"),
593
- getPath(obj, "priceString"),
594
- getPath(obj, "price.currentPrice.priceString"),
595
- getPath(obj, "priceInfo.currentPrice.priceString"),
596
- getPath(obj, "secondaryOfferPrice.currentPrice.priceString"),
597
- );
598
- const priceNumber = getPath(obj, "price.currentPrice.price")
599
- ?? getPath(obj, "priceInfo.currentPrice.price")
600
- ?? getPath(obj, "currentPrice.price")
601
- ?? getPath(obj, "salePrice");
602
- const rating = getPath(obj, "averageRating")
603
- ?? getPath(obj, "rating")
604
- ?? getPath(obj, "rating.average")
605
- ?? getPath(obj, "customerRating")
606
- ?? getPath(obj, "reviews.averageRating");
607
- const reviewCount = getPath(obj, "numberOfReviews")
608
- ?? getPath(obj, "reviewCount")
609
- ?? getPath(obj, "reviews.count")
610
- ?? getPath(obj, "ratingsTotal");
611
- const image = firstNonEmptyString(
612
- getPath(obj, "image"),
613
- getPath(obj, "imageUrl"),
614
- getPath(obj, "thumbnail"),
615
- getPath(obj, "primaryImageUrl"),
616
- );
617
- const brand = firstNonEmptyString(getPath(obj, "brand"), getPath(obj, "brandName"), getPath(obj, "seller"));
618
-
619
- if (!title && !id) return;
620
- if (title && blockedTitles.has(title.trim().toLowerCase())) return;
621
- if (headingKeys.length >= 6) return;
622
- if (!url && !id) return;
623
- if (
624
- priceString == null &&
625
- typeof priceNumber !== "number" &&
626
- rating == null &&
627
- reviewCount == null &&
628
- !image &&
629
- !brand
630
- ) {
631
- return;
632
- }
633
- if (url && /(amazon-adsystem|doubleclick|googlesyndication|pubmatic|taboola|outbrain|aax-|googleadservices|adservice)\./i.test(url)) {
634
- return;
635
- }
636
-
637
- const stable = String(url ?? id ?? title);
638
- if (seen.has(stable)) return;
639
- rows.push({
640
- ...(id ? { id } : {}),
641
- ...(title ? { title } : {}),
642
- ...(url ? { url } : {}),
643
- ...(priceString ? { price: priceString } : typeof priceNumber === "number" ? { price: priceNumber } : {}),
644
- ...(typeof rating === "number" || hasNonEmptyString(rating) ? { rating } : {}),
645
- ...(typeof reviewCount === "number" || hasNonEmptyString(reviewCount) ? { review_count: reviewCount } : {}),
646
- ...(image ? { image } : {}),
647
- ...(brand ? { brand } : {}),
648
- });
649
- seen.add(stable);
650
- });
651
-
652
- return rows;
653
- }
654
-
655
- function normalizeStockQuote(data: unknown): Record<string, unknown> | null {
656
- let best: Record<string, unknown> | null = null;
657
-
658
- collectNestedObjects(data, (obj) => {
659
- const symbol = firstNonEmptyString(
660
- getPath(obj, "symbol"),
661
- getPath(obj, "ticker"),
662
- getPath(obj, "quote.symbol"),
663
- );
664
- const price = getPath(obj, "regularMarketPrice.raw")
665
- ?? getPath(obj, "regularMarketPrice")
666
- ?? getPath(obj, "postMarketPrice.raw")
667
- ?? getPath(obj, "price.regularMarketPrice.raw")
668
- ?? getPath(obj, "price.regularMarketPrice")
669
- ?? getPath(obj, "price.currentPrice.raw")
670
- ?? getPath(obj, "price.currentPrice");
671
- if (!symbol || typeof price !== "number") return;
672
- const name = firstNonEmptyString(
673
- getPath(obj, "shortName"),
674
- getPath(obj, "longName"),
675
- getPath(obj, "displayName"),
676
- getPath(obj, "price.shortName"),
677
- getPath(obj, "price.longName"),
678
- );
679
- const currency = firstNonEmptyString(getPath(obj, "currency"), getPath(obj, "financialCurrency"), getPath(obj, "price.currency"));
680
- const marketState = firstNonEmptyString(getPath(obj, "marketState"), getPath(obj, "price.marketState"));
681
- const changePercent = getPath(obj, "regularMarketChangePercent.raw")
682
- ?? getPath(obj, "regularMarketChangePercent")
683
- ?? getPath(obj, "price.regularMarketChangePercent.raw")
684
- ?? getPath(obj, "price.regularMarketChangePercent");
685
- const marketCap = getPath(obj, "marketCap.raw")
686
- ?? getPath(obj, "marketCap")
687
- ?? getPath(obj, "price.marketCap.raw")
688
- ?? getPath(obj, "price.marketCap");
689
- best = {
690
- symbol,
691
- ...(name ? { name } : {}),
692
- price,
693
- ...(currency ? { currency } : {}),
694
- ...(typeof changePercent === "number" ? { change_percent: changePercent } : {}),
695
- ...(typeof marketCap === "number" ? { market_cap: marketCap } : {}),
696
- ...(marketState ? { market_state: marketState } : {}),
697
- };
698
- });
699
-
700
- return best;
701
- }
702
-
703
- function normalizeChannelRows(data: unknown): Record<string, unknown>[] {
704
- const rows: Record<string, unknown>[] = [];
705
- const seen = new Set<string>();
706
-
707
- collectNestedObjects(data, (obj) => {
708
- const name = firstNonEmptyString(
709
- getPath(obj, "name"),
710
- getPath(obj, "title"),
711
- getPath(obj, "channel_name"),
712
- getPath(obj, "workspace_name"),
713
- getPath(obj, "team_name"),
714
- );
715
- const id = firstNonEmptyString(
716
- getPath(obj, "id"),
717
- getPath(obj, "channel_id"),
718
- getPath(obj, "guild_id"),
719
- getPath(obj, "server_id"),
720
- getPath(obj, "workspace_id"),
721
- getPath(obj, "team_id"),
722
- );
723
- const url = firstNonEmptyString(
724
- getPath(obj, "url"),
725
- getPath(obj, "channel_url"),
726
- getPath(obj, "server_url"),
727
- getPath(obj, "workspace_url"),
728
- );
729
- if (!name || (!id && !url)) return;
730
- const stable = String(id ?? url ?? name);
731
- if (seen.has(stable)) return;
732
- rows.push({
733
- ...(id ? { id } : {}),
734
- name,
735
- ...(url ? { url } : {}),
736
- ...(firstNonEmptyString(getPath(obj, "topic"), getPath(obj, "purpose"), getPath(obj, "description")) ? {
737
- description: firstNonEmptyString(getPath(obj, "topic"), getPath(obj, "purpose"), getPath(obj, "description")),
738
- } : {}),
739
- ...(typeof getPath(obj, "member_count") === "number" ? { member_count: getPath(obj, "member_count") } : {}),
740
- ...(firstNonEmptyString(getPath(obj, "type"), getPath(obj, "channel_type"), getPath(obj, "kind")) ? {
741
- type: firstNonEmptyString(getPath(obj, "type"), getPath(obj, "channel_type"), getPath(obj, "kind")),
742
- } : {}),
743
- });
744
- seen.add(stable);
745
- });
746
-
747
- return rows;
748
- }
749
-
750
- function collectNestedObjects(value: unknown, visit: (obj: Record<string, unknown>) => void): void {
751
- if (Array.isArray(value)) {
752
- for (const item of value) collectNestedObjects(item, visit);
753
- return;
754
- }
755
- if (!isRecord(value)) return;
756
- visit(value);
757
- for (const child of Object.values(value)) collectNestedObjects(child, visit);
758
- }
759
-
760
- function normalizeXTweets(data: unknown): Record<string, unknown>[] {
761
- const rows: Record<string, unknown>[] = [];
762
- const seen = new Set<string>();
763
- collectNestedObjects(data, (obj) => {
764
- const result = getPath(obj, "tweet_results.result");
765
- if (!isRecord(result)) return;
766
- const restId = getPath(result, "rest_id");
767
- const text = getPath(result, "legacy.full_text");
768
- const screenName = getPath(result, "core.user_results.result.core.screen_name")
769
- ?? getPath(result, "core.user_results.result.legacy.screen_name");
770
- if (!hasNonEmptyString(restId) || !hasNonEmptyString(text) || !hasNonEmptyString(screenName)) return;
771
- if (seen.has(String(restId))) return;
772
- rows.push({
773
- id: String(restId),
774
- username: String(screenName),
775
- text: String(text),
776
- url: `https://x.com/${encodeURIComponent(String(screenName))}/status/${encodeURIComponent(String(restId))}`,
777
- });
778
- seen.add(String(restId));
779
- });
780
- return rows;
781
- }
782
-
783
- function normalizeXUsers(data: unknown): Record<string, unknown>[] {
784
- const rows: Record<string, unknown>[] = [];
785
- const seen = new Set<string>();
786
- collectNestedObjects(data, (obj) => {
787
- const result = getPath(obj, "user_results.result");
788
- const target = isRecord(result) ? result : obj;
789
- const screenName = getPath(target, "core.screen_name") ?? getPath(target, "legacy.screen_name");
790
- const name = getPath(target, "core.name") ?? getPath(target, "legacy.name");
791
- const description = getPath(target, "legacy.description");
792
- const followers = getPath(target, "legacy.followers_count");
793
- if (!hasNonEmptyString(screenName) || !hasNonEmptyString(name)) return;
794
- if (seen.has(String(screenName))) return;
795
- rows.push({
796
- username: String(screenName),
797
- public_identifier: String(screenName),
798
- name: String(name),
799
- ...(hasNonEmptyString(description) ? { description: String(description) } : {}),
800
- ...(typeof followers === "number" ? { followers_count: followers } : {}),
801
- url: `https://x.com/${encodeURIComponent(String(screenName))}`,
802
- });
803
- seen.add(String(screenName));
804
- });
805
- return rows;
806
- }
807
-
808
- function joinNameParts(...values: unknown[]): string | undefined {
809
- const parts = values.filter((value) => hasNonEmptyString(value)).map((value) => String(value).trim());
810
- return parts.length > 0 ? parts.join(" ") : undefined;
811
- }
812
-
813
- function absoluteLinkedInUrl(value: string | undefined): string | undefined {
814
- if (!value) return undefined;
815
- return value.startsWith("http") ? value : `https://www.linkedin.com${value.startsWith("/") ? "" : "/"}${value}`;
816
- }
817
-
818
- function normalizeLinkedInFeedPosts(data: unknown): Record<string, unknown>[] {
819
- if (!isRecord(data)) return [];
820
-
821
- const included = Array.isArray(getPath(data, "included"))
822
- ? getPath(data, "included") as unknown[]
823
- : Array.isArray(getPath(data, "data.included"))
824
- ? getPath(data, "data.included") as unknown[]
825
- : [];
826
- if (included.length === 0) return [];
827
-
828
- const elementRefs = Array.isArray(getPath(data, "data.data.feedDashMainFeedByMainFeed.*elements"))
829
- ? getPath(data, "data.data.feedDashMainFeedByMainFeed.*elements") as unknown[]
830
- : Array.isArray(getPath(data, "data.feedDashMainFeedByMainFeed.*elements"))
831
- ? getPath(data, "data.feedDashMainFeedByMainFeed.*elements") as unknown[]
832
- : Array.isArray(getPath(data, "feedDashMainFeedByMainFeed.*elements"))
833
- ? getPath(data, "feedDashMainFeedByMainFeed.*elements") as unknown[]
834
- : [];
835
- if (elementRefs.length === 0) return [];
836
-
837
- const entityIndex = new Map<string, Record<string, unknown>>();
838
- for (const item of included) {
839
- if (!isRecord(item)) continue;
840
- const urn = getPath(item, "entityUrn");
841
- if (!hasNonEmptyString(urn)) continue;
842
- entityIndex.set(String(urn), item);
843
- }
844
-
845
- const rows: Record<string, unknown>[] = [];
846
- const seen = new Set<string>();
847
-
848
- for (const ref of elementRefs) {
849
- if (!hasNonEmptyString(ref) || seen.has(String(ref))) continue;
850
- const update = entityIndex.get(String(ref));
851
- if (!update) continue;
852
-
853
- const actorProfileUrn = firstNonEmptyString(getPath(update, "actor.*profileUrn"), getPath(update, "actor.entityUrn"));
854
- const actorProfile = actorProfileUrn ? entityIndex.get(actorProfileUrn) : undefined;
855
- const author = firstNonEmptyString(
856
- getPath(update, "actor.name.text"),
857
- getPath(actorProfile, "name"),
858
- joinNameParts(getPath(actorProfile, "firstName"), getPath(actorProfile, "lastName")),
859
- );
860
- const username = firstNonEmptyString(getPath(actorProfile, "publicIdentifier"));
861
- const content = firstNonEmptyString(
862
- getPath(update, "commentary.text.text"),
863
- getPath(update, "commentary.accessibilityText"),
864
- getPath(update, "content.text"),
865
- getPath(update, "content.title"),
866
- getPath(update, "header.text"),
867
- getPath(update, "header.title"),
868
- );
869
- const permalink = firstNonEmptyString(
870
- getPath(update, "permalink"),
871
- getPath(update, "socialContent.shareUrl"),
872
- getPath(update, "url"),
873
- );
874
- const url = absoluteLinkedInUrl(permalink) ?? `https://www.linkedin.com/feed/update/${encodeURIComponent(String(ref))}/`;
875
-
876
- if (!content && !username && !author) continue;
877
-
878
- rows.push({
879
- id: String(ref),
880
- url,
881
- ...(content ? { content } : {}),
882
- ...(author ? { author } : {}),
883
- ...(username ? { username, public_identifier: username } : {}),
884
- ...(typeof getPath(update, "createdAt") === "number" ? { created_at: getPath(update, "createdAt") } : {}),
885
- });
886
- seen.add(String(ref));
887
- }
888
-
889
- return rows;
890
- }
891
-
892
- function normalizeGenericPeopleRows(data: unknown): Record<string, unknown>[] {
893
- const sourceRows = Array.isArray(data)
894
- ? data
895
- : isRecord(data) && Array.isArray(data.results)
896
- ? data.results
897
- : [];
898
- const rows: Record<string, unknown>[] = [];
899
- const seen = new Set<string>();
900
-
901
- for (const item of sourceRows) {
902
- if (!isRecord(item)) continue;
903
- if (
904
- hasAnyPath(item, ["full_name", "repository.name", "path_with_namespace", "stargazers_count", "stars", "star_count", "forks_count", "version", "price"])
905
- ) {
906
- continue;
907
- }
908
- const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "title"));
909
- const headline = firstNonEmptyString(
910
- getPath(item, "headline"),
911
- getPath(item, "description"),
912
- getPath(item, "subtitle"),
913
- getPath(item, "bio"),
914
- );
915
- const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "link"));
916
- const publicIdentifier = firstNonEmptyString(getPath(item, "public_identifier"), getPath(item, "username"));
917
- if (!name || name.includes("/") || name.includes("://")) continue;
918
- if (!headline && !url && !publicIdentifier) continue;
919
- const id = publicIdentifier ?? url ?? name;
920
- if (seen.has(id)) continue;
921
- rows.push({
922
- name,
923
- ...(headline ? { headline } : {}),
924
- ...(url ? { url } : {}),
925
- ...(publicIdentifier ? { public_identifier: publicIdentifier } : {}),
926
- });
927
- seen.add(id);
928
- }
929
-
930
- return rows;
931
- }
932
-
933
- function normalizeRedditPosts(data: unknown): Record<string, unknown>[] {
934
- const rows: Record<string, unknown>[] = [];
935
- const seen = new Set<string>();
936
- collectNestedObjects(data, (obj) => {
937
- const kind = getPath(obj, "kind");
938
- const payload = isRecord(getPath(obj, "data")) ? getPath(obj, "data") : obj;
939
- if (!isRecord(payload)) return;
940
- const title = getPath(payload, "title");
941
- const author = getPath(payload, "author");
942
- const permalink = getPath(payload, "permalink");
943
- if (!hasNonEmptyString(title) || !hasNonEmptyString(author) || !hasNonEmptyString(permalink)) return;
944
- if (hasNonEmptyString(kind) && kind !== "t3") return;
945
- const id = getPath(payload, "id") ?? getPath(payload, "name") ?? permalink;
946
- if (!hasNonEmptyString(id) || seen.has(String(id))) return;
947
- rows.push({
948
- id: String(id),
949
- title: String(title),
950
- author: String(author),
951
- ...(typeof getPath(payload, "score") === "number" ? { score: getPath(payload, "score") } : {}),
952
- ...(typeof getPath(payload, "num_comments") === "number" ? { num_comments: getPath(payload, "num_comments") } : {}),
953
- ...(hasNonEmptyString(getPath(payload, "subreddit")) ? { subreddit: String(getPath(payload, "subreddit")) } : {}),
954
- permalink: String(permalink),
955
- url: `https://www.reddit.com${String(permalink)}`,
956
- });
957
- seen.add(String(id));
958
- });
959
- return rows;
960
- }
961
-
962
- function normalizeRedditComments(data: unknown): Record<string, unknown>[] {
963
- const rows: Record<string, unknown>[] = [];
964
- const seen = new Set<string>();
965
- collectNestedObjects(data, (obj) => {
966
- const kind = getPath(obj, "kind");
967
- const payload = isRecord(getPath(obj, "data")) ? getPath(obj, "data") : obj;
968
- if (!isRecord(payload)) return;
969
- const body = getPath(payload, "body");
970
- const author = getPath(payload, "author");
971
- const permalink = getPath(payload, "permalink");
972
- if (!hasNonEmptyString(body) || !hasNonEmptyString(author) || !hasNonEmptyString(permalink)) return;
973
- if (hasNonEmptyString(kind) && kind !== "t1") return;
974
- const id = getPath(payload, "id") ?? getPath(payload, "name") ?? permalink;
975
- if (!hasNonEmptyString(id) || seen.has(String(id))) return;
976
- rows.push({
977
- id: String(id),
978
- author: String(author),
979
- body: String(body),
980
- permalink: String(permalink),
981
- url: `https://www.reddit.com${String(permalink)}`,
982
- });
983
- seen.add(String(id));
984
- });
985
- return rows;
986
- }
987
-
988
- function normalizeCompanies(data: unknown): Record<string, unknown>[] {
989
- const rows: Record<string, unknown>[] = [];
990
- const seen = new Set<string>();
991
- collectNestedObjects(data, (obj) => {
992
- const name = getPath(obj, "name") ?? getPath(obj, "title");
993
- const publicIdentifier =
994
- getPath(obj, "public_identifier") ??
995
- getPath(obj, "vanityName") ??
996
- getPath(obj, "universalName");
997
- const description =
998
- getPath(obj, "description") ??
999
- getPath(obj, "tagline") ??
1000
- getPath(obj, "headline");
1001
- const industry = getPath(obj, "industry") ?? getPath(obj, "primaryIndustry");
1002
- const employeeCount =
1003
- getPath(obj, "employee_count") ??
1004
- getPath(obj, "staffCount") ??
1005
- getPath(obj, "employeeCount");
1006
- const followerCount =
1007
- getPath(obj, "follower_count") ??
1008
- getPath(obj, "followerCount");
1009
- const website = getPath(obj, "website") ?? getPath(obj, "websiteUrl");
1010
- const url =
1011
- getPath(obj, "url") ??
1012
- (hasNonEmptyString(publicIdentifier)
1013
- ? `https://www.linkedin.com/company/${encodeURIComponent(String(publicIdentifier))}/`
1014
- : undefined);
1015
- if (!hasNonEmptyString(name)) return;
1016
- if (
1017
- !hasNonEmptyString(description) &&
1018
- !hasNonEmptyString(industry) &&
1019
- typeof employeeCount !== "number" &&
1020
- typeof followerCount !== "number" &&
1021
- !hasNonEmptyString(website) &&
1022
- !hasNonEmptyString(url) &&
1023
- !hasNonEmptyString(publicIdentifier)
1024
- ) {
1025
- return;
1026
- }
1027
- const id = String(publicIdentifier ?? url ?? name);
1028
- if (seen.has(id)) return;
1029
- rows.push({
1030
- name: String(name),
1031
- ...(hasNonEmptyString(publicIdentifier) ? { public_identifier: String(publicIdentifier) } : {}),
1032
- ...(hasNonEmptyString(description) ? { description: String(description) } : {}),
1033
- ...(hasNonEmptyString(industry) ? { industry: String(industry) } : {}),
1034
- ...(typeof employeeCount === "number" ? { employee_count: employeeCount } : {}),
1035
- ...(typeof followerCount === "number" ? { follower_count: followerCount } : {}),
1036
- ...(hasNonEmptyString(website) ? { website: String(website) } : {}),
1037
- ...(hasNonEmptyString(url) ? { url: String(url) } : {}),
1038
- });
1039
- seen.add(id);
1040
- });
1041
- return rows;
1042
- }
1043
-
1044
- function unwrapCarrier(data: unknown): unknown {
1045
- if (!isRecord(data)) return data;
1046
- const keys = Object.keys(data);
1047
- const isCarrierOnly = keys.every((key) => key === "data" || key === "_extraction");
1048
- if (isCarrierOnly && "data" in data && ("_extraction" in data || Array.isArray(data.data) || isRecord(data.data))) {
1049
- return unwrapCarrier(data.data);
1050
- }
1051
- return data;
1052
- }
1053
-
1054
- export function projectIntentData(data: unknown, intent?: string): unknown {
1055
- const unwrapped = unwrapCarrier(data);
1056
- if (!intent) return unwrapped;
1057
- const lower = intent.toLowerCase();
1058
-
1059
- if (/\b(stock|stocks|ticker|tickers|quote|quotes)\b/.test(lower)) {
1060
- const normalizedQuote = normalizeStockQuote(unwrapped);
1061
- if (normalizedQuote) return normalizedQuote;
1062
- }
1063
-
1064
- if (/\b(product|products|item|items)\b/.test(lower)) {
1065
- const normalizedProducts = normalizeProductRows(unwrapped);
1066
- if (normalizedProducts.length > 0) return normalizedProducts;
1067
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).products)) {
1068
- return (unwrapped as Record<string, unknown>).products;
1069
- }
1070
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).items)) {
1071
- return (unwrapped as Record<string, unknown>).items;
1072
- }
1073
- }
1074
-
1075
- if (/\b(channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(lower)) {
1076
- const normalizedChannels = normalizeChannelRows(unwrapped);
1077
- if (normalizedChannels.length > 0) return normalizedChannels;
1078
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).channels)) {
1079
- return (unwrapped as Record<string, unknown>).channels;
1080
- }
1081
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).guilds)) {
1082
- return (unwrapped as Record<string, unknown>).guilds;
1083
- }
1084
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).workspaces)) {
1085
- return (unwrapped as Record<string, unknown>).workspaces;
1086
- }
1087
- }
1088
-
1089
- if (/\b(package|packages)\b/.test(lower)) {
1090
- if (/\bsearch\b/.test(lower)) {
1091
- const normalizedCratesSearch = normalizeCratesPackageSearchResults(unwrapped);
1092
- if (normalizedCratesSearch.length > 0) return normalizedCratesSearch;
1093
- const normalizedPackageSearch = normalizePackageSearchResults(unwrapped);
1094
- if (normalizedPackageSearch.length > 0) return normalizedPackageSearch;
1095
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).packages)) {
1096
- return (unwrapped as Record<string, unknown>).packages;
1097
- }
1098
- } else {
1099
- const normalizedNpmPackage = normalizeNpmPackageInfo(unwrapped);
1100
- if (normalizedNpmPackage) return normalizedNpmPackage;
1101
- const normalizedPyPIPackage = normalizePyPIPackageInfo(unwrapped);
1102
- if (normalizedPyPIPackage) return normalizedPyPIPackage;
1103
- const normalizedPubDevPackage = normalizePubDevPackageInfo(unwrapped);
1104
- if (normalizedPubDevPackage) return normalizedPubDevPackage;
1105
- const normalizedRubyGemsPackage = normalizeRubyGemsPackageInfo(unwrapped);
1106
- if (normalizedRubyGemsPackage) return normalizedRubyGemsPackage;
1107
- if (isRecord(unwrapped) && isRecord((unwrapped as Record<string, unknown>).package)) {
1108
- return (unwrapped as Record<string, unknown>).package;
1109
- }
1110
- }
1111
- }
1112
-
1113
- if (/\b(doc|docs|documentation)\b/.test(lower)) {
1114
- const normalizedDocuments = normalizeDocumentRows(unwrapped);
1115
- if (normalizedDocuments.length > 0) return normalizedDocuments;
1116
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).documents)) {
1117
- return (unwrapped as Record<string, unknown>).documents;
1118
- }
1119
- }
1120
-
1121
- if (/\bsearch images\b/.test(lower)) {
1122
- const normalizedImageSearch = normalizeDockerImageSearchResults(unwrapped);
1123
- if (normalizedImageSearch.length > 0) return normalizedImageSearch;
1124
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).images)) {
1125
- return (unwrapped as Record<string, unknown>).images;
1126
- }
1127
- }
1128
-
1129
- if (/\b(get )?image tags\b/.test(lower)) {
1130
- const normalizedImageTags = normalizeDockerTagResults(unwrapped);
1131
- if (normalizedImageTags.length > 0) return normalizedImageTags;
1132
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).tags)) {
1133
- return (unwrapped as Record<string, unknown>).tags;
1134
- }
1135
- }
1136
-
1137
- if (/\b(comment|comments)\b/.test(lower)) {
1138
- const normalizedRedditComments = normalizeRedditComments(unwrapped);
1139
- if (normalizedRedditComments.length > 0) return normalizedRedditComments;
1140
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).comments)) {
1141
- return (unwrapped as Record<string, unknown>).comments;
1142
- }
1143
- }
1144
-
1145
- if (/\b(reddit|subreddit)\b/.test(lower)) {
1146
- const normalizedRedditPosts = normalizeRedditPosts(unwrapped);
1147
- if (normalizedRedditPosts.length > 0) return normalizedRedditPosts;
1148
- }
1149
-
1150
- if (/\b(company|companies|organization|organisations|business)\b/.test(lower)) {
1151
- const normalizedCompanies = normalizeCompanies(unwrapped);
1152
- if (normalizedCompanies.length === 1) return normalizedCompanies[0];
1153
- if (normalizedCompanies.length > 1) return normalizedCompanies;
1154
- if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).companies)) {
1155
- return (unwrapped as Record<string, unknown>).companies;
1156
- }
1157
- }
1158
-
1159
- if (!isRecord(unwrapped) && !Array.isArray(unwrapped)) return unwrapped;
1160
-
1161
- if (/\b(post|posts|tweet|tweets|status|statuses)\b/.test(lower)) {
1162
- const normalizedDevPosts = normalizeDevToPosts(unwrapped);
1163
- if (normalizedDevPosts.length > 0) return normalizedDevPosts;
1164
- const normalizedLobsters = normalizeLobstersPosts(unwrapped);
1165
- if (normalizedLobsters.length > 0) return normalizedLobsters;
1166
- const normalizedLinkedInFeed = normalizeLinkedInFeedPosts(unwrapped);
1167
- if (normalizedLinkedInFeed.length > 0) return normalizedLinkedInFeed;
1168
- const normalizedRedditPosts = normalizeRedditPosts(unwrapped);
1169
- if (normalizedRedditPosts.length > 0) return normalizedRedditPosts;
1170
- const normalizedXTweets = normalizeXTweets(unwrapped);
1171
- if (normalizedXTweets.length > 0) return normalizedXTweets;
1172
- if (Array.isArray(unwrapped)) return unwrapped;
1173
- if (Array.isArray(unwrapped.statuses)) return unwrapped.statuses;
1174
- if (Array.isArray(unwrapped.posts)) return unwrapped.posts;
1175
- if (Array.isArray(unwrapped.tweets)) return unwrapped.tweets;
1176
- }
1177
-
1178
- if (/\b(person|people|user|users|profile|profiles|member|members)\b/.test(lower)) {
1179
- if (Array.isArray(unwrapped.people)) return unwrapped.people;
1180
- if (Array.isArray(unwrapped.users)) return unwrapped.users;
1181
- if (Array.isArray(unwrapped.accounts)) return unwrapped.accounts;
1182
- if (Array.isArray(unwrapped.elements)) return unwrapped.elements;
1183
- if (Array.isArray(unwrapped.included)) return unwrapped.included;
1184
- const normalizedXUsers = normalizeXUsers(unwrapped);
1185
- if (normalizedXUsers.length > 0) return normalizedXUsers;
1186
- const normalizedGenericPeople = normalizeGenericPeopleRows(unwrapped);
1187
- if (normalizedGenericPeople.length > 0) return normalizedGenericPeople;
1188
- }
1189
-
1190
- if (/\b(topic|topics|trend|trending|hashtag|hashtags)\b/.test(lower)) {
1191
- const storyItems = getPath(unwrapped, "story_topic.stories.items");
1192
- if (Array.isArray(storyItems)) {
1193
- const normalized = storyItems
1194
- .map((item) => {
1195
- const core = getPath(item, "trend_results.result.core");
1196
- const name = getPath(core, "name");
1197
- const restId = getPath(item, "trend_results.result.rest_id");
1198
- if (!hasNonEmptyString(name)) return null;
1199
- return {
1200
- name,
1201
- ...(hasNonEmptyString(getPath(core, "category")) ? { category: getPath(core, "category") } : {}),
1202
- ...(hasNonEmptyString(getPath(item, "trend_results.result.post_count")) ? { post_count: getPath(item, "trend_results.result.post_count") } : {}),
1203
- ...(hasNonEmptyString(restId)
1204
- ? { url: `https://x.com/search?q=${encodeURIComponent(String(name))}&src=trend_click&trend_id=${encodeURIComponent(String(restId))}` }
1205
- : { url: `https://x.com/search?q=${encodeURIComponent(String(name))}` }),
1206
- };
1207
- })
1208
- .filter((item): item is Record<string, unknown> => !!item);
1209
- if (normalized.length > 0) return normalized;
1210
- }
1211
- if (Array.isArray(unwrapped.topics)) return unwrapped.topics;
1212
- if (Array.isArray(unwrapped.trends)) return unwrapped.trends;
1213
- if (Array.isArray(unwrapped.hashtags)) return unwrapped.hashtags;
1214
- }
1215
-
1216
- if (/\b(model|models)\b/.test(lower)) {
1217
- const normalizedModels = normalizeHuggingFaceModels(unwrapped);
1218
- if (normalizedModels.length > 0) return normalizedModels;
1219
- if (Array.isArray(unwrapped.models)) return unwrapped.models;
1220
- }
1221
-
1222
- if (/\b(news|story|stories|article|articles|hacker news)\b/.test(lower)) {
1223
- if (Array.isArray(unwrapped)) return unwrapped;
1224
- const normalizedStories = normalizeHackerNewsStories(unwrapped);
1225
- if (normalizedStories.length > 0) return normalizedStories;
1226
- if (Array.isArray(unwrapped.hits)) return unwrapped.hits;
1227
- if (Array.isArray(unwrapped.stories)) return unwrapped.stories;
1228
- if (Array.isArray(unwrapped.articles)) return unwrapped.articles;
1229
- }
1230
-
1231
- if (/\b(question|questions)\b/.test(lower)) {
1232
- if (Array.isArray(unwrapped)) return unwrapped;
1233
- const normalizedQuestions = normalizeStackExchangeQuestions(unwrapped);
1234
- if (normalizedQuestions.length > 0) return normalizedQuestions;
1235
- if (Array.isArray(unwrapped.items)) return unwrapped.items;
1236
- }
1237
-
1238
- if (/\b(email|emails|mail|inbox)\b/.test(lower)) {
1239
- const normalizedEmails = normalizeEmailRows(unwrapped);
1240
- if (normalizedEmails.length > 0) return normalizedEmails;
1241
- if (Array.isArray(unwrapped.emails)) return unwrapped.emails;
1242
- if (Array.isArray(unwrapped.messages)) return unwrapped.messages;
1243
- if (Array.isArray(unwrapped.results)) return unwrapped.results;
1244
- }
1245
-
1246
- if (/\b(repo|repository|repositories|project|projects)\b/.test(lower)) {
1247
- if (Array.isArray(unwrapped.repositories)) return unwrapped.repositories;
1248
- if (Array.isArray(unwrapped.items)) return unwrapped.items;
1249
- if (Array.isArray(unwrapped.projects)) return unwrapped.projects;
1250
- }
1251
-
1252
- return unwrapped;
1253
- }
1254
-
1255
- function classifyRows(rows: unknown[], intent: string): { verdict: "pass" | "fail" | "skip"; reason: string } {
1256
- if (rows.length === 0) return { verdict: "fail", reason: "empty_array" };
1257
- if (rows.every((row) => !isRecord(row))) return { verdict: "fail", reason: "primitive_rows" };
1258
-
1259
- const objects = rows.filter(isRecord);
1260
- if (objects.length === 0) return { verdict: "fail", reason: "primitive_rows" };
1261
-
1262
- const lower = intent.toLowerCase();
1263
-
1264
- if (/\b(repo|repository|repositories|project|projects)\b/.test(lower)) {
1265
- const matching = objects.filter((row) =>
1266
- hasAnyPath(row, ["full_name", "name", "repository.name", "path_with_namespace"]) &&
1267
- hasAnyPath(row, ["url", "web_url", "description", "stargazers_count", "stars", "star_count", "forks_count", "owner.login", "owner"])
1268
- );
1269
- return matching.length >= 1 ? { verdict: "pass", reason: "repository_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1270
- }
1271
-
1272
- if (/\b(package|packages)\b/.test(lower)) {
1273
- const matching = objects.filter((row) =>
1274
- hasAnyPath(row, ["name", "package.name"]) &&
1275
- hasAnyPath(row, ["version", "description", "summary", "url", "keywords", "requires_dist", "dependencies"])
1276
- );
1277
- return matching.length >= 1 ? { verdict: "pass", reason: "package_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1278
- }
1279
-
1280
- if (/\b(model|models)\b/.test(lower)) {
1281
- const matching = objects.filter((row) =>
1282
- hasAnyPath(row, ["id", "modelId", "name"]) &&
1283
- hasAnyPath(row, ["downloads", "likes", "pipeline_tag", "url"])
1284
- );
1285
- return matching.length >= 1 ? { verdict: "pass", reason: "model_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1286
- }
1287
-
1288
- if (/\b(news|story|stories|article|articles|hacker news)\b/.test(lower)) {
1289
- const matching = objects.filter((row) =>
1290
- hasAnyPath(row, ["objectID", "id", "url", "link"]) &&
1291
- hasAnyPath(row, ["title", "story_title", "author", "points", "num_comments", "meta"])
1292
- );
1293
- return matching.length >= 1 ? { verdict: "pass", reason: "story_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1294
- }
1295
-
1296
- if (/\bsearch images\b/.test(lower)) {
1297
- const matching = objects.filter((row) =>
1298
- hasAnyPath(row, ["repo_name", "name", "full_name"]) &&
1299
- hasAnyPath(row, ["short_description", "description", "star_count", "pull_count", "url"])
1300
- );
1301
- return matching.length >= 1 ? { verdict: "pass", reason: "image_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1302
- }
1303
-
1304
- if (/\b(get )?image tags\b/.test(lower)) {
1305
- const matching = objects.filter((row) =>
1306
- hasAnyPath(row, ["name", "tag"]) &&
1307
- hasAnyPath(row, ["last_updated", "updated_at", "full_size", "digest"])
1308
- );
1309
- return matching.length >= 1 ? { verdict: "pass", reason: "tag_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1310
- }
1311
-
1312
- if (/\b(company|companies|organization|organisations|business)\b/.test(lower)) {
1313
- const matching = objects.filter((row) =>
1314
- hasAnyPath(row, ["name", "title", "public_identifier"]) &&
1315
- hasAnyPath(row, ["description", "industry", "url", "website", "employee_count", "follower_count"])
1316
- );
1317
- return matching.length >= 1 ? { verdict: "pass", reason: "company_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1318
- }
1319
-
1320
- if (/\b(person|people|profile|profiles|member|members|user|users)\b/.test(lower)) {
1321
- const matching = objects.filter((row) =>
1322
- hasAnyPath(row, ["name", "title", "public_identifier", "username"]) &&
1323
- hasAnyPath(row, ["headline", "url", "public_identifier", "username"])
1324
- );
1325
- const minRows = /\b(profile|profiles|user|users|person)\b/.test(lower) && !/\bsearch\b/.test(lower) ? 1 : 2;
1326
- return matching.length >= minRows ? { verdict: "pass", reason: "people_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1327
- }
1328
-
1329
- if (/\b(comment|comments)\b/.test(lower)) {
1330
- const matching = objects.filter((row) =>
1331
- hasAnyPath(row, ["id", "url", "permalink"]) &&
1332
- hasAnyPath(row, ["author", "body", "text"])
1333
- );
1334
- return matching.length >= 1 ? { verdict: "pass", reason: "comment_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1335
- }
1336
-
1337
- if (/\b(email|emails|mail|inbox)\b/.test(lower)) {
1338
- const matching = objects.filter((row) =>
1339
- hasAnyPath(row, ["id", "thread.doc_id", "matchedEmail.id"]) &&
1340
- hasAnyPath(row, ["subject", "thread.subject", "from", "sender", "thread.latest_sender_name"]) &&
1341
- hasAnyPath(row, ["date", "thread.formatted_date", "preview", "thread.preview", "matchedEmail.content_markdown"])
1342
- );
1343
- return matching.length >= 1 ? { verdict: "pass", reason: "email_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1344
- }
1345
-
1346
- if (/\b(post|posts|tweet|tweets|status|statuses)\b/.test(lower)) {
1347
- const matching = objects.filter((row) =>
1348
- hasAnyPath(row, ["id", "url", "uri", "permalink", "_id", "entityUrn", "rest_id"]) &&
1349
- countMatchingGroups(row, [
1350
- ["content", "text", "title", "body", "message", "description", "summary", "commentary.text.text"],
1351
- ["account.username", "account.acct", "username", "author", "user.name", "from.name", "actor.name", "actor.name.text"],
1352
- ["score", "points", "likes", "favorite_count", "num_comments", "reply_count", "reactions", "repost_count"],
1353
- ["date", "created_at", "published_at", "timestamp", "meta", "created_time", "createdAt", "updated_at", "time"],
1354
- ]) >= 2
1355
- );
1356
- return matching.length >= 1 ? { verdict: "pass", reason: "post_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1357
- }
1358
-
1359
- if (/\b(reddit|subreddit)\b/.test(lower)) {
1360
- const matching = objects.filter((row) =>
1361
- hasAnyPath(row, ["id", "url", "permalink"]) &&
1362
- hasAnyPath(row, ["title", "subreddit", "author"]) &&
1363
- hasAnyPath(row, ["num_comments", "score", "permalink"])
1364
- );
1365
- return matching.length >= 1 ? { verdict: "pass", reason: "reddit_post_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1366
- }
1367
-
1368
- if (/\b(topic|topics|trend|trending|hashtag|hashtags)\b/.test(lower)) {
1369
- const matching = objects.filter((row) =>
1370
- hasAnyPath(row, ["name", "title", "query"]) &&
1371
- hasAnyPath(row, ["url", "name", "query"])
1372
- );
1373
- return matching.length >= 2 ? { verdict: "pass", reason: "topic_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1374
- }
1375
-
1376
- if (/\b(doc|docs|documentation)\b/.test(lower)) {
1377
- const matching = objects.filter((row) =>
1378
- hasAnyPath(row, ["title", "name", "slug"]) &&
1379
- hasAnyPath(row, ["url", "link", "href", "mdn_url"]) &&
1380
- hasAnyPath(row, ["summary", "description", "slug", "popularity", "score"])
1381
- );
1382
- return matching.length >= 1 ? { verdict: "pass", reason: "document_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1383
- }
1384
-
1385
- if (/\b(paper|papers)\b/.test(lower)) {
1386
- const matching = objects.filter((row) =>
1387
- hasAnyPath(row, ["title", "name"]) &&
1388
- hasAnyPath(row, ["url", "link", "href"]) &&
1389
- hasAnyPath(row, ["summary", "description", "author", "authors", "date", "published", "meta"])
1390
- );
1391
- return matching.length >= 1 ? { verdict: "pass", reason: "paper_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1392
- }
1393
-
1394
- if (/\b(question|questions)\b/.test(lower)) {
1395
- const matching = objects.filter((row) =>
1396
- hasAnyPath(row, ["title", "name"]) &&
1397
- hasAnyPath(row, ["url", "link", "href", "permalink"]) &&
1398
- countMatchingGroups(row, [
1399
- ["votes", "score", "points"],
1400
- ["answers", "answer_count", "num_answers", "num_comments"],
1401
- ["author", "date", "created_at", "meta"],
1402
- ]) >= 2
1403
- );
1404
- return matching.length >= 2 ? { verdict: "pass", reason: "question_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1405
- }
1406
-
1407
- if (/\b(recipe|recipes)\b/.test(lower)) {
1408
- const matching = objects.filter((row) =>
1409
- hasAnyPath(row, ["title", "name"]) &&
1410
- hasAnyPath(row, ["url", "link", "href"]) &&
1411
- hasAnyPath(row, ["rating", "review_count", "author", "description", "summary", "cook_time", "total_time"])
1412
- );
1413
- return matching.length >= 1 ? { verdict: "pass", reason: "recipe_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1414
- }
1415
-
1416
- if (/\b(course|courses)\b/.test(lower)) {
1417
- const matching = objects.filter((row) =>
1418
- hasAnyPath(row, ["title", "name"]) &&
1419
- hasAnyPath(row, ["url", "link", "href"]) &&
1420
- hasAnyPath(row, ["rating", "partner", "provider", "instructor", "description", "summary", "duration", "level"])
1421
- );
1422
- return matching.length >= 1 ? { verdict: "pass", reason: "course_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1423
- }
1424
-
1425
- if (/\b(definition|dictionary|meaning)\b/.test(lower)) {
1426
- const matching = objects.filter((row) =>
1427
- hasAnyPath(row, ["term", "title", "name", "word"]) &&
1428
- hasAnyPath(row, ["definition", "summary", "body", "description"])
1429
- );
1430
- return matching.length >= 1 ? { verdict: "pass", reason: "definition_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1431
- }
1432
-
1433
- if (/\b(stock|stocks|ticker|tickers|quote|quotes)\b/.test(lower)) {
1434
- const matching = objects.filter((row) =>
1435
- hasAnyPath(row, ["symbol", "ticker"]) &&
1436
- hasAnyPath(row, ["price", "regularMarketPrice", "current_price"]) &&
1437
- hasAnyPath(row, ["name", "currency", "change_percent", "market_cap", "market_state"])
1438
- );
1439
- return matching.length >= 1 ? { verdict: "pass", reason: "quote_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1440
- }
1441
-
1442
- if (/\b(product|products|item|items)\b/.test(lower)) {
1443
- const matching = objects.filter((row) =>
1444
- hasAnyPath(row, ["title", "name", "product_name"]) &&
1445
- (
1446
- hasAnyPath(row, ["url", "id", "product_id", "sku"]) ||
1447
- hasAnyPath(row, ["description", "summary"])
1448
- ) &&
1449
- hasAnyPath(row, ["price", "rating", "review_count", "brand", "image"]) &&
1450
- !/^results?$/i.test(String(getPath(row, "title") ?? getPath(row, "name") ?? "")) &&
1451
- !Object.keys(row).some((key) => /^heading_\d+$/i.test(key)) &&
1452
- !/(amazon-adsystem|doubleclick|googlesyndication|pubmatic|taboola|outbrain|aax-|googleadservices|adservice)/i.test(
1453
- String(getPath(row, "url") ?? ""),
1454
- )
1455
- );
1456
- return matching.length >= 1 ? { verdict: "pass", reason: "product_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1457
- }
1458
-
1459
- if (/\b(channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(lower)) {
1460
- const matching = objects.filter((row) =>
1461
- hasAnyPath(row, ["name", "title"]) &&
1462
- hasAnyPath(row, ["id", "url", "channel_id", "guild_id", "workspace_id"]) &&
1463
- hasAnyPath(row, ["description", "type", "member_count", "url"])
1464
- );
1465
- return matching.length >= 1 ? { verdict: "pass", reason: "channel_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
1466
- }
1467
-
1468
- return { verdict: "skip", reason: "unclassified_array" };
1469
- }
1470
-
1471
- export function assessIntentResult(data: unknown, intent?: string): {
1472
- verdict: "pass" | "fail" | "skip";
1473
- reason: string;
1474
- projected: unknown;
1475
- } {
1476
- const projected = projectIntentData(data, intent);
1477
-
1478
- if (projected == null) return { verdict: "fail", reason: "no_data", projected };
1479
- if (typeof projected === "string") {
1480
- const trimmed = projected.trim().toLowerCase();
1481
- if (!trimmed) return { verdict: "fail", reason: "empty_text", projected };
1482
- if (trimmed.startsWith("<!doctype") || trimmed.startsWith("<html") || trimmed.startsWith("<body")) {
1483
- return { verdict: "fail", reason: "html_payload", projected };
1484
- }
1485
- return { verdict: "skip", reason: "plain_text", projected };
1486
- }
1487
-
1488
- if (Array.isArray(projected)) {
1489
- const classified = classifyRows(projected, intent ?? "");
1490
- return { ...classified, projected };
1491
- }
1492
-
1493
- if (isRecord(projected)) {
1494
- if ("error" in projected) return { verdict: "fail", reason: String(projected.error || "error_payload"), projected };
1495
- if ("available_endpoints" in projected || "available_operations" in projected) {
1496
- return { verdict: "fail", reason: "deferral_payload", projected };
1497
- }
1498
- if ("learned_skill_id" in projected && !("data" in projected)) {
1499
- return { verdict: "fail", reason: "learned_skill_only", projected };
1500
- }
1501
- const lower = (intent ?? "").toLowerCase();
1502
- if (("message" in projected || "flash" in projected) && !("data" in projected)) {
1503
- if (/\b(message|messages|flash|alert|success|error|warning)\b/.test(lower)) {
1504
- return { verdict: "skip", reason: "message_record", projected };
1505
- }
1506
- return { verdict: "fail", reason: "message_only", projected };
1507
- }
1508
- if (/\b(company|companies|organization|organisations|business|person|people|profile|profiles|member|members|user|users|repo|repository|repositories|project|projects|package|packages|doc|docs|documentation|question|questions|recipe|recipes|course|courses|definition|dictionary|meaning|product|products|item|items|stock|stocks|ticker|tickers|quote|quotes|channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(lower)) {
1509
- const classified = classifyRows([projected], intent ?? "");
1510
- return { ...classified, projected };
1511
- }
1512
- }
1513
-
1514
- return { verdict: "skip", reason: "unclassified_payload", projected };
1515
- }