unbrowse 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +455 -96
- package/dist/index.js +2 -6
- package/dist/mcp.js +695 -46
- package/dist/server.js +25811 -0
- package/package.json +1 -2
- package/vendor/kuri/darwin-arm64/kuri +0 -0
- package/vendor/kuri/darwin-x64/kuri +0 -0
- package/vendor/kuri/linux-arm64/kuri +0 -0
- package/vendor/kuri/linux-x64/kuri +0 -0
- package/vendor/kuri/manifest.json +7 -10
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/analytics-session.ts +0 -55
- package/runtime-src/api/browse-index.ts +0 -317
- package/runtime-src/api/browse-session.ts +0 -572
- package/runtime-src/api/browse-submit-prereqs.ts +0 -48
- package/runtime-src/api/browse-submit.ts +0 -1184
- package/runtime-src/api/routes.ts +0 -1823
- package/runtime-src/auth/browser-cookies.ts +0 -423
- package/runtime-src/auth/index.ts +0 -535
- package/runtime-src/auth/runtime.ts +0 -116
- package/runtime-src/browser/index.ts +0 -659
- package/runtime-src/browser/types.ts +0 -41
- package/runtime-src/build-info.generated.ts +0 -6
- package/runtime-src/capture/index.ts +0 -1794
- package/runtime-src/capture/prefetch.ts +0 -95
- package/runtime-src/capture/rsc.ts +0 -45
- package/runtime-src/cli/shortcuts.ts +0 -273
- package/runtime-src/cli.ts +0 -1572
- package/runtime-src/client/graph-client.ts +0 -100
- package/runtime-src/client/index.ts +0 -1425
- package/runtime-src/debug-trace.ts +0 -18
- package/runtime-src/domain.ts +0 -38
- package/runtime-src/execution/index.ts +0 -3397
- package/runtime-src/execution/retry.ts +0 -46
- package/runtime-src/execution/robots.ts +0 -167
- package/runtime-src/execution/search-forms.ts +0 -188
- package/runtime-src/extraction/index.ts +0 -1507
- package/runtime-src/foundry/publish-bundle.ts +0 -392
- package/runtime-src/graph/agent-augment.ts +0 -315
- package/runtime-src/graph/index.ts +0 -1524
- package/runtime-src/graph/local-fixtures.ts +0 -393
- package/runtime-src/graph/local-harness.ts +0 -646
- package/runtime-src/graph/planner.ts +0 -411
- package/runtime-src/graph/session.ts +0 -294
- package/runtime-src/graph/trace-store.ts +0 -136
- package/runtime-src/index.ts +0 -24
- package/runtime-src/indexer/index.ts +0 -465
- package/runtime-src/intent-match.ts +0 -1515
- package/runtime-src/kuri/client.ts +0 -1839
- package/runtime-src/logger.ts +0 -30
- package/runtime-src/marketplace/index.ts +0 -103
- package/runtime-src/mcp.ts +0 -1747
- package/runtime-src/orchestrator/browser-agent.ts +0 -374
- package/runtime-src/orchestrator/dag-advisor.ts +0 -59
- package/runtime-src/orchestrator/dag-feedback.ts +0 -257
- package/runtime-src/orchestrator/first-pass-action.ts +0 -403
- package/runtime-src/orchestrator/index.ts +0 -4480
- package/runtime-src/orchestrator/passive-publish.ts +0 -187
- package/runtime-src/orchestrator/timing-economics.ts +0 -80
- package/runtime-src/payments/cascade.ts +0 -137
- package/runtime-src/payments/index.ts +0 -270
- package/runtime-src/payments/lobster-pay.ts +0 -182
- package/runtime-src/payments/wallet.ts +0 -98
- package/runtime-src/publish/review-context.ts +0 -93
- package/runtime-src/publish/sanitize.ts +0 -197
- package/runtime-src/publish/schema-review.ts +0 -192
- package/runtime-src/publish-admission.ts +0 -388
- package/runtime-src/ratelimit/index.ts +0 -23
- package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
- package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
- package/runtime-src/reverse-engineer/index.ts +0 -1551
- package/runtime-src/router.ts +0 -17
- package/runtime-src/routing-telemetry.ts +0 -395
- package/runtime-src/runtime/browser-access.ts +0 -11
- package/runtime-src/runtime/browser-auth.ts +0 -12
- package/runtime-src/runtime/browser-host.ts +0 -48
- package/runtime-src/runtime/lifecycle.ts +0 -17
- package/runtime-src/runtime/local-server.ts +0 -311
- package/runtime-src/runtime/paths.ts +0 -99
- package/runtime-src/runtime/setup.ts +0 -251
- package/runtime-src/runtime/supervisor.ts +0 -69
- package/runtime-src/runtime/update-hints.ts +0 -351
- package/runtime-src/server.ts +0 -100
- package/runtime-src/session-logs.ts +0 -142
- package/runtime-src/settings.ts +0 -221
- package/runtime-src/single-binary.ts +0 -143
- package/runtime-src/site-policy.ts +0 -54
- package/runtime-src/stale-cleanup-runner.ts +0 -144
- package/runtime-src/stale-cleanup.ts +0 -133
- package/runtime-src/telemetry-attribution.ts +0 -120
- package/runtime-src/telemetry.ts +0 -253
- package/runtime-src/template-params.ts +0 -141
- package/runtime-src/transform/drift.ts +0 -60
- package/runtime-src/transform/index.ts +0 -277
- package/runtime-src/types/index.ts +0 -1
- package/runtime-src/types/skill.ts +0 -912
- package/runtime-src/vault/index.ts +0 -196
- package/runtime-src/verification/auth-gate.ts +0 -8
- package/runtime-src/verification/candidates.ts +0 -27
- package/runtime-src/verification/index.ts +0 -120
- package/runtime-src/verification/matrix.ts +0 -30
- package/runtime-src/version.ts +0 -148
- package/runtime-src/workflow/artifact.ts +0 -161
- package/runtime-src/workflow/compile.ts +0 -808
- package/runtime-src/workflow/publish.ts +0 -225
- package/runtime-src/workflow/runtime.ts +0 -213
- package/vendor/kuri/win-x64/kuri.exe +0 -0
|
@@ -1,1515 +0,0 @@
|
|
|
1
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
2
|
-
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
3
|
-
}
|
|
4
|
-
|
|
5
|
-
function getPath(obj: unknown, path: string): unknown {
|
|
6
|
-
if (!isRecord(obj)) return undefined;
|
|
7
|
-
let cur: unknown = obj;
|
|
8
|
-
for (const key of path.split(".")) {
|
|
9
|
-
if (!isRecord(cur)) return undefined;
|
|
10
|
-
cur = cur[key];
|
|
11
|
-
}
|
|
12
|
-
return cur;
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
function hasNonEmptyString(value: unknown): boolean {
|
|
16
|
-
return typeof value === "string" && value.trim().length > 0;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
function hasAnyPath(obj: unknown, paths: string[]): boolean {
|
|
20
|
-
return paths.some((path) => {
|
|
21
|
-
const value = getPath(obj, path);
|
|
22
|
-
return hasNonEmptyString(value) || typeof value === "number";
|
|
23
|
-
});
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
function countMatchingGroups(obj: unknown, groups: string[][]): number {
|
|
27
|
-
let matches = 0;
|
|
28
|
-
for (const group of groups) {
|
|
29
|
-
if (hasAnyPath(obj, group)) matches++;
|
|
30
|
-
}
|
|
31
|
-
return matches;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
function toStringArray(value: unknown): string[] | undefined {
|
|
35
|
-
if (Array.isArray(value)) {
|
|
36
|
-
const out = value
|
|
37
|
-
.filter((item): item is string => hasNonEmptyString(item))
|
|
38
|
-
.map((item) => String(item).trim());
|
|
39
|
-
return out.length > 0 ? out : undefined;
|
|
40
|
-
}
|
|
41
|
-
if (hasNonEmptyString(value)) return [String(value).trim()];
|
|
42
|
-
return undefined;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
function firstNonEmptyString(...values: unknown[]): string | undefined {
|
|
46
|
-
for (const value of values) {
|
|
47
|
-
if (hasNonEmptyString(value)) return String(value).trim();
|
|
48
|
-
}
|
|
49
|
-
return undefined;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
function normalizePackageSearchResults(data: unknown): Record<string, unknown>[] {
|
|
53
|
-
const sourceRows = Array.isArray(data)
|
|
54
|
-
? data
|
|
55
|
-
: isRecord(data) && Array.isArray(data.objects)
|
|
56
|
-
? data.objects
|
|
57
|
-
: isRecord(data) && Array.isArray(data.packages)
|
|
58
|
-
? data.packages
|
|
59
|
-
: [];
|
|
60
|
-
const rows: Record<string, unknown>[] = [];
|
|
61
|
-
const seen = new Set<string>();
|
|
62
|
-
|
|
63
|
-
for (const item of sourceRows) {
|
|
64
|
-
if (!isRecord(item)) continue;
|
|
65
|
-
const pkg = isRecord(item.package) ? item.package : item;
|
|
66
|
-
const name = getPath(pkg, "name");
|
|
67
|
-
if (!hasNonEmptyString(name) || seen.has(String(name))) continue;
|
|
68
|
-
const version = firstNonEmptyString(getPath(pkg, "version"));
|
|
69
|
-
const description = firstNonEmptyString(getPath(pkg, "description"), getPath(pkg, "summary"));
|
|
70
|
-
const url =
|
|
71
|
-
firstNonEmptyString(
|
|
72
|
-
getPath(pkg, "links.npm"),
|
|
73
|
-
getPath(pkg, "project_url"),
|
|
74
|
-
getPath(pkg, "package_url"),
|
|
75
|
-
getPath(pkg, "url"),
|
|
76
|
-
) ?? `https://www.npmjs.com/package/${encodeURIComponent(String(name))}`;
|
|
77
|
-
const keywords = toStringArray(getPath(pkg, "keywords"));
|
|
78
|
-
rows.push({
|
|
79
|
-
name: String(name),
|
|
80
|
-
...(version ? { version } : {}),
|
|
81
|
-
...(description ? { description } : {}),
|
|
82
|
-
...(keywords ? { keywords } : {}),
|
|
83
|
-
url,
|
|
84
|
-
});
|
|
85
|
-
seen.add(String(name));
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
return rows;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
function normalizeCratesPackageSearchResults(data: unknown): Record<string, unknown>[] {
|
|
92
|
-
const sourceRows = isRecord(data) && Array.isArray(data.crates)
|
|
93
|
-
? data.crates
|
|
94
|
-
: Array.isArray(data)
|
|
95
|
-
? data
|
|
96
|
-
: [];
|
|
97
|
-
const rows: Record<string, unknown>[] = [];
|
|
98
|
-
const seen = new Set<string>();
|
|
99
|
-
|
|
100
|
-
for (const item of sourceRows) {
|
|
101
|
-
if (!isRecord(item)) continue;
|
|
102
|
-
const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "id"));
|
|
103
|
-
if (!name || seen.has(name)) continue;
|
|
104
|
-
const description = firstNonEmptyString(getPath(item, "description"), getPath(item, "summary"));
|
|
105
|
-
const version = firstNonEmptyString(getPath(item, "max_version"), getPath(item, "default_version"), getPath(item, "version"));
|
|
106
|
-
rows.push({
|
|
107
|
-
name,
|
|
108
|
-
...(description ? { description, summary: description } : {}),
|
|
109
|
-
...(version ? { version } : {}),
|
|
110
|
-
...(typeof getPath(item, "downloads") === "number" ? { downloads: getPath(item, "downloads") } : {}),
|
|
111
|
-
url: `https://crates.io/crates/${encodeURIComponent(name)}`,
|
|
112
|
-
});
|
|
113
|
-
seen.add(name);
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
return rows;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
function normalizeNpmPackageInfo(data: unknown): Record<string, unknown> | null {
|
|
120
|
-
if (!isRecord(data) || !hasNonEmptyString(data.name)) return null;
|
|
121
|
-
const distTags = isRecord(data["dist-tags"]) ? data["dist-tags"] : undefined;
|
|
122
|
-
const versions = isRecord(data.versions) ? data.versions : undefined;
|
|
123
|
-
const latestVersion = firstNonEmptyString(distTags?.latest, data.version);
|
|
124
|
-
const latestRecord =
|
|
125
|
-
latestVersion && versions && isRecord(versions[latestVersion])
|
|
126
|
-
? versions[latestVersion]
|
|
127
|
-
: undefined;
|
|
128
|
-
const description = firstNonEmptyString(
|
|
129
|
-
getPath(latestRecord, "description"),
|
|
130
|
-
getPath(data, "description"),
|
|
131
|
-
);
|
|
132
|
-
const homepage = firstNonEmptyString(
|
|
133
|
-
getPath(latestRecord, "homepage"),
|
|
134
|
-
getPath(data, "homepage"),
|
|
135
|
-
getPath(latestRecord, "repository.url"),
|
|
136
|
-
getPath(data, "repository.url"),
|
|
137
|
-
);
|
|
138
|
-
const dependencies = isRecord(getPath(latestRecord, "dependencies"))
|
|
139
|
-
? Object.keys(getPath(latestRecord, "dependencies") as Record<string, unknown>)
|
|
140
|
-
: undefined;
|
|
141
|
-
const keywords = toStringArray(getPath(latestRecord, "keywords") ?? getPath(data, "keywords"));
|
|
142
|
-
const author = firstNonEmptyString(
|
|
143
|
-
getPath(latestRecord, "author.name"),
|
|
144
|
-
getPath(latestRecord, "author"),
|
|
145
|
-
getPath(data, "author.name"),
|
|
146
|
-
getPath(data, "author"),
|
|
147
|
-
);
|
|
148
|
-
|
|
149
|
-
return {
|
|
150
|
-
name: String(data.name),
|
|
151
|
-
...(latestVersion ? { version: latestVersion } : {}),
|
|
152
|
-
...(description ? { description } : {}),
|
|
153
|
-
...(keywords ? { keywords } : {}),
|
|
154
|
-
...(dependencies && dependencies.length > 0 ? { dependencies } : {}),
|
|
155
|
-
...(author ? { author } : {}),
|
|
156
|
-
url: homepage ?? firstNonEmptyString(getPath(data, "url"), getPath(data, "package_url"), getPath(data, "project_url"))
|
|
157
|
-
?? `https://www.npmjs.com/package/${encodeURIComponent(String(data.name))}`,
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
function normalizeDocumentRows(data: unknown): Record<string, unknown>[] {
|
|
162
|
-
const sourceRows = Array.isArray(data)
|
|
163
|
-
? data
|
|
164
|
-
: isRecord(data) && Array.isArray(data.documents)
|
|
165
|
-
? data.documents
|
|
166
|
-
: isRecord(data) && Array.isArray(data.results)
|
|
167
|
-
? data.results
|
|
168
|
-
: [];
|
|
169
|
-
const rows: Record<string, unknown>[] = [];
|
|
170
|
-
const seen = new Set<string>();
|
|
171
|
-
|
|
172
|
-
for (const item of sourceRows) {
|
|
173
|
-
if (!isRecord(item)) continue;
|
|
174
|
-
const title = firstNonEmptyString(getPath(item, "title"), getPath(item, "name"));
|
|
175
|
-
const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "mdn_url"), getPath(item, "link"), getPath(item, "href"));
|
|
176
|
-
if (!title || !url) continue;
|
|
177
|
-
const id = `${title}|${url}`;
|
|
178
|
-
if (seen.has(id)) continue;
|
|
179
|
-
rows.push({
|
|
180
|
-
title,
|
|
181
|
-
url,
|
|
182
|
-
...(firstNonEmptyString(getPath(item, "summary"), getPath(item, "description"), getPath(item, "slug")) ? {
|
|
183
|
-
summary: firstNonEmptyString(getPath(item, "summary"), getPath(item, "description"), getPath(item, "slug")),
|
|
184
|
-
} : {}),
|
|
185
|
-
...(typeof getPath(item, "score") === "number" ? { score: getPath(item, "score") } : {}),
|
|
186
|
-
...(typeof getPath(item, "popularity") === "number" ? { popularity: getPath(item, "popularity") } : {}),
|
|
187
|
-
...(firstNonEmptyString(getPath(item, "locale")) ? { locale: firstNonEmptyString(getPath(item, "locale")) } : {}),
|
|
188
|
-
});
|
|
189
|
-
seen.add(id);
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
return rows;
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
function normalizePyPIPackageInfo(data: unknown): Record<string, unknown> | null {
|
|
196
|
-
if (!isRecord(data)) return null;
|
|
197
|
-
const info = isRecord(data.info) ? data.info : data;
|
|
198
|
-
const name = getPath(info, "name");
|
|
199
|
-
if (!hasNonEmptyString(name)) return null;
|
|
200
|
-
const summary = firstNonEmptyString(getPath(info, "summary"), getPath(info, "description"));
|
|
201
|
-
const version = firstNonEmptyString(getPath(info, "version"));
|
|
202
|
-
const author = firstNonEmptyString(getPath(info, "author"));
|
|
203
|
-
const requiresDist = Array.isArray(getPath(info, "requires_dist"))
|
|
204
|
-
? (getPath(info, "requires_dist") as unknown[]).filter((item) => hasNonEmptyString(item)).map(String)
|
|
205
|
-
: undefined;
|
|
206
|
-
const url = firstNonEmptyString(
|
|
207
|
-
getPath(info, "project_url"),
|
|
208
|
-
getPath(info, "package_url"),
|
|
209
|
-
getPath(info, "home_page"),
|
|
210
|
-
getPath(info, "project_urls.Homepage"),
|
|
211
|
-
);
|
|
212
|
-
|
|
213
|
-
return {
|
|
214
|
-
name: String(name),
|
|
215
|
-
...(version ? { version } : {}),
|
|
216
|
-
...(summary ? { summary, description: summary } : {}),
|
|
217
|
-
...(author ? { author } : {}),
|
|
218
|
-
...(requiresDist && requiresDist.length > 0 ? { requires_dist: requiresDist } : {}),
|
|
219
|
-
...(url ? { url } : {}),
|
|
220
|
-
};
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
function normalizePubDevPackageInfo(data: unknown): Record<string, unknown> | null {
|
|
224
|
-
if (!isRecord(data)) return null;
|
|
225
|
-
const latest = isRecord(data.latest) ? data.latest : undefined;
|
|
226
|
-
const pubspec = isRecord(latest?.pubspec) ? latest?.pubspec : undefined;
|
|
227
|
-
const name = firstNonEmptyString(getPath(data, "name"), getPath(pubspec, "name"));
|
|
228
|
-
if (!name) return null;
|
|
229
|
-
const version = firstNonEmptyString(getPath(latest, "version"), getPath(pubspec, "version"));
|
|
230
|
-
const description = firstNonEmptyString(getPath(pubspec, "description"));
|
|
231
|
-
const repository = firstNonEmptyString(getPath(pubspec, "repository"), getPath(pubspec, "homepage"));
|
|
232
|
-
const topics = toStringArray(getPath(pubspec, "topics"));
|
|
233
|
-
return {
|
|
234
|
-
name,
|
|
235
|
-
...(version ? { version } : {}),
|
|
236
|
-
...(description ? { description, summary: description } : {}),
|
|
237
|
-
...(topics ? { keywords: topics } : {}),
|
|
238
|
-
url: repository ?? `https://pub.dev/packages/${encodeURIComponent(name)}`,
|
|
239
|
-
};
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
function normalizeRubyGemsPackageInfo(data: unknown): Record<string, unknown> | null {
|
|
243
|
-
if (!isRecord(data)) return null;
|
|
244
|
-
const name = firstNonEmptyString(getPath(data, "name"));
|
|
245
|
-
if (!name) return null;
|
|
246
|
-
const description = firstNonEmptyString(getPath(data, "info"));
|
|
247
|
-
const version = firstNonEmptyString(getPath(data, "version"));
|
|
248
|
-
const author = firstNonEmptyString(getPath(data, "authors"), getPath(data, "author"));
|
|
249
|
-
const homepage = firstNonEmptyString(getPath(data, "homepage_uri"), getPath(data, "project_uri"), getPath(data, "gem_uri"));
|
|
250
|
-
return {
|
|
251
|
-
name,
|
|
252
|
-
...(version ? { version } : {}),
|
|
253
|
-
...(description ? { description, summary: description } : {}),
|
|
254
|
-
...(author ? { author } : {}),
|
|
255
|
-
url: homepage ?? `https://rubygems.org/gems/${encodeURIComponent(name)}`,
|
|
256
|
-
};
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
function normalizeDockerImageSearchResults(data: unknown): Record<string, unknown>[] {
|
|
260
|
-
const sourceRows = isRecord(data) && Array.isArray(data.results)
|
|
261
|
-
? data.results
|
|
262
|
-
: Array.isArray(data)
|
|
263
|
-
? data
|
|
264
|
-
: [];
|
|
265
|
-
const rows: Record<string, unknown>[] = [];
|
|
266
|
-
const seen = new Set<string>();
|
|
267
|
-
|
|
268
|
-
for (const item of sourceRows) {
|
|
269
|
-
if (!isRecord(item)) continue;
|
|
270
|
-
const repoName = firstNonEmptyString(getPath(item, "repo_name"), getPath(item, "name"));
|
|
271
|
-
if (!repoName || seen.has(repoName)) continue;
|
|
272
|
-
rows.push({
|
|
273
|
-
repo_name: repoName,
|
|
274
|
-
...(firstNonEmptyString(getPath(item, "short_description"), getPath(item, "description")) ? {
|
|
275
|
-
short_description: firstNonEmptyString(getPath(item, "short_description"), getPath(item, "description")),
|
|
276
|
-
} : {}),
|
|
277
|
-
...(typeof getPath(item, "star_count") === "number" ? { star_count: getPath(item, "star_count") } : {}),
|
|
278
|
-
...(typeof getPath(item, "pull_count") === "number" ? { pull_count: getPath(item, "pull_count") } : {}),
|
|
279
|
-
url: `https://hub.docker.com/r/${repoName}`,
|
|
280
|
-
});
|
|
281
|
-
seen.add(repoName);
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
return rows;
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
function normalizeDockerTagResults(data: unknown): Record<string, unknown>[] {
|
|
288
|
-
const sourceRows = isRecord(data) && Array.isArray(data.results)
|
|
289
|
-
? data.results
|
|
290
|
-
: Array.isArray(data)
|
|
291
|
-
? data
|
|
292
|
-
: [];
|
|
293
|
-
const rows: Record<string, unknown>[] = [];
|
|
294
|
-
const seen = new Set<string>();
|
|
295
|
-
|
|
296
|
-
for (const item of sourceRows) {
|
|
297
|
-
if (!isRecord(item)) continue;
|
|
298
|
-
const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "tag"));
|
|
299
|
-
if (!name || seen.has(name)) continue;
|
|
300
|
-
rows.push({
|
|
301
|
-
name,
|
|
302
|
-
...(typeof getPath(item, "full_size") === "number" ? { full_size: getPath(item, "full_size") } : {}),
|
|
303
|
-
...(firstNonEmptyString(getPath(item, "last_updated"), getPath(item, "updated_at")) ? {
|
|
304
|
-
last_updated: firstNonEmptyString(getPath(item, "last_updated"), getPath(item, "updated_at")),
|
|
305
|
-
} : {}),
|
|
306
|
-
...(firstNonEmptyString(getPath(item, "digest")) ? { digest: firstNonEmptyString(getPath(item, "digest")) } : {}),
|
|
307
|
-
});
|
|
308
|
-
seen.add(name);
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
return rows;
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
function normalizeHackerNewsStories(data: unknown): Record<string, unknown>[] {
|
|
315
|
-
const sourceRows = isRecord(data) && Array.isArray(data.hits)
|
|
316
|
-
? data.hits
|
|
317
|
-
: Array.isArray(data)
|
|
318
|
-
? data
|
|
319
|
-
: [];
|
|
320
|
-
const rows: Record<string, unknown>[] = [];
|
|
321
|
-
const seen = new Set<string>();
|
|
322
|
-
|
|
323
|
-
for (const item of sourceRows) {
|
|
324
|
-
if (!isRecord(item)) continue;
|
|
325
|
-
const id = firstNonEmptyString(getPath(item, "objectID"), getPath(item, "id"), getPath(item, "story_id"));
|
|
326
|
-
const title = firstNonEmptyString(getPath(item, "title"), getPath(item, "story_title"));
|
|
327
|
-
const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "story_url"), getPath(item, "link"));
|
|
328
|
-
const author = firstNonEmptyString(getPath(item, "author"));
|
|
329
|
-
const meta = firstNonEmptyString(getPath(item, "meta"));
|
|
330
|
-
if (!title) continue;
|
|
331
|
-
const stableId = id ?? url ?? title;
|
|
332
|
-
if (seen.has(stableId)) continue;
|
|
333
|
-
const pointsMatch = meta?.match(/(^|\|)\s*([0-9,]+)\s+points?\b/i);
|
|
334
|
-
const commentsMatch = meta?.match(/(^|\|)\s*([0-9,]+)\s+comments?\b/i);
|
|
335
|
-
|
|
336
|
-
rows.push({
|
|
337
|
-
...(stableId ? { id: stableId } : {}),
|
|
338
|
-
title,
|
|
339
|
-
...(url ? { url } : {}),
|
|
340
|
-
...(author ? { author } : {}),
|
|
341
|
-
...(typeof getPath(item, "points") === "number"
|
|
342
|
-
? { points: getPath(item, "points") }
|
|
343
|
-
: pointsMatch ? { points: Number(pointsMatch[2]?.replace(/,/g, "")) } : {}),
|
|
344
|
-
...(typeof getPath(item, "num_comments") === "number"
|
|
345
|
-
? { num_comments: getPath(item, "num_comments") }
|
|
346
|
-
: commentsMatch ? { num_comments: Number(commentsMatch[2]?.replace(/,/g, "")) } : {}),
|
|
347
|
-
...(meta ? { meta } : {}),
|
|
348
|
-
});
|
|
349
|
-
seen.add(stableId);
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
return rows;
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
function normalizeStackExchangeQuestions(data: unknown): Record<string, unknown>[] {
|
|
356
|
-
const sourceRows = isRecord(data) && Array.isArray(data.items)
|
|
357
|
-
? data.items
|
|
358
|
-
: Array.isArray(data)
|
|
359
|
-
? data
|
|
360
|
-
: [];
|
|
361
|
-
const rows: Record<string, unknown>[] = [];
|
|
362
|
-
const seen = new Set<string>();
|
|
363
|
-
for (const item of sourceRows) {
|
|
364
|
-
if (!isRecord(item)) continue;
|
|
365
|
-
const title = firstNonEmptyString(getPath(item, "title"));
|
|
366
|
-
const url = firstNonEmptyString(getPath(item, "link"));
|
|
367
|
-
if (!title || !url || seen.has(url)) continue;
|
|
368
|
-
rows.push({
|
|
369
|
-
title,
|
|
370
|
-
url,
|
|
371
|
-
...(typeof getPath(item, "score") === "number" ? { score: getPath(item, "score") } : {}),
|
|
372
|
-
...(typeof getPath(item, "answer_count") === "number" ? { answer_count: getPath(item, "answer_count") } : {}),
|
|
373
|
-
...(firstNonEmptyString(getPath(item, "owner.display_name")) ? { author: firstNonEmptyString(getPath(item, "owner.display_name")) } : {}),
|
|
374
|
-
...(typeof getPath(item, "last_activity_date") === "number" ? { date: getPath(item, "last_activity_date") } : {}),
|
|
375
|
-
});
|
|
376
|
-
seen.add(url);
|
|
377
|
-
}
|
|
378
|
-
return rows;
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
function normalizeDevToPosts(data: unknown): Record<string, unknown>[] {
|
|
382
|
-
const sourceRows = Array.isArray(data)
|
|
383
|
-
? data
|
|
384
|
-
: isRecord(data) && Array.isArray(data.result)
|
|
385
|
-
? data.result
|
|
386
|
-
: [];
|
|
387
|
-
const rows: Record<string, unknown>[] = [];
|
|
388
|
-
const seen = new Set<string>();
|
|
389
|
-
const inferAuthor = (value: unknown): string | undefined => {
|
|
390
|
-
const raw = hasNonEmptyString(value) ? String(value) : undefined;
|
|
391
|
-
if (!raw) return undefined;
|
|
392
|
-
try {
|
|
393
|
-
const pathname = raw.startsWith("http") ? new URL(raw).pathname : raw;
|
|
394
|
-
const segments = pathname.split("/").filter(Boolean);
|
|
395
|
-
const candidate = segments[0];
|
|
396
|
-
if (!candidate || candidate === "t" || candidate === "s" || candidate === "tag" || candidate === "tags" || candidate === "top" || candidate === "latest") {
|
|
397
|
-
return undefined;
|
|
398
|
-
}
|
|
399
|
-
return candidate;
|
|
400
|
-
} catch {
|
|
401
|
-
return undefined;
|
|
402
|
-
}
|
|
403
|
-
};
|
|
404
|
-
for (const item of sourceRows) {
|
|
405
|
-
if (!isRecord(item)) continue;
|
|
406
|
-
const title = firstNonEmptyString(getPath(item, "title"));
|
|
407
|
-
const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "path"));
|
|
408
|
-
if (!title || !url) continue;
|
|
409
|
-
const looksLikeDevTo = (
|
|
410
|
-
firstNonEmptyString(getPath(item, "type_of")) === "article" ||
|
|
411
|
-
hasNonEmptyString(getPath(item, "user.name")) ||
|
|
412
|
-
hasNonEmptyString(getPath(item, "user.username")) ||
|
|
413
|
-
typeof getPath(item, "positive_reactions_count") === "number" ||
|
|
414
|
-
typeof getPath(item, "comments_count") === "number" ||
|
|
415
|
-
hasNonEmptyString(getPath(item, "published_at")) ||
|
|
416
|
-
hasNonEmptyString(getPath(item, "readable_publish_date")) ||
|
|
417
|
-
url.startsWith("https://dev.to/") ||
|
|
418
|
-
url.startsWith("http://dev.to/") ||
|
|
419
|
-
(url.startsWith("/") && !!inferAuthor(url))
|
|
420
|
-
);
|
|
421
|
-
if (!looksLikeDevTo) continue;
|
|
422
|
-
const canonicalUrl = url.startsWith("http") ? url : `https://dev.to${url}`;
|
|
423
|
-
if (seen.has(canonicalUrl)) continue;
|
|
424
|
-
const author = firstNonEmptyString(
|
|
425
|
-
getPath(item, "user.name"),
|
|
426
|
-
getPath(item, "user.username"),
|
|
427
|
-
inferAuthor(getPath(item, "path")),
|
|
428
|
-
inferAuthor(canonicalUrl),
|
|
429
|
-
);
|
|
430
|
-
rows.push({
|
|
431
|
-
...(firstNonEmptyString(getPath(item, "id")) ? { id: firstNonEmptyString(getPath(item, "id")) } : {}),
|
|
432
|
-
title,
|
|
433
|
-
url: canonicalUrl,
|
|
434
|
-
...(firstNonEmptyString(getPath(item, "description")) ? { description: firstNonEmptyString(getPath(item, "description")) } : {}),
|
|
435
|
-
...(author ? { author } : {}),
|
|
436
|
-
...(typeof getPath(item, "positive_reactions_count") === "number" ? { likes: getPath(item, "positive_reactions_count") } : {}),
|
|
437
|
-
...(typeof getPath(item, "comments_count") === "number" ? { comments: getPath(item, "comments_count") } : {}),
|
|
438
|
-
...(firstNonEmptyString(getPath(item, "published_at"), getPath(item, "readable_publish_date")) ? { date: firstNonEmptyString(getPath(item, "published_at"), getPath(item, "readable_publish_date")) } : {}),
|
|
439
|
-
});
|
|
440
|
-
seen.add(canonicalUrl);
|
|
441
|
-
}
|
|
442
|
-
return rows;
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
function normalizeLobstersPosts(data: unknown): Record<string, unknown>[] {
|
|
446
|
-
const sourceRows = Array.isArray(data)
|
|
447
|
-
? data
|
|
448
|
-
: isRecord(data) && Array.isArray(data.results)
|
|
449
|
-
? data.results
|
|
450
|
-
: [];
|
|
451
|
-
const rows: Record<string, unknown>[] = [];
|
|
452
|
-
const seen = new Set<string>();
|
|
453
|
-
for (const item of sourceRows) {
|
|
454
|
-
if (!isRecord(item)) continue;
|
|
455
|
-
const title = firstNonEmptyString(getPath(item, "title"));
|
|
456
|
-
const rawUrl = firstNonEmptyString(getPath(item, "url"), getPath(item, "link"), getPath(item, "href"));
|
|
457
|
-
const text = firstNonEmptyString(getPath(item, "text"), getPath(item, "body"));
|
|
458
|
-
if (!title || !rawUrl || /^\d+\s+comments?$/i.test(title)) continue;
|
|
459
|
-
const scoreMatch = text?.match(/^\s*(\d{1,4})\s+/)?.[1];
|
|
460
|
-
const score = typeof getPath(item, "score") === "number"
|
|
461
|
-
? getPath(item, "score")
|
|
462
|
-
: typeof getPath(item, "points") === "number"
|
|
463
|
-
? getPath(item, "points")
|
|
464
|
-
: scoreMatch ? Number(scoreMatch) : undefined;
|
|
465
|
-
const url = rawUrl.startsWith("http") ? rawUrl : `https://lobste.rs${rawUrl.startsWith("/") ? "" : "/"}${rawUrl}`;
|
|
466
|
-
const stable = `${title}|${url}`;
|
|
467
|
-
if (seen.has(stable)) continue;
|
|
468
|
-
rows.push({
|
|
469
|
-
title,
|
|
470
|
-
url,
|
|
471
|
-
...(typeof score === "number" ? { score } : {}),
|
|
472
|
-
...(firstNonEmptyString(getPath(item, "author"), getPath(item, "username")) ? { author: firstNonEmptyString(getPath(item, "author"), getPath(item, "username")) } : {}),
|
|
473
|
-
});
|
|
474
|
-
seen.add(stable);
|
|
475
|
-
}
|
|
476
|
-
return rows;
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
function normalizeHuggingFaceModels(data: unknown): Record<string, unknown>[] {
|
|
480
|
-
const sourceRows = Array.isArray(data)
|
|
481
|
-
? data
|
|
482
|
-
: isRecord(data) && Array.isArray(data.models)
|
|
483
|
-
? data.models
|
|
484
|
-
: [];
|
|
485
|
-
const rows: Record<string, unknown>[] = [];
|
|
486
|
-
const seen = new Set<string>();
|
|
487
|
-
|
|
488
|
-
for (const item of sourceRows) {
|
|
489
|
-
if (!isRecord(item)) continue;
|
|
490
|
-
const id = firstNonEmptyString(getPath(item, "id"), getPath(item, "modelId"), getPath(item, "name"), getPath(item, "title"));
|
|
491
|
-
if (!id || seen.has(id)) continue;
|
|
492
|
-
const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "title"), getPath(item, "modelId"), getPath(item, "id"));
|
|
493
|
-
rows.push({
|
|
494
|
-
id,
|
|
495
|
-
...(name ? { name } : {}),
|
|
496
|
-
...(typeof getPath(item, "downloads") === "number" ? { downloads: getPath(item, "downloads") } : {}),
|
|
497
|
-
...(typeof getPath(item, "likes") === "number" ? { likes: getPath(item, "likes") } : {}),
|
|
498
|
-
...(firstNonEmptyString(getPath(item, "pipeline_tag"), getPath(item, "task")) ? {
|
|
499
|
-
pipeline_tag: firstNonEmptyString(getPath(item, "pipeline_tag"), getPath(item, "task")),
|
|
500
|
-
} : {}),
|
|
501
|
-
url: firstNonEmptyString(getPath(item, "url"), getPath(item, "link"))
|
|
502
|
-
?? `https://huggingface.co/${encodeURI(id)}`,
|
|
503
|
-
});
|
|
504
|
-
seen.add(id);
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
return rows;
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
function normalizeEmailRows(data: unknown): Record<string, unknown>[] {
|
|
511
|
-
const sourceRows = Array.isArray(data)
|
|
512
|
-
? data
|
|
513
|
-
: isRecord(data) && Array.isArray(data.results)
|
|
514
|
-
? data.results
|
|
515
|
-
: isRecord(data) && Array.isArray(data.emails)
|
|
516
|
-
? data.emails
|
|
517
|
-
: isRecord(data) && Array.isArray(data.messages)
|
|
518
|
-
? data.messages
|
|
519
|
-
: [];
|
|
520
|
-
const rows: Record<string, unknown>[] = [];
|
|
521
|
-
const seen = new Set<string>();
|
|
522
|
-
|
|
523
|
-
for (const item of sourceRows) {
|
|
524
|
-
if (!isRecord(item)) continue;
|
|
525
|
-
const thread = isRecord(getPath(item, "thread")) ? getPath(item, "thread") as Record<string, unknown> : item;
|
|
526
|
-
const message = isRecord(getPath(item, "matchedEmail")) ? getPath(item, "matchedEmail") as Record<string, unknown> : item;
|
|
527
|
-
const id = firstNonEmptyString(
|
|
528
|
-
getPath(message, "id"),
|
|
529
|
-
getPath(thread, "doc_id"),
|
|
530
|
-
getPath(item, "id"),
|
|
531
|
-
getPath(item, "doc_id"),
|
|
532
|
-
);
|
|
533
|
-
if (!id || seen.has(id)) continue;
|
|
534
|
-
const subject = firstNonEmptyString(getPath(thread, "subject"), getPath(item, "subject"), getPath(message, "subject"));
|
|
535
|
-
const from = firstNonEmptyString(
|
|
536
|
-
getPath(message, "sender"),
|
|
537
|
-
getPath(thread, "latest_sender_name"),
|
|
538
|
-
getPath(item, "from"),
|
|
539
|
-
getPath(item, "sender"),
|
|
540
|
-
);
|
|
541
|
-
const date = firstNonEmptyString(getPath(thread, "formatted_date"), getPath(item, "date"), getPath(message, "date"));
|
|
542
|
-
const preview = firstNonEmptyString(
|
|
543
|
-
getPath(thread, "preview"),
|
|
544
|
-
getPath(message, "content_markdown"),
|
|
545
|
-
getPath(item, "preview"),
|
|
546
|
-
getPath(item, "snippet"),
|
|
547
|
-
);
|
|
548
|
-
if (!subject && !from) continue;
|
|
549
|
-
rows.push({
|
|
550
|
-
id,
|
|
551
|
-
...(subject ? { subject } : {}),
|
|
552
|
-
...(from ? { from } : {}),
|
|
553
|
-
...(date ? { date } : {}),
|
|
554
|
-
...(preview ? { preview } : {}),
|
|
555
|
-
});
|
|
556
|
-
seen.add(id);
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
return rows;
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
function normalizeProductRows(data: unknown): Record<string, unknown>[] {
|
|
563
|
-
const rows: Record<string, unknown>[] = [];
|
|
564
|
-
const seen = new Set<string>();
|
|
565
|
-
const blockedTitles = new Set(["results", "more results", "related searches", "need help?"]);
|
|
566
|
-
|
|
567
|
-
collectNestedObjects(data, (obj) => {
|
|
568
|
-
const headingKeys = Object.keys(obj).filter((key) => /^heading_\d+$/i.test(key));
|
|
569
|
-
const title = firstNonEmptyString(
|
|
570
|
-
getPath(obj, "title"),
|
|
571
|
-
getPath(obj, "name"),
|
|
572
|
-
getPath(obj, "productName"),
|
|
573
|
-
getPath(obj, "productTitle"),
|
|
574
|
-
getPath(obj, "title.text"),
|
|
575
|
-
);
|
|
576
|
-
const url = firstNonEmptyString(
|
|
577
|
-
getPath(obj, "url"),
|
|
578
|
-
getPath(obj, "link"),
|
|
579
|
-
getPath(obj, "href"),
|
|
580
|
-
getPath(obj, "canonicalUrl"),
|
|
581
|
-
getPath(obj, "productUrl"),
|
|
582
|
-
getPath(obj, "productPageUrl"),
|
|
583
|
-
);
|
|
584
|
-
const id = firstNonEmptyString(
|
|
585
|
-
getPath(obj, "id"),
|
|
586
|
-
getPath(obj, "usItemId"),
|
|
587
|
-
getPath(obj, "itemId"),
|
|
588
|
-
getPath(obj, "productId"),
|
|
589
|
-
getPath(obj, "sku"),
|
|
590
|
-
);
|
|
591
|
-
const priceString = firstNonEmptyString(
|
|
592
|
-
getPath(obj, "price"),
|
|
593
|
-
getPath(obj, "priceString"),
|
|
594
|
-
getPath(obj, "price.currentPrice.priceString"),
|
|
595
|
-
getPath(obj, "priceInfo.currentPrice.priceString"),
|
|
596
|
-
getPath(obj, "secondaryOfferPrice.currentPrice.priceString"),
|
|
597
|
-
);
|
|
598
|
-
const priceNumber = getPath(obj, "price.currentPrice.price")
|
|
599
|
-
?? getPath(obj, "priceInfo.currentPrice.price")
|
|
600
|
-
?? getPath(obj, "currentPrice.price")
|
|
601
|
-
?? getPath(obj, "salePrice");
|
|
602
|
-
const rating = getPath(obj, "averageRating")
|
|
603
|
-
?? getPath(obj, "rating")
|
|
604
|
-
?? getPath(obj, "rating.average")
|
|
605
|
-
?? getPath(obj, "customerRating")
|
|
606
|
-
?? getPath(obj, "reviews.averageRating");
|
|
607
|
-
const reviewCount = getPath(obj, "numberOfReviews")
|
|
608
|
-
?? getPath(obj, "reviewCount")
|
|
609
|
-
?? getPath(obj, "reviews.count")
|
|
610
|
-
?? getPath(obj, "ratingsTotal");
|
|
611
|
-
const image = firstNonEmptyString(
|
|
612
|
-
getPath(obj, "image"),
|
|
613
|
-
getPath(obj, "imageUrl"),
|
|
614
|
-
getPath(obj, "thumbnail"),
|
|
615
|
-
getPath(obj, "primaryImageUrl"),
|
|
616
|
-
);
|
|
617
|
-
const brand = firstNonEmptyString(getPath(obj, "brand"), getPath(obj, "brandName"), getPath(obj, "seller"));
|
|
618
|
-
|
|
619
|
-
if (!title && !id) return;
|
|
620
|
-
if (title && blockedTitles.has(title.trim().toLowerCase())) return;
|
|
621
|
-
if (headingKeys.length >= 6) return;
|
|
622
|
-
if (!url && !id) return;
|
|
623
|
-
if (
|
|
624
|
-
priceString == null &&
|
|
625
|
-
typeof priceNumber !== "number" &&
|
|
626
|
-
rating == null &&
|
|
627
|
-
reviewCount == null &&
|
|
628
|
-
!image &&
|
|
629
|
-
!brand
|
|
630
|
-
) {
|
|
631
|
-
return;
|
|
632
|
-
}
|
|
633
|
-
if (url && /(amazon-adsystem|doubleclick|googlesyndication|pubmatic|taboola|outbrain|aax-|googleadservices|adservice)\./i.test(url)) {
|
|
634
|
-
return;
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
const stable = String(url ?? id ?? title);
|
|
638
|
-
if (seen.has(stable)) return;
|
|
639
|
-
rows.push({
|
|
640
|
-
...(id ? { id } : {}),
|
|
641
|
-
...(title ? { title } : {}),
|
|
642
|
-
...(url ? { url } : {}),
|
|
643
|
-
...(priceString ? { price: priceString } : typeof priceNumber === "number" ? { price: priceNumber } : {}),
|
|
644
|
-
...(typeof rating === "number" || hasNonEmptyString(rating) ? { rating } : {}),
|
|
645
|
-
...(typeof reviewCount === "number" || hasNonEmptyString(reviewCount) ? { review_count: reviewCount } : {}),
|
|
646
|
-
...(image ? { image } : {}),
|
|
647
|
-
...(brand ? { brand } : {}),
|
|
648
|
-
});
|
|
649
|
-
seen.add(stable);
|
|
650
|
-
});
|
|
651
|
-
|
|
652
|
-
return rows;
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
function normalizeStockQuote(data: unknown): Record<string, unknown> | null {
|
|
656
|
-
let best: Record<string, unknown> | null = null;
|
|
657
|
-
|
|
658
|
-
collectNestedObjects(data, (obj) => {
|
|
659
|
-
const symbol = firstNonEmptyString(
|
|
660
|
-
getPath(obj, "symbol"),
|
|
661
|
-
getPath(obj, "ticker"),
|
|
662
|
-
getPath(obj, "quote.symbol"),
|
|
663
|
-
);
|
|
664
|
-
const price = getPath(obj, "regularMarketPrice.raw")
|
|
665
|
-
?? getPath(obj, "regularMarketPrice")
|
|
666
|
-
?? getPath(obj, "postMarketPrice.raw")
|
|
667
|
-
?? getPath(obj, "price.regularMarketPrice.raw")
|
|
668
|
-
?? getPath(obj, "price.regularMarketPrice")
|
|
669
|
-
?? getPath(obj, "price.currentPrice.raw")
|
|
670
|
-
?? getPath(obj, "price.currentPrice");
|
|
671
|
-
if (!symbol || typeof price !== "number") return;
|
|
672
|
-
const name = firstNonEmptyString(
|
|
673
|
-
getPath(obj, "shortName"),
|
|
674
|
-
getPath(obj, "longName"),
|
|
675
|
-
getPath(obj, "displayName"),
|
|
676
|
-
getPath(obj, "price.shortName"),
|
|
677
|
-
getPath(obj, "price.longName"),
|
|
678
|
-
);
|
|
679
|
-
const currency = firstNonEmptyString(getPath(obj, "currency"), getPath(obj, "financialCurrency"), getPath(obj, "price.currency"));
|
|
680
|
-
const marketState = firstNonEmptyString(getPath(obj, "marketState"), getPath(obj, "price.marketState"));
|
|
681
|
-
const changePercent = getPath(obj, "regularMarketChangePercent.raw")
|
|
682
|
-
?? getPath(obj, "regularMarketChangePercent")
|
|
683
|
-
?? getPath(obj, "price.regularMarketChangePercent.raw")
|
|
684
|
-
?? getPath(obj, "price.regularMarketChangePercent");
|
|
685
|
-
const marketCap = getPath(obj, "marketCap.raw")
|
|
686
|
-
?? getPath(obj, "marketCap")
|
|
687
|
-
?? getPath(obj, "price.marketCap.raw")
|
|
688
|
-
?? getPath(obj, "price.marketCap");
|
|
689
|
-
best = {
|
|
690
|
-
symbol,
|
|
691
|
-
...(name ? { name } : {}),
|
|
692
|
-
price,
|
|
693
|
-
...(currency ? { currency } : {}),
|
|
694
|
-
...(typeof changePercent === "number" ? { change_percent: changePercent } : {}),
|
|
695
|
-
...(typeof marketCap === "number" ? { market_cap: marketCap } : {}),
|
|
696
|
-
...(marketState ? { market_state: marketState } : {}),
|
|
697
|
-
};
|
|
698
|
-
});
|
|
699
|
-
|
|
700
|
-
return best;
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
function normalizeChannelRows(data: unknown): Record<string, unknown>[] {
|
|
704
|
-
const rows: Record<string, unknown>[] = [];
|
|
705
|
-
const seen = new Set<string>();
|
|
706
|
-
|
|
707
|
-
collectNestedObjects(data, (obj) => {
|
|
708
|
-
const name = firstNonEmptyString(
|
|
709
|
-
getPath(obj, "name"),
|
|
710
|
-
getPath(obj, "title"),
|
|
711
|
-
getPath(obj, "channel_name"),
|
|
712
|
-
getPath(obj, "workspace_name"),
|
|
713
|
-
getPath(obj, "team_name"),
|
|
714
|
-
);
|
|
715
|
-
const id = firstNonEmptyString(
|
|
716
|
-
getPath(obj, "id"),
|
|
717
|
-
getPath(obj, "channel_id"),
|
|
718
|
-
getPath(obj, "guild_id"),
|
|
719
|
-
getPath(obj, "server_id"),
|
|
720
|
-
getPath(obj, "workspace_id"),
|
|
721
|
-
getPath(obj, "team_id"),
|
|
722
|
-
);
|
|
723
|
-
const url = firstNonEmptyString(
|
|
724
|
-
getPath(obj, "url"),
|
|
725
|
-
getPath(obj, "channel_url"),
|
|
726
|
-
getPath(obj, "server_url"),
|
|
727
|
-
getPath(obj, "workspace_url"),
|
|
728
|
-
);
|
|
729
|
-
if (!name || (!id && !url)) return;
|
|
730
|
-
const stable = String(id ?? url ?? name);
|
|
731
|
-
if (seen.has(stable)) return;
|
|
732
|
-
rows.push({
|
|
733
|
-
...(id ? { id } : {}),
|
|
734
|
-
name,
|
|
735
|
-
...(url ? { url } : {}),
|
|
736
|
-
...(firstNonEmptyString(getPath(obj, "topic"), getPath(obj, "purpose"), getPath(obj, "description")) ? {
|
|
737
|
-
description: firstNonEmptyString(getPath(obj, "topic"), getPath(obj, "purpose"), getPath(obj, "description")),
|
|
738
|
-
} : {}),
|
|
739
|
-
...(typeof getPath(obj, "member_count") === "number" ? { member_count: getPath(obj, "member_count") } : {}),
|
|
740
|
-
...(firstNonEmptyString(getPath(obj, "type"), getPath(obj, "channel_type"), getPath(obj, "kind")) ? {
|
|
741
|
-
type: firstNonEmptyString(getPath(obj, "type"), getPath(obj, "channel_type"), getPath(obj, "kind")),
|
|
742
|
-
} : {}),
|
|
743
|
-
});
|
|
744
|
-
seen.add(stable);
|
|
745
|
-
});
|
|
746
|
-
|
|
747
|
-
return rows;
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
function collectNestedObjects(value: unknown, visit: (obj: Record<string, unknown>) => void): void {
|
|
751
|
-
if (Array.isArray(value)) {
|
|
752
|
-
for (const item of value) collectNestedObjects(item, visit);
|
|
753
|
-
return;
|
|
754
|
-
}
|
|
755
|
-
if (!isRecord(value)) return;
|
|
756
|
-
visit(value);
|
|
757
|
-
for (const child of Object.values(value)) collectNestedObjects(child, visit);
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
function normalizeXTweets(data: unknown): Record<string, unknown>[] {
|
|
761
|
-
const rows: Record<string, unknown>[] = [];
|
|
762
|
-
const seen = new Set<string>();
|
|
763
|
-
collectNestedObjects(data, (obj) => {
|
|
764
|
-
const result = getPath(obj, "tweet_results.result");
|
|
765
|
-
if (!isRecord(result)) return;
|
|
766
|
-
const restId = getPath(result, "rest_id");
|
|
767
|
-
const text = getPath(result, "legacy.full_text");
|
|
768
|
-
const screenName = getPath(result, "core.user_results.result.core.screen_name")
|
|
769
|
-
?? getPath(result, "core.user_results.result.legacy.screen_name");
|
|
770
|
-
if (!hasNonEmptyString(restId) || !hasNonEmptyString(text) || !hasNonEmptyString(screenName)) return;
|
|
771
|
-
if (seen.has(String(restId))) return;
|
|
772
|
-
rows.push({
|
|
773
|
-
id: String(restId),
|
|
774
|
-
username: String(screenName),
|
|
775
|
-
text: String(text),
|
|
776
|
-
url: `https://x.com/${encodeURIComponent(String(screenName))}/status/${encodeURIComponent(String(restId))}`,
|
|
777
|
-
});
|
|
778
|
-
seen.add(String(restId));
|
|
779
|
-
});
|
|
780
|
-
return rows;
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
function normalizeXUsers(data: unknown): Record<string, unknown>[] {
|
|
784
|
-
const rows: Record<string, unknown>[] = [];
|
|
785
|
-
const seen = new Set<string>();
|
|
786
|
-
collectNestedObjects(data, (obj) => {
|
|
787
|
-
const result = getPath(obj, "user_results.result");
|
|
788
|
-
const target = isRecord(result) ? result : obj;
|
|
789
|
-
const screenName = getPath(target, "core.screen_name") ?? getPath(target, "legacy.screen_name");
|
|
790
|
-
const name = getPath(target, "core.name") ?? getPath(target, "legacy.name");
|
|
791
|
-
const description = getPath(target, "legacy.description");
|
|
792
|
-
const followers = getPath(target, "legacy.followers_count");
|
|
793
|
-
if (!hasNonEmptyString(screenName) || !hasNonEmptyString(name)) return;
|
|
794
|
-
if (seen.has(String(screenName))) return;
|
|
795
|
-
rows.push({
|
|
796
|
-
username: String(screenName),
|
|
797
|
-
public_identifier: String(screenName),
|
|
798
|
-
name: String(name),
|
|
799
|
-
...(hasNonEmptyString(description) ? { description: String(description) } : {}),
|
|
800
|
-
...(typeof followers === "number" ? { followers_count: followers } : {}),
|
|
801
|
-
url: `https://x.com/${encodeURIComponent(String(screenName))}`,
|
|
802
|
-
});
|
|
803
|
-
seen.add(String(screenName));
|
|
804
|
-
});
|
|
805
|
-
return rows;
|
|
806
|
-
}
|
|
807
|
-
|
|
808
|
-
function joinNameParts(...values: unknown[]): string | undefined {
|
|
809
|
-
const parts = values.filter((value) => hasNonEmptyString(value)).map((value) => String(value).trim());
|
|
810
|
-
return parts.length > 0 ? parts.join(" ") : undefined;
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
function absoluteLinkedInUrl(value: string | undefined): string | undefined {
|
|
814
|
-
if (!value) return undefined;
|
|
815
|
-
return value.startsWith("http") ? value : `https://www.linkedin.com${value.startsWith("/") ? "" : "/"}${value}`;
|
|
816
|
-
}
|
|
817
|
-
|
|
818
|
-
function normalizeLinkedInFeedPosts(data: unknown): Record<string, unknown>[] {
|
|
819
|
-
if (!isRecord(data)) return [];
|
|
820
|
-
|
|
821
|
-
const included = Array.isArray(getPath(data, "included"))
|
|
822
|
-
? getPath(data, "included") as unknown[]
|
|
823
|
-
: Array.isArray(getPath(data, "data.included"))
|
|
824
|
-
? getPath(data, "data.included") as unknown[]
|
|
825
|
-
: [];
|
|
826
|
-
if (included.length === 0) return [];
|
|
827
|
-
|
|
828
|
-
const elementRefs = Array.isArray(getPath(data, "data.data.feedDashMainFeedByMainFeed.*elements"))
|
|
829
|
-
? getPath(data, "data.data.feedDashMainFeedByMainFeed.*elements") as unknown[]
|
|
830
|
-
: Array.isArray(getPath(data, "data.feedDashMainFeedByMainFeed.*elements"))
|
|
831
|
-
? getPath(data, "data.feedDashMainFeedByMainFeed.*elements") as unknown[]
|
|
832
|
-
: Array.isArray(getPath(data, "feedDashMainFeedByMainFeed.*elements"))
|
|
833
|
-
? getPath(data, "feedDashMainFeedByMainFeed.*elements") as unknown[]
|
|
834
|
-
: [];
|
|
835
|
-
if (elementRefs.length === 0) return [];
|
|
836
|
-
|
|
837
|
-
const entityIndex = new Map<string, Record<string, unknown>>();
|
|
838
|
-
for (const item of included) {
|
|
839
|
-
if (!isRecord(item)) continue;
|
|
840
|
-
const urn = getPath(item, "entityUrn");
|
|
841
|
-
if (!hasNonEmptyString(urn)) continue;
|
|
842
|
-
entityIndex.set(String(urn), item);
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
const rows: Record<string, unknown>[] = [];
|
|
846
|
-
const seen = new Set<string>();
|
|
847
|
-
|
|
848
|
-
for (const ref of elementRefs) {
|
|
849
|
-
if (!hasNonEmptyString(ref) || seen.has(String(ref))) continue;
|
|
850
|
-
const update = entityIndex.get(String(ref));
|
|
851
|
-
if (!update) continue;
|
|
852
|
-
|
|
853
|
-
const actorProfileUrn = firstNonEmptyString(getPath(update, "actor.*profileUrn"), getPath(update, "actor.entityUrn"));
|
|
854
|
-
const actorProfile = actorProfileUrn ? entityIndex.get(actorProfileUrn) : undefined;
|
|
855
|
-
const author = firstNonEmptyString(
|
|
856
|
-
getPath(update, "actor.name.text"),
|
|
857
|
-
getPath(actorProfile, "name"),
|
|
858
|
-
joinNameParts(getPath(actorProfile, "firstName"), getPath(actorProfile, "lastName")),
|
|
859
|
-
);
|
|
860
|
-
const username = firstNonEmptyString(getPath(actorProfile, "publicIdentifier"));
|
|
861
|
-
const content = firstNonEmptyString(
|
|
862
|
-
getPath(update, "commentary.text.text"),
|
|
863
|
-
getPath(update, "commentary.accessibilityText"),
|
|
864
|
-
getPath(update, "content.text"),
|
|
865
|
-
getPath(update, "content.title"),
|
|
866
|
-
getPath(update, "header.text"),
|
|
867
|
-
getPath(update, "header.title"),
|
|
868
|
-
);
|
|
869
|
-
const permalink = firstNonEmptyString(
|
|
870
|
-
getPath(update, "permalink"),
|
|
871
|
-
getPath(update, "socialContent.shareUrl"),
|
|
872
|
-
getPath(update, "url"),
|
|
873
|
-
);
|
|
874
|
-
const url = absoluteLinkedInUrl(permalink) ?? `https://www.linkedin.com/feed/update/${encodeURIComponent(String(ref))}/`;
|
|
875
|
-
|
|
876
|
-
if (!content && !username && !author) continue;
|
|
877
|
-
|
|
878
|
-
rows.push({
|
|
879
|
-
id: String(ref),
|
|
880
|
-
url,
|
|
881
|
-
...(content ? { content } : {}),
|
|
882
|
-
...(author ? { author } : {}),
|
|
883
|
-
...(username ? { username, public_identifier: username } : {}),
|
|
884
|
-
...(typeof getPath(update, "createdAt") === "number" ? { created_at: getPath(update, "createdAt") } : {}),
|
|
885
|
-
});
|
|
886
|
-
seen.add(String(ref));
|
|
887
|
-
}
|
|
888
|
-
|
|
889
|
-
return rows;
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
function normalizeGenericPeopleRows(data: unknown): Record<string, unknown>[] {
|
|
893
|
-
const sourceRows = Array.isArray(data)
|
|
894
|
-
? data
|
|
895
|
-
: isRecord(data) && Array.isArray(data.results)
|
|
896
|
-
? data.results
|
|
897
|
-
: [];
|
|
898
|
-
const rows: Record<string, unknown>[] = [];
|
|
899
|
-
const seen = new Set<string>();
|
|
900
|
-
|
|
901
|
-
for (const item of sourceRows) {
|
|
902
|
-
if (!isRecord(item)) continue;
|
|
903
|
-
if (
|
|
904
|
-
hasAnyPath(item, ["full_name", "repository.name", "path_with_namespace", "stargazers_count", "stars", "star_count", "forks_count", "version", "price"])
|
|
905
|
-
) {
|
|
906
|
-
continue;
|
|
907
|
-
}
|
|
908
|
-
const name = firstNonEmptyString(getPath(item, "name"), getPath(item, "title"));
|
|
909
|
-
const headline = firstNonEmptyString(
|
|
910
|
-
getPath(item, "headline"),
|
|
911
|
-
getPath(item, "description"),
|
|
912
|
-
getPath(item, "subtitle"),
|
|
913
|
-
getPath(item, "bio"),
|
|
914
|
-
);
|
|
915
|
-
const url = firstNonEmptyString(getPath(item, "url"), getPath(item, "link"));
|
|
916
|
-
const publicIdentifier = firstNonEmptyString(getPath(item, "public_identifier"), getPath(item, "username"));
|
|
917
|
-
if (!name || name.includes("/") || name.includes("://")) continue;
|
|
918
|
-
if (!headline && !url && !publicIdentifier) continue;
|
|
919
|
-
const id = publicIdentifier ?? url ?? name;
|
|
920
|
-
if (seen.has(id)) continue;
|
|
921
|
-
rows.push({
|
|
922
|
-
name,
|
|
923
|
-
...(headline ? { headline } : {}),
|
|
924
|
-
...(url ? { url } : {}),
|
|
925
|
-
...(publicIdentifier ? { public_identifier: publicIdentifier } : {}),
|
|
926
|
-
});
|
|
927
|
-
seen.add(id);
|
|
928
|
-
}
|
|
929
|
-
|
|
930
|
-
return rows;
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
function normalizeRedditPosts(data: unknown): Record<string, unknown>[] {
|
|
934
|
-
const rows: Record<string, unknown>[] = [];
|
|
935
|
-
const seen = new Set<string>();
|
|
936
|
-
collectNestedObjects(data, (obj) => {
|
|
937
|
-
const kind = getPath(obj, "kind");
|
|
938
|
-
const payload = isRecord(getPath(obj, "data")) ? getPath(obj, "data") : obj;
|
|
939
|
-
if (!isRecord(payload)) return;
|
|
940
|
-
const title = getPath(payload, "title");
|
|
941
|
-
const author = getPath(payload, "author");
|
|
942
|
-
const permalink = getPath(payload, "permalink");
|
|
943
|
-
if (!hasNonEmptyString(title) || !hasNonEmptyString(author) || !hasNonEmptyString(permalink)) return;
|
|
944
|
-
if (hasNonEmptyString(kind) && kind !== "t3") return;
|
|
945
|
-
const id = getPath(payload, "id") ?? getPath(payload, "name") ?? permalink;
|
|
946
|
-
if (!hasNonEmptyString(id) || seen.has(String(id))) return;
|
|
947
|
-
rows.push({
|
|
948
|
-
id: String(id),
|
|
949
|
-
title: String(title),
|
|
950
|
-
author: String(author),
|
|
951
|
-
...(typeof getPath(payload, "score") === "number" ? { score: getPath(payload, "score") } : {}),
|
|
952
|
-
...(typeof getPath(payload, "num_comments") === "number" ? { num_comments: getPath(payload, "num_comments") } : {}),
|
|
953
|
-
...(hasNonEmptyString(getPath(payload, "subreddit")) ? { subreddit: String(getPath(payload, "subreddit")) } : {}),
|
|
954
|
-
permalink: String(permalink),
|
|
955
|
-
url: `https://www.reddit.com${String(permalink)}`,
|
|
956
|
-
});
|
|
957
|
-
seen.add(String(id));
|
|
958
|
-
});
|
|
959
|
-
return rows;
|
|
960
|
-
}
|
|
961
|
-
|
|
962
|
-
function normalizeRedditComments(data: unknown): Record<string, unknown>[] {
|
|
963
|
-
const rows: Record<string, unknown>[] = [];
|
|
964
|
-
const seen = new Set<string>();
|
|
965
|
-
collectNestedObjects(data, (obj) => {
|
|
966
|
-
const kind = getPath(obj, "kind");
|
|
967
|
-
const payload = isRecord(getPath(obj, "data")) ? getPath(obj, "data") : obj;
|
|
968
|
-
if (!isRecord(payload)) return;
|
|
969
|
-
const body = getPath(payload, "body");
|
|
970
|
-
const author = getPath(payload, "author");
|
|
971
|
-
const permalink = getPath(payload, "permalink");
|
|
972
|
-
if (!hasNonEmptyString(body) || !hasNonEmptyString(author) || !hasNonEmptyString(permalink)) return;
|
|
973
|
-
if (hasNonEmptyString(kind) && kind !== "t1") return;
|
|
974
|
-
const id = getPath(payload, "id") ?? getPath(payload, "name") ?? permalink;
|
|
975
|
-
if (!hasNonEmptyString(id) || seen.has(String(id))) return;
|
|
976
|
-
rows.push({
|
|
977
|
-
id: String(id),
|
|
978
|
-
author: String(author),
|
|
979
|
-
body: String(body),
|
|
980
|
-
permalink: String(permalink),
|
|
981
|
-
url: `https://www.reddit.com${String(permalink)}`,
|
|
982
|
-
});
|
|
983
|
-
seen.add(String(id));
|
|
984
|
-
});
|
|
985
|
-
return rows;
|
|
986
|
-
}
|
|
987
|
-
|
|
988
|
-
function normalizeCompanies(data: unknown): Record<string, unknown>[] {
|
|
989
|
-
const rows: Record<string, unknown>[] = [];
|
|
990
|
-
const seen = new Set<string>();
|
|
991
|
-
collectNestedObjects(data, (obj) => {
|
|
992
|
-
const name = getPath(obj, "name") ?? getPath(obj, "title");
|
|
993
|
-
const publicIdentifier =
|
|
994
|
-
getPath(obj, "public_identifier") ??
|
|
995
|
-
getPath(obj, "vanityName") ??
|
|
996
|
-
getPath(obj, "universalName");
|
|
997
|
-
const description =
|
|
998
|
-
getPath(obj, "description") ??
|
|
999
|
-
getPath(obj, "tagline") ??
|
|
1000
|
-
getPath(obj, "headline");
|
|
1001
|
-
const industry = getPath(obj, "industry") ?? getPath(obj, "primaryIndustry");
|
|
1002
|
-
const employeeCount =
|
|
1003
|
-
getPath(obj, "employee_count") ??
|
|
1004
|
-
getPath(obj, "staffCount") ??
|
|
1005
|
-
getPath(obj, "employeeCount");
|
|
1006
|
-
const followerCount =
|
|
1007
|
-
getPath(obj, "follower_count") ??
|
|
1008
|
-
getPath(obj, "followerCount");
|
|
1009
|
-
const website = getPath(obj, "website") ?? getPath(obj, "websiteUrl");
|
|
1010
|
-
const url =
|
|
1011
|
-
getPath(obj, "url") ??
|
|
1012
|
-
(hasNonEmptyString(publicIdentifier)
|
|
1013
|
-
? `https://www.linkedin.com/company/${encodeURIComponent(String(publicIdentifier))}/`
|
|
1014
|
-
: undefined);
|
|
1015
|
-
if (!hasNonEmptyString(name)) return;
|
|
1016
|
-
if (
|
|
1017
|
-
!hasNonEmptyString(description) &&
|
|
1018
|
-
!hasNonEmptyString(industry) &&
|
|
1019
|
-
typeof employeeCount !== "number" &&
|
|
1020
|
-
typeof followerCount !== "number" &&
|
|
1021
|
-
!hasNonEmptyString(website) &&
|
|
1022
|
-
!hasNonEmptyString(url) &&
|
|
1023
|
-
!hasNonEmptyString(publicIdentifier)
|
|
1024
|
-
) {
|
|
1025
|
-
return;
|
|
1026
|
-
}
|
|
1027
|
-
const id = String(publicIdentifier ?? url ?? name);
|
|
1028
|
-
if (seen.has(id)) return;
|
|
1029
|
-
rows.push({
|
|
1030
|
-
name: String(name),
|
|
1031
|
-
...(hasNonEmptyString(publicIdentifier) ? { public_identifier: String(publicIdentifier) } : {}),
|
|
1032
|
-
...(hasNonEmptyString(description) ? { description: String(description) } : {}),
|
|
1033
|
-
...(hasNonEmptyString(industry) ? { industry: String(industry) } : {}),
|
|
1034
|
-
...(typeof employeeCount === "number" ? { employee_count: employeeCount } : {}),
|
|
1035
|
-
...(typeof followerCount === "number" ? { follower_count: followerCount } : {}),
|
|
1036
|
-
...(hasNonEmptyString(website) ? { website: String(website) } : {}),
|
|
1037
|
-
...(hasNonEmptyString(url) ? { url: String(url) } : {}),
|
|
1038
|
-
});
|
|
1039
|
-
seen.add(id);
|
|
1040
|
-
});
|
|
1041
|
-
return rows;
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
function unwrapCarrier(data: unknown): unknown {
|
|
1045
|
-
if (!isRecord(data)) return data;
|
|
1046
|
-
const keys = Object.keys(data);
|
|
1047
|
-
const isCarrierOnly = keys.every((key) => key === "data" || key === "_extraction");
|
|
1048
|
-
if (isCarrierOnly && "data" in data && ("_extraction" in data || Array.isArray(data.data) || isRecord(data.data))) {
|
|
1049
|
-
return unwrapCarrier(data.data);
|
|
1050
|
-
}
|
|
1051
|
-
return data;
|
|
1052
|
-
}
|
|
1053
|
-
|
|
1054
|
-
export function projectIntentData(data: unknown, intent?: string): unknown {
|
|
1055
|
-
const unwrapped = unwrapCarrier(data);
|
|
1056
|
-
if (!intent) return unwrapped;
|
|
1057
|
-
const lower = intent.toLowerCase();
|
|
1058
|
-
|
|
1059
|
-
if (/\b(stock|stocks|ticker|tickers|quote|quotes)\b/.test(lower)) {
|
|
1060
|
-
const normalizedQuote = normalizeStockQuote(unwrapped);
|
|
1061
|
-
if (normalizedQuote) return normalizedQuote;
|
|
1062
|
-
}
|
|
1063
|
-
|
|
1064
|
-
if (/\b(product|products|item|items)\b/.test(lower)) {
|
|
1065
|
-
const normalizedProducts = normalizeProductRows(unwrapped);
|
|
1066
|
-
if (normalizedProducts.length > 0) return normalizedProducts;
|
|
1067
|
-
if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).products)) {
|
|
1068
|
-
return (unwrapped as Record<string, unknown>).products;
|
|
1069
|
-
}
|
|
1070
|
-
if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).items)) {
|
|
1071
|
-
return (unwrapped as Record<string, unknown>).items;
|
|
1072
|
-
}
|
|
1073
|
-
}
|
|
1074
|
-
|
|
1075
|
-
if (/\b(channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(lower)) {
|
|
1076
|
-
const normalizedChannels = normalizeChannelRows(unwrapped);
|
|
1077
|
-
if (normalizedChannels.length > 0) return normalizedChannels;
|
|
1078
|
-
if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).channels)) {
|
|
1079
|
-
return (unwrapped as Record<string, unknown>).channels;
|
|
1080
|
-
}
|
|
1081
|
-
if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).guilds)) {
|
|
1082
|
-
return (unwrapped as Record<string, unknown>).guilds;
|
|
1083
|
-
}
|
|
1084
|
-
if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).workspaces)) {
|
|
1085
|
-
return (unwrapped as Record<string, unknown>).workspaces;
|
|
1086
|
-
}
|
|
1087
|
-
}
|
|
1088
|
-
|
|
1089
|
-
if (/\b(package|packages)\b/.test(lower)) {
|
|
1090
|
-
if (/\bsearch\b/.test(lower)) {
|
|
1091
|
-
const normalizedCratesSearch = normalizeCratesPackageSearchResults(unwrapped);
|
|
1092
|
-
if (normalizedCratesSearch.length > 0) return normalizedCratesSearch;
|
|
1093
|
-
const normalizedPackageSearch = normalizePackageSearchResults(unwrapped);
|
|
1094
|
-
if (normalizedPackageSearch.length > 0) return normalizedPackageSearch;
|
|
1095
|
-
if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).packages)) {
|
|
1096
|
-
return (unwrapped as Record<string, unknown>).packages;
|
|
1097
|
-
}
|
|
1098
|
-
} else {
|
|
1099
|
-
const normalizedNpmPackage = normalizeNpmPackageInfo(unwrapped);
|
|
1100
|
-
if (normalizedNpmPackage) return normalizedNpmPackage;
|
|
1101
|
-
const normalizedPyPIPackage = normalizePyPIPackageInfo(unwrapped);
|
|
1102
|
-
if (normalizedPyPIPackage) return normalizedPyPIPackage;
|
|
1103
|
-
const normalizedPubDevPackage = normalizePubDevPackageInfo(unwrapped);
|
|
1104
|
-
if (normalizedPubDevPackage) return normalizedPubDevPackage;
|
|
1105
|
-
const normalizedRubyGemsPackage = normalizeRubyGemsPackageInfo(unwrapped);
|
|
1106
|
-
if (normalizedRubyGemsPackage) return normalizedRubyGemsPackage;
|
|
1107
|
-
if (isRecord(unwrapped) && isRecord((unwrapped as Record<string, unknown>).package)) {
|
|
1108
|
-
return (unwrapped as Record<string, unknown>).package;
|
|
1109
|
-
}
|
|
1110
|
-
}
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
if (/\b(doc|docs|documentation)\b/.test(lower)) {
|
|
1114
|
-
const normalizedDocuments = normalizeDocumentRows(unwrapped);
|
|
1115
|
-
if (normalizedDocuments.length > 0) return normalizedDocuments;
|
|
1116
|
-
if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).documents)) {
|
|
1117
|
-
return (unwrapped as Record<string, unknown>).documents;
|
|
1118
|
-
}
|
|
1119
|
-
}
|
|
1120
|
-
|
|
1121
|
-
if (/\bsearch images\b/.test(lower)) {
|
|
1122
|
-
const normalizedImageSearch = normalizeDockerImageSearchResults(unwrapped);
|
|
1123
|
-
if (normalizedImageSearch.length > 0) return normalizedImageSearch;
|
|
1124
|
-
if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).images)) {
|
|
1125
|
-
return (unwrapped as Record<string, unknown>).images;
|
|
1126
|
-
}
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
|
-
if (/\b(get )?image tags\b/.test(lower)) {
|
|
1130
|
-
const normalizedImageTags = normalizeDockerTagResults(unwrapped);
|
|
1131
|
-
if (normalizedImageTags.length > 0) return normalizedImageTags;
|
|
1132
|
-
if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).tags)) {
|
|
1133
|
-
return (unwrapped as Record<string, unknown>).tags;
|
|
1134
|
-
}
|
|
1135
|
-
}
|
|
1136
|
-
|
|
1137
|
-
if (/\b(comment|comments)\b/.test(lower)) {
|
|
1138
|
-
const normalizedRedditComments = normalizeRedditComments(unwrapped);
|
|
1139
|
-
if (normalizedRedditComments.length > 0) return normalizedRedditComments;
|
|
1140
|
-
if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).comments)) {
|
|
1141
|
-
return (unwrapped as Record<string, unknown>).comments;
|
|
1142
|
-
}
|
|
1143
|
-
}
|
|
1144
|
-
|
|
1145
|
-
if (/\b(reddit|subreddit)\b/.test(lower)) {
|
|
1146
|
-
const normalizedRedditPosts = normalizeRedditPosts(unwrapped);
|
|
1147
|
-
if (normalizedRedditPosts.length > 0) return normalizedRedditPosts;
|
|
1148
|
-
}
|
|
1149
|
-
|
|
1150
|
-
if (/\b(company|companies|organization|organisations|business)\b/.test(lower)) {
|
|
1151
|
-
const normalizedCompanies = normalizeCompanies(unwrapped);
|
|
1152
|
-
if (normalizedCompanies.length === 1) return normalizedCompanies[0];
|
|
1153
|
-
if (normalizedCompanies.length > 1) return normalizedCompanies;
|
|
1154
|
-
if (isRecord(unwrapped) && Array.isArray((unwrapped as Record<string, unknown>).companies)) {
|
|
1155
|
-
return (unwrapped as Record<string, unknown>).companies;
|
|
1156
|
-
}
|
|
1157
|
-
}
|
|
1158
|
-
|
|
1159
|
-
if (!isRecord(unwrapped) && !Array.isArray(unwrapped)) return unwrapped;
|
|
1160
|
-
|
|
1161
|
-
if (/\b(post|posts|tweet|tweets|status|statuses)\b/.test(lower)) {
|
|
1162
|
-
const normalizedDevPosts = normalizeDevToPosts(unwrapped);
|
|
1163
|
-
if (normalizedDevPosts.length > 0) return normalizedDevPosts;
|
|
1164
|
-
const normalizedLobsters = normalizeLobstersPosts(unwrapped);
|
|
1165
|
-
if (normalizedLobsters.length > 0) return normalizedLobsters;
|
|
1166
|
-
const normalizedLinkedInFeed = normalizeLinkedInFeedPosts(unwrapped);
|
|
1167
|
-
if (normalizedLinkedInFeed.length > 0) return normalizedLinkedInFeed;
|
|
1168
|
-
const normalizedRedditPosts = normalizeRedditPosts(unwrapped);
|
|
1169
|
-
if (normalizedRedditPosts.length > 0) return normalizedRedditPosts;
|
|
1170
|
-
const normalizedXTweets = normalizeXTweets(unwrapped);
|
|
1171
|
-
if (normalizedXTweets.length > 0) return normalizedXTweets;
|
|
1172
|
-
if (Array.isArray(unwrapped)) return unwrapped;
|
|
1173
|
-
if (Array.isArray(unwrapped.statuses)) return unwrapped.statuses;
|
|
1174
|
-
if (Array.isArray(unwrapped.posts)) return unwrapped.posts;
|
|
1175
|
-
if (Array.isArray(unwrapped.tweets)) return unwrapped.tweets;
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
|
-
if (/\b(person|people|user|users|profile|profiles|member|members)\b/.test(lower)) {
|
|
1179
|
-
if (Array.isArray(unwrapped.people)) return unwrapped.people;
|
|
1180
|
-
if (Array.isArray(unwrapped.users)) return unwrapped.users;
|
|
1181
|
-
if (Array.isArray(unwrapped.accounts)) return unwrapped.accounts;
|
|
1182
|
-
if (Array.isArray(unwrapped.elements)) return unwrapped.elements;
|
|
1183
|
-
if (Array.isArray(unwrapped.included)) return unwrapped.included;
|
|
1184
|
-
const normalizedXUsers = normalizeXUsers(unwrapped);
|
|
1185
|
-
if (normalizedXUsers.length > 0) return normalizedXUsers;
|
|
1186
|
-
const normalizedGenericPeople = normalizeGenericPeopleRows(unwrapped);
|
|
1187
|
-
if (normalizedGenericPeople.length > 0) return normalizedGenericPeople;
|
|
1188
|
-
}
|
|
1189
|
-
|
|
1190
|
-
if (/\b(topic|topics|trend|trending|hashtag|hashtags)\b/.test(lower)) {
|
|
1191
|
-
const storyItems = getPath(unwrapped, "story_topic.stories.items");
|
|
1192
|
-
if (Array.isArray(storyItems)) {
|
|
1193
|
-
const normalized = storyItems
|
|
1194
|
-
.map((item) => {
|
|
1195
|
-
const core = getPath(item, "trend_results.result.core");
|
|
1196
|
-
const name = getPath(core, "name");
|
|
1197
|
-
const restId = getPath(item, "trend_results.result.rest_id");
|
|
1198
|
-
if (!hasNonEmptyString(name)) return null;
|
|
1199
|
-
return {
|
|
1200
|
-
name,
|
|
1201
|
-
...(hasNonEmptyString(getPath(core, "category")) ? { category: getPath(core, "category") } : {}),
|
|
1202
|
-
...(hasNonEmptyString(getPath(item, "trend_results.result.post_count")) ? { post_count: getPath(item, "trend_results.result.post_count") } : {}),
|
|
1203
|
-
...(hasNonEmptyString(restId)
|
|
1204
|
-
? { url: `https://x.com/search?q=${encodeURIComponent(String(name))}&src=trend_click&trend_id=${encodeURIComponent(String(restId))}` }
|
|
1205
|
-
: { url: `https://x.com/search?q=${encodeURIComponent(String(name))}` }),
|
|
1206
|
-
};
|
|
1207
|
-
})
|
|
1208
|
-
.filter((item): item is Record<string, unknown> => !!item);
|
|
1209
|
-
if (normalized.length > 0) return normalized;
|
|
1210
|
-
}
|
|
1211
|
-
if (Array.isArray(unwrapped.topics)) return unwrapped.topics;
|
|
1212
|
-
if (Array.isArray(unwrapped.trends)) return unwrapped.trends;
|
|
1213
|
-
if (Array.isArray(unwrapped.hashtags)) return unwrapped.hashtags;
|
|
1214
|
-
}
|
|
1215
|
-
|
|
1216
|
-
if (/\b(model|models)\b/.test(lower)) {
|
|
1217
|
-
const normalizedModels = normalizeHuggingFaceModels(unwrapped);
|
|
1218
|
-
if (normalizedModels.length > 0) return normalizedModels;
|
|
1219
|
-
if (Array.isArray(unwrapped.models)) return unwrapped.models;
|
|
1220
|
-
}
|
|
1221
|
-
|
|
1222
|
-
if (/\b(news|story|stories|article|articles|hacker news)\b/.test(lower)) {
|
|
1223
|
-
if (Array.isArray(unwrapped)) return unwrapped;
|
|
1224
|
-
const normalizedStories = normalizeHackerNewsStories(unwrapped);
|
|
1225
|
-
if (normalizedStories.length > 0) return normalizedStories;
|
|
1226
|
-
if (Array.isArray(unwrapped.hits)) return unwrapped.hits;
|
|
1227
|
-
if (Array.isArray(unwrapped.stories)) return unwrapped.stories;
|
|
1228
|
-
if (Array.isArray(unwrapped.articles)) return unwrapped.articles;
|
|
1229
|
-
}
|
|
1230
|
-
|
|
1231
|
-
if (/\b(question|questions)\b/.test(lower)) {
|
|
1232
|
-
if (Array.isArray(unwrapped)) return unwrapped;
|
|
1233
|
-
const normalizedQuestions = normalizeStackExchangeQuestions(unwrapped);
|
|
1234
|
-
if (normalizedQuestions.length > 0) return normalizedQuestions;
|
|
1235
|
-
if (Array.isArray(unwrapped.items)) return unwrapped.items;
|
|
1236
|
-
}
|
|
1237
|
-
|
|
1238
|
-
if (/\b(email|emails|mail|inbox)\b/.test(lower)) {
|
|
1239
|
-
const normalizedEmails = normalizeEmailRows(unwrapped);
|
|
1240
|
-
if (normalizedEmails.length > 0) return normalizedEmails;
|
|
1241
|
-
if (Array.isArray(unwrapped.emails)) return unwrapped.emails;
|
|
1242
|
-
if (Array.isArray(unwrapped.messages)) return unwrapped.messages;
|
|
1243
|
-
if (Array.isArray(unwrapped.results)) return unwrapped.results;
|
|
1244
|
-
}
|
|
1245
|
-
|
|
1246
|
-
if (/\b(repo|repository|repositories|project|projects)\b/.test(lower)) {
|
|
1247
|
-
if (Array.isArray(unwrapped.repositories)) return unwrapped.repositories;
|
|
1248
|
-
if (Array.isArray(unwrapped.items)) return unwrapped.items;
|
|
1249
|
-
if (Array.isArray(unwrapped.projects)) return unwrapped.projects;
|
|
1250
|
-
}
|
|
1251
|
-
|
|
1252
|
-
return unwrapped;
|
|
1253
|
-
}
|
|
1254
|
-
|
|
1255
|
-
function classifyRows(rows: unknown[], intent: string): { verdict: "pass" | "fail" | "skip"; reason: string } {
|
|
1256
|
-
if (rows.length === 0) return { verdict: "fail", reason: "empty_array" };
|
|
1257
|
-
if (rows.every((row) => !isRecord(row))) return { verdict: "fail", reason: "primitive_rows" };
|
|
1258
|
-
|
|
1259
|
-
const objects = rows.filter(isRecord);
|
|
1260
|
-
if (objects.length === 0) return { verdict: "fail", reason: "primitive_rows" };
|
|
1261
|
-
|
|
1262
|
-
const lower = intent.toLowerCase();
|
|
1263
|
-
|
|
1264
|
-
if (/\b(repo|repository|repositories|project|projects)\b/.test(lower)) {
|
|
1265
|
-
const matching = objects.filter((row) =>
|
|
1266
|
-
hasAnyPath(row, ["full_name", "name", "repository.name", "path_with_namespace"]) &&
|
|
1267
|
-
hasAnyPath(row, ["url", "web_url", "description", "stargazers_count", "stars", "star_count", "forks_count", "owner.login", "owner"])
|
|
1268
|
-
);
|
|
1269
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "repository_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1270
|
-
}
|
|
1271
|
-
|
|
1272
|
-
if (/\b(package|packages)\b/.test(lower)) {
|
|
1273
|
-
const matching = objects.filter((row) =>
|
|
1274
|
-
hasAnyPath(row, ["name", "package.name"]) &&
|
|
1275
|
-
hasAnyPath(row, ["version", "description", "summary", "url", "keywords", "requires_dist", "dependencies"])
|
|
1276
|
-
);
|
|
1277
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "package_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1278
|
-
}
|
|
1279
|
-
|
|
1280
|
-
if (/\b(model|models)\b/.test(lower)) {
|
|
1281
|
-
const matching = objects.filter((row) =>
|
|
1282
|
-
hasAnyPath(row, ["id", "modelId", "name"]) &&
|
|
1283
|
-
hasAnyPath(row, ["downloads", "likes", "pipeline_tag", "url"])
|
|
1284
|
-
);
|
|
1285
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "model_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1286
|
-
}
|
|
1287
|
-
|
|
1288
|
-
if (/\b(news|story|stories|article|articles|hacker news)\b/.test(lower)) {
|
|
1289
|
-
const matching = objects.filter((row) =>
|
|
1290
|
-
hasAnyPath(row, ["objectID", "id", "url", "link"]) &&
|
|
1291
|
-
hasAnyPath(row, ["title", "story_title", "author", "points", "num_comments", "meta"])
|
|
1292
|
-
);
|
|
1293
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "story_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
|
-
if (/\bsearch images\b/.test(lower)) {
|
|
1297
|
-
const matching = objects.filter((row) =>
|
|
1298
|
-
hasAnyPath(row, ["repo_name", "name", "full_name"]) &&
|
|
1299
|
-
hasAnyPath(row, ["short_description", "description", "star_count", "pull_count", "url"])
|
|
1300
|
-
);
|
|
1301
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "image_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1302
|
-
}
|
|
1303
|
-
|
|
1304
|
-
if (/\b(get )?image tags\b/.test(lower)) {
|
|
1305
|
-
const matching = objects.filter((row) =>
|
|
1306
|
-
hasAnyPath(row, ["name", "tag"]) &&
|
|
1307
|
-
hasAnyPath(row, ["last_updated", "updated_at", "full_size", "digest"])
|
|
1308
|
-
);
|
|
1309
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "tag_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
if (/\b(company|companies|organization|organisations|business)\b/.test(lower)) {
|
|
1313
|
-
const matching = objects.filter((row) =>
|
|
1314
|
-
hasAnyPath(row, ["name", "title", "public_identifier"]) &&
|
|
1315
|
-
hasAnyPath(row, ["description", "industry", "url", "website", "employee_count", "follower_count"])
|
|
1316
|
-
);
|
|
1317
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "company_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1318
|
-
}
|
|
1319
|
-
|
|
1320
|
-
if (/\b(person|people|profile|profiles|member|members|user|users)\b/.test(lower)) {
|
|
1321
|
-
const matching = objects.filter((row) =>
|
|
1322
|
-
hasAnyPath(row, ["name", "title", "public_identifier", "username"]) &&
|
|
1323
|
-
hasAnyPath(row, ["headline", "url", "public_identifier", "username"])
|
|
1324
|
-
);
|
|
1325
|
-
const minRows = /\b(profile|profiles|user|users|person)\b/.test(lower) && !/\bsearch\b/.test(lower) ? 1 : 2;
|
|
1326
|
-
return matching.length >= minRows ? { verdict: "pass", reason: "people_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1327
|
-
}
|
|
1328
|
-
|
|
1329
|
-
if (/\b(comment|comments)\b/.test(lower)) {
|
|
1330
|
-
const matching = objects.filter((row) =>
|
|
1331
|
-
hasAnyPath(row, ["id", "url", "permalink"]) &&
|
|
1332
|
-
hasAnyPath(row, ["author", "body", "text"])
|
|
1333
|
-
);
|
|
1334
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "comment_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1335
|
-
}
|
|
1336
|
-
|
|
1337
|
-
if (/\b(email|emails|mail|inbox)\b/.test(lower)) {
|
|
1338
|
-
const matching = objects.filter((row) =>
|
|
1339
|
-
hasAnyPath(row, ["id", "thread.doc_id", "matchedEmail.id"]) &&
|
|
1340
|
-
hasAnyPath(row, ["subject", "thread.subject", "from", "sender", "thread.latest_sender_name"]) &&
|
|
1341
|
-
hasAnyPath(row, ["date", "thread.formatted_date", "preview", "thread.preview", "matchedEmail.content_markdown"])
|
|
1342
|
-
);
|
|
1343
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "email_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1344
|
-
}
|
|
1345
|
-
|
|
1346
|
-
if (/\b(post|posts|tweet|tweets|status|statuses)\b/.test(lower)) {
|
|
1347
|
-
const matching = objects.filter((row) =>
|
|
1348
|
-
hasAnyPath(row, ["id", "url", "uri", "permalink", "_id", "entityUrn", "rest_id"]) &&
|
|
1349
|
-
countMatchingGroups(row, [
|
|
1350
|
-
["content", "text", "title", "body", "message", "description", "summary", "commentary.text.text"],
|
|
1351
|
-
["account.username", "account.acct", "username", "author", "user.name", "from.name", "actor.name", "actor.name.text"],
|
|
1352
|
-
["score", "points", "likes", "favorite_count", "num_comments", "reply_count", "reactions", "repost_count"],
|
|
1353
|
-
["date", "created_at", "published_at", "timestamp", "meta", "created_time", "createdAt", "updated_at", "time"],
|
|
1354
|
-
]) >= 2
|
|
1355
|
-
);
|
|
1356
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "post_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1357
|
-
}
|
|
1358
|
-
|
|
1359
|
-
if (/\b(reddit|subreddit)\b/.test(lower)) {
|
|
1360
|
-
const matching = objects.filter((row) =>
|
|
1361
|
-
hasAnyPath(row, ["id", "url", "permalink"]) &&
|
|
1362
|
-
hasAnyPath(row, ["title", "subreddit", "author"]) &&
|
|
1363
|
-
hasAnyPath(row, ["num_comments", "score", "permalink"])
|
|
1364
|
-
);
|
|
1365
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "reddit_post_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1366
|
-
}
|
|
1367
|
-
|
|
1368
|
-
if (/\b(topic|topics|trend|trending|hashtag|hashtags)\b/.test(lower)) {
|
|
1369
|
-
const matching = objects.filter((row) =>
|
|
1370
|
-
hasAnyPath(row, ["name", "title", "query"]) &&
|
|
1371
|
-
hasAnyPath(row, ["url", "name", "query"])
|
|
1372
|
-
);
|
|
1373
|
-
return matching.length >= 2 ? { verdict: "pass", reason: "topic_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1374
|
-
}
|
|
1375
|
-
|
|
1376
|
-
if (/\b(doc|docs|documentation)\b/.test(lower)) {
|
|
1377
|
-
const matching = objects.filter((row) =>
|
|
1378
|
-
hasAnyPath(row, ["title", "name", "slug"]) &&
|
|
1379
|
-
hasAnyPath(row, ["url", "link", "href", "mdn_url"]) &&
|
|
1380
|
-
hasAnyPath(row, ["summary", "description", "slug", "popularity", "score"])
|
|
1381
|
-
);
|
|
1382
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "document_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1383
|
-
}
|
|
1384
|
-
|
|
1385
|
-
if (/\b(paper|papers)\b/.test(lower)) {
|
|
1386
|
-
const matching = objects.filter((row) =>
|
|
1387
|
-
hasAnyPath(row, ["title", "name"]) &&
|
|
1388
|
-
hasAnyPath(row, ["url", "link", "href"]) &&
|
|
1389
|
-
hasAnyPath(row, ["summary", "description", "author", "authors", "date", "published", "meta"])
|
|
1390
|
-
);
|
|
1391
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "paper_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1392
|
-
}
|
|
1393
|
-
|
|
1394
|
-
if (/\b(question|questions)\b/.test(lower)) {
|
|
1395
|
-
const matching = objects.filter((row) =>
|
|
1396
|
-
hasAnyPath(row, ["title", "name"]) &&
|
|
1397
|
-
hasAnyPath(row, ["url", "link", "href", "permalink"]) &&
|
|
1398
|
-
countMatchingGroups(row, [
|
|
1399
|
-
["votes", "score", "points"],
|
|
1400
|
-
["answers", "answer_count", "num_answers", "num_comments"],
|
|
1401
|
-
["author", "date", "created_at", "meta"],
|
|
1402
|
-
]) >= 2
|
|
1403
|
-
);
|
|
1404
|
-
return matching.length >= 2 ? { verdict: "pass", reason: "question_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1405
|
-
}
|
|
1406
|
-
|
|
1407
|
-
if (/\b(recipe|recipes)\b/.test(lower)) {
|
|
1408
|
-
const matching = objects.filter((row) =>
|
|
1409
|
-
hasAnyPath(row, ["title", "name"]) &&
|
|
1410
|
-
hasAnyPath(row, ["url", "link", "href"]) &&
|
|
1411
|
-
hasAnyPath(row, ["rating", "review_count", "author", "description", "summary", "cook_time", "total_time"])
|
|
1412
|
-
);
|
|
1413
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "recipe_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1414
|
-
}
|
|
1415
|
-
|
|
1416
|
-
if (/\b(course|courses)\b/.test(lower)) {
|
|
1417
|
-
const matching = objects.filter((row) =>
|
|
1418
|
-
hasAnyPath(row, ["title", "name"]) &&
|
|
1419
|
-
hasAnyPath(row, ["url", "link", "href"]) &&
|
|
1420
|
-
hasAnyPath(row, ["rating", "partner", "provider", "instructor", "description", "summary", "duration", "level"])
|
|
1421
|
-
);
|
|
1422
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "course_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1423
|
-
}
|
|
1424
|
-
|
|
1425
|
-
if (/\b(definition|dictionary|meaning)\b/.test(lower)) {
|
|
1426
|
-
const matching = objects.filter((row) =>
|
|
1427
|
-
hasAnyPath(row, ["term", "title", "name", "word"]) &&
|
|
1428
|
-
hasAnyPath(row, ["definition", "summary", "body", "description"])
|
|
1429
|
-
);
|
|
1430
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "definition_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1431
|
-
}
|
|
1432
|
-
|
|
1433
|
-
if (/\b(stock|stocks|ticker|tickers|quote|quotes)\b/.test(lower)) {
|
|
1434
|
-
const matching = objects.filter((row) =>
|
|
1435
|
-
hasAnyPath(row, ["symbol", "ticker"]) &&
|
|
1436
|
-
hasAnyPath(row, ["price", "regularMarketPrice", "current_price"]) &&
|
|
1437
|
-
hasAnyPath(row, ["name", "currency", "change_percent", "market_cap", "market_state"])
|
|
1438
|
-
);
|
|
1439
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "quote_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1440
|
-
}
|
|
1441
|
-
|
|
1442
|
-
if (/\b(product|products|item|items)\b/.test(lower)) {
|
|
1443
|
-
const matching = objects.filter((row) =>
|
|
1444
|
-
hasAnyPath(row, ["title", "name", "product_name"]) &&
|
|
1445
|
-
(
|
|
1446
|
-
hasAnyPath(row, ["url", "id", "product_id", "sku"]) ||
|
|
1447
|
-
hasAnyPath(row, ["description", "summary"])
|
|
1448
|
-
) &&
|
|
1449
|
-
hasAnyPath(row, ["price", "rating", "review_count", "brand", "image"]) &&
|
|
1450
|
-
!/^results?$/i.test(String(getPath(row, "title") ?? getPath(row, "name") ?? "")) &&
|
|
1451
|
-
!Object.keys(row).some((key) => /^heading_\d+$/i.test(key)) &&
|
|
1452
|
-
!/(amazon-adsystem|doubleclick|googlesyndication|pubmatic|taboola|outbrain|aax-|googleadservices|adservice)/i.test(
|
|
1453
|
-
String(getPath(row, "url") ?? ""),
|
|
1454
|
-
)
|
|
1455
|
-
);
|
|
1456
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "product_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1457
|
-
}
|
|
1458
|
-
|
|
1459
|
-
if (/\b(channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(lower)) {
|
|
1460
|
-
const matching = objects.filter((row) =>
|
|
1461
|
-
hasAnyPath(row, ["name", "title"]) &&
|
|
1462
|
-
hasAnyPath(row, ["id", "url", "channel_id", "guild_id", "workspace_id"]) &&
|
|
1463
|
-
hasAnyPath(row, ["description", "type", "member_count", "url"])
|
|
1464
|
-
);
|
|
1465
|
-
return matching.length >= 1 ? { verdict: "pass", reason: "channel_rows" } : { verdict: "skip", reason: "wrong_entity_type" };
|
|
1466
|
-
}
|
|
1467
|
-
|
|
1468
|
-
return { verdict: "skip", reason: "unclassified_array" };
|
|
1469
|
-
}
|
|
1470
|
-
|
|
1471
|
-
export function assessIntentResult(data: unknown, intent?: string): {
|
|
1472
|
-
verdict: "pass" | "fail" | "skip";
|
|
1473
|
-
reason: string;
|
|
1474
|
-
projected: unknown;
|
|
1475
|
-
} {
|
|
1476
|
-
const projected = projectIntentData(data, intent);
|
|
1477
|
-
|
|
1478
|
-
if (projected == null) return { verdict: "fail", reason: "no_data", projected };
|
|
1479
|
-
if (typeof projected === "string") {
|
|
1480
|
-
const trimmed = projected.trim().toLowerCase();
|
|
1481
|
-
if (!trimmed) return { verdict: "fail", reason: "empty_text", projected };
|
|
1482
|
-
if (trimmed.startsWith("<!doctype") || trimmed.startsWith("<html") || trimmed.startsWith("<body")) {
|
|
1483
|
-
return { verdict: "fail", reason: "html_payload", projected };
|
|
1484
|
-
}
|
|
1485
|
-
return { verdict: "skip", reason: "plain_text", projected };
|
|
1486
|
-
}
|
|
1487
|
-
|
|
1488
|
-
if (Array.isArray(projected)) {
|
|
1489
|
-
const classified = classifyRows(projected, intent ?? "");
|
|
1490
|
-
return { ...classified, projected };
|
|
1491
|
-
}
|
|
1492
|
-
|
|
1493
|
-
if (isRecord(projected)) {
|
|
1494
|
-
if ("error" in projected) return { verdict: "fail", reason: String(projected.error || "error_payload"), projected };
|
|
1495
|
-
if ("available_endpoints" in projected || "available_operations" in projected) {
|
|
1496
|
-
return { verdict: "fail", reason: "deferral_payload", projected };
|
|
1497
|
-
}
|
|
1498
|
-
if ("learned_skill_id" in projected && !("data" in projected)) {
|
|
1499
|
-
return { verdict: "fail", reason: "learned_skill_only", projected };
|
|
1500
|
-
}
|
|
1501
|
-
const lower = (intent ?? "").toLowerCase();
|
|
1502
|
-
if (("message" in projected || "flash" in projected) && !("data" in projected)) {
|
|
1503
|
-
if (/\b(message|messages|flash|alert|success|error|warning)\b/.test(lower)) {
|
|
1504
|
-
return { verdict: "skip", reason: "message_record", projected };
|
|
1505
|
-
}
|
|
1506
|
-
return { verdict: "fail", reason: "message_only", projected };
|
|
1507
|
-
}
|
|
1508
|
-
if (/\b(company|companies|organization|organisations|business|person|people|profile|profiles|member|members|user|users|repo|repository|repositories|project|projects|package|packages|doc|docs|documentation|question|questions|recipe|recipes|course|courses|definition|dictionary|meaning|product|products|item|items|stock|stocks|ticker|tickers|quote|quotes|channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(lower)) {
|
|
1509
|
-
const classified = classifyRows([projected], intent ?? "");
|
|
1510
|
-
return { ...classified, projected };
|
|
1511
|
-
}
|
|
1512
|
-
}
|
|
1513
|
-
|
|
1514
|
-
return { verdict: "skip", reason: "unclassified_payload", projected };
|
|
1515
|
-
}
|