pi-provider-kimi-code 0.6.12 → 0.6.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +25 -7
- package/index.ts +218 -170
- package/package.json +16 -13
- package/src/config.ts +33 -1
- package/src/constants.ts +86 -9
- package/src/errors.ts +31 -0
- package/src/models.ts +95 -13
- package/src/oauth.ts +60 -1
- package/src/payload.ts +266 -1
- package/src/schema-dedup.ts +8 -0
- package/src/settings-ui.ts +28 -0
- package/src/stream.ts +17 -3
- package/src/tools/common.ts +2 -3
- package/src/tools/datasource-catalog.ts +420 -0
- package/src/tools/datasource.ts +54 -68
- package/src/tools/moonshot.ts +7 -7
- package/src/usage.ts +116 -7
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
// Lazy, cached sync of the kimi-datasource catalog.
|
|
2
|
+
//
|
|
3
|
+
// The upstream kimi-datasource plugin moved out of the kimi-code repo into the
|
|
4
|
+
// server-side plugin marketplace (#4525). There is no "list sources" RPC on the
|
|
5
|
+
// /tools backend: the authoritative source list is the `enum` inside the
|
|
6
|
+
// plugin's `bin/kimi-datasource.mjs` tool schema. So the sync pipeline is:
|
|
7
|
+
//
|
|
8
|
+
// GET {base}/coding/v1/plugins/marketplace (no auth required)
|
|
9
|
+
// -> entry "kimi-datasource": { version, source (zip URL), sha256sum }
|
|
10
|
+
// -> zip -> bin/kimi-datasource.mjs -> parse enum + capability text
|
|
11
|
+
//
|
|
12
|
+
// The result is cached on disk for CATALOG_TTL_MS (24h). Callers fall back to
|
|
13
|
+
// stale cache, then to the bundled snapshot, so tool calls never fail just
|
|
14
|
+
// because the marketplace is unreachable.
|
|
15
|
+
|
|
16
|
+
import { createHash } from "node:crypto";
|
|
17
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
18
|
+
import { dirname, join } from "node:path";
|
|
19
|
+
import { inflateRawSync } from "node:zlib";
|
|
20
|
+
|
|
21
|
+
import { getKimiCodeHome } from "../constants.ts";
|
|
22
|
+
import { getKimiBaseV1 } from "./common.ts";
|
|
23
|
+
|
|
24
|
+
export const CATALOG_TTL_MS = 24 * 60 * 60 * 1000;
|
|
25
|
+
// After a failed refresh, suppress retries for an hour instead of a full day.
|
|
26
|
+
const CATALOG_RETRY_BACKOFF_MS = 60 * 60 * 1000;
|
|
27
|
+
const CATALOG_FETCH_TIMEOUT_MS = 15_000;
|
|
28
|
+
const MARKETPLACE_PLUGIN_ID = "kimi-datasource";
|
|
29
|
+
const BIN_ENTRY_SUFFIX = "bin/kimi-datasource.mjs";
|
|
30
|
+
|
|
31
|
+
// Bundled snapshot of marketplace kimi-datasource 3.5.1 (2026-11-03). Used when
|
|
32
|
+
// no cache exists and the marketplace cannot be reached.
|
|
33
|
+
export const BUNDLED_DATASOURCE_VERSION = "3.5.1";
|
|
34
|
+
export const BUNDLED_DATASOURCE_SOURCES: readonly string[] = [
|
|
35
|
+
"stock_finance_data",
|
|
36
|
+
"yahoo_finance",
|
|
37
|
+
"world_bank_open_data",
|
|
38
|
+
"tianyancha",
|
|
39
|
+
"arxiv",
|
|
40
|
+
"scholar",
|
|
41
|
+
"yuandian_law",
|
|
42
|
+
"wind",
|
|
43
|
+
"imf",
|
|
44
|
+
"gildata",
|
|
45
|
+
"sec_edgar",
|
|
46
|
+
"sp_data",
|
|
47
|
+
"china_nda",
|
|
48
|
+
"china_nbs",
|
|
49
|
+
"china_standards",
|
|
50
|
+
"who",
|
|
51
|
+
"fao",
|
|
52
|
+
"unsd",
|
|
53
|
+
"ecb",
|
|
54
|
+
"eurostat",
|
|
55
|
+
"unicef",
|
|
56
|
+
"oecd",
|
|
57
|
+
"fred",
|
|
58
|
+
"xhcj",
|
|
59
|
+
"caixin",
|
|
60
|
+
"chembl",
|
|
61
|
+
"materials_project",
|
|
62
|
+
"ncbi_blast",
|
|
63
|
+
"oqmd",
|
|
64
|
+
"pdb",
|
|
65
|
+
"pubchem",
|
|
66
|
+
"uniprot",
|
|
67
|
+
"dongcai",
|
|
68
|
+
"cls",
|
|
69
|
+
"finance_research",
|
|
70
|
+
];
|
|
71
|
+
|
|
72
|
+
export const BUNDLED_DATASOURCE_CAPABILITIES =
|
|
73
|
+
"stock_finance_data / yahoo_finance = general quotes and financials " +
|
|
74
|
+
"(yahoo_finance FX history is limited to about 2 years); world_bank_open_data = historical macro; " +
|
|
75
|
+
"imf = FX rates, CPI, GDP forecasts, balance of payments; tianyancha = CN company registry; " +
|
|
76
|
+
"arxiv / scholar = papers; yuandian_law = CN laws and cases; " +
|
|
77
|
+
"wind = A-share intraday minute series, funds, bonds (map PE/PB/ROE-style field names via wind_search_fields first); " +
|
|
78
|
+
"gildata = natural-language stock/fund screening; " +
|
|
79
|
+
"sec_edgar = US filings (10-K/10-Q, S-1, Form 4, 13F, 8-K); " +
|
|
80
|
+
"sp_data = S&P fundamentals (consensus estimates, valuation ratios, transcripts); " +
|
|
81
|
+
"china_nda = CN government open data catalogs; china_nbs = CN NBS macro indicators and time series; " +
|
|
82
|
+
"china_standards = CN standards (GB national, HB industry, DB local, TT association); " +
|
|
83
|
+
"who / fao / unsd / ecb / eurostat / unicef / oecd / fred = international organization open data " +
|
|
84
|
+
"(global health, food & agriculture, UN statistics, ECB & EU statistics, child indicators, OECD datasets, US & global macro series); " +
|
|
85
|
+
"xhcj = Xinhua Finance (CNFIC) news flashes, announcements, and policies; " +
|
|
86
|
+
"caixin = Caixin database (600+ data APIs, discover via caixin_api_search first); " +
|
|
87
|
+
"chembl = curated ChEMBL compounds, targets, assays, bioactivities, and mechanisms of action; " +
|
|
88
|
+
"materials_project = Materials Project DFT properties and MP ID lookup (requires MP_API_KEY); " +
|
|
89
|
+
"ncbi_blast = asynchronous nucleotide/protein sequence similarity searches; " +
|
|
90
|
+
"oqmd = OQMD DFT material properties and OPTIMADE crystal structures; " +
|
|
91
|
+
"pdb = RCSB PDB experimental 3D structures, polymer entities, and ligands; " +
|
|
92
|
+
"pubchem = PubChem chemical identities, synonyms, computed properties, and 2D similarity; " +
|
|
93
|
+
"uniprot = UniProt protein sequences, functions, and annotations; " +
|
|
94
|
+
"dongcai = East Money Miaoxiang market intelligence; cls = Cailian Press financial news; " +
|
|
95
|
+
"finance_research = sell-side research reports";
|
|
96
|
+
|
|
97
|
+
export interface DatasourceCatalog {
|
|
98
|
+
readonly version: string;
|
|
99
|
+
readonly sources: readonly string[];
|
|
100
|
+
readonly capabilities: string;
|
|
101
|
+
readonly fetchedAt: number;
|
|
102
|
+
readonly origin: "network" | "cache" | "bundled";
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
interface CatalogCacheFile {
|
|
106
|
+
readonly version?: unknown;
|
|
107
|
+
readonly sources?: unknown;
|
|
108
|
+
readonly capabilities?: unknown;
|
|
109
|
+
readonly fetchedAt?: unknown;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function catalogCachePath(): string {
|
|
113
|
+
const explicit = process.env.KIMI_DATASOURCE_CATALOG_PATH?.trim();
|
|
114
|
+
if (explicit) return explicit;
|
|
115
|
+
return join(getKimiCodeHome(), "kimi-datasource-catalog.json");
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function parseCacheFile(raw: string): DatasourceCatalog | undefined {
|
|
119
|
+
let parsed: CatalogCacheFile;
|
|
120
|
+
try {
|
|
121
|
+
parsed = JSON.parse(raw) as CatalogCacheFile;
|
|
122
|
+
} catch {
|
|
123
|
+
return undefined;
|
|
124
|
+
}
|
|
125
|
+
if (typeof parsed.version !== "string" || !Array.isArray(parsed.sources)) return undefined;
|
|
126
|
+
const sources = parsed.sources.filter((s): s is string => typeof s === "string" && s.length > 0);
|
|
127
|
+
if (sources.length === 0) return undefined;
|
|
128
|
+
const fetchedAt = typeof parsed.fetchedAt === "number" ? parsed.fetchedAt : 0;
|
|
129
|
+
return {
|
|
130
|
+
version: parsed.version,
|
|
131
|
+
sources,
|
|
132
|
+
capabilities:
|
|
133
|
+
typeof parsed.capabilities === "string" && parsed.capabilities.length > 0
|
|
134
|
+
? parsed.capabilities
|
|
135
|
+
: BUNDLED_DATASOURCE_CAPABILITIES,
|
|
136
|
+
fetchedAt,
|
|
137
|
+
origin: "cache",
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Synchronous cache read for tool-registration time (description embedding). */
|
|
142
|
+
export function readCatalogCacheSync(): DatasourceCatalog | undefined {
|
|
143
|
+
try {
|
|
144
|
+
const path = catalogCachePath();
|
|
145
|
+
if (!existsSync(path)) return undefined;
|
|
146
|
+
return parseCacheFile(readFileSync(path, "utf8"));
|
|
147
|
+
} catch {
|
|
148
|
+
return undefined;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function writeCatalogCache(catalog: DatasourceCatalog): void {
|
|
153
|
+
try {
|
|
154
|
+
const path = catalogCachePath();
|
|
155
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
156
|
+
writeFileSync(
|
|
157
|
+
path,
|
|
158
|
+
`${JSON.stringify(
|
|
159
|
+
{
|
|
160
|
+
version: catalog.version,
|
|
161
|
+
sources: catalog.sources,
|
|
162
|
+
capabilities: catalog.capabilities,
|
|
163
|
+
fetchedAt: catalog.fetchedAt,
|
|
164
|
+
},
|
|
165
|
+
null,
|
|
166
|
+
2,
|
|
167
|
+
)}\n`,
|
|
168
|
+
"utf8",
|
|
169
|
+
);
|
|
170
|
+
} catch {
|
|
171
|
+
// Cache is best-effort; a read-only home dir must not break the tool.
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// ---------------------------------------------------------------------------
|
|
176
|
+
// Minimal zip reader (stored + deflate), enough to pull one text entry.
|
|
177
|
+
// ---------------------------------------------------------------------------
|
|
178
|
+
|
|
179
|
+
function findEndOfCentralDirectory(buf: Buffer): number {
|
|
180
|
+
const min = Math.max(0, buf.length - 66_000);
|
|
181
|
+
for (let i = buf.length - 22; i >= min; i--) {
|
|
182
|
+
if (buf.readUInt32LE(i) === 0x06054b50) return i;
|
|
183
|
+
}
|
|
184
|
+
throw new Error("invalid zip: end of central directory not found");
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function extractZipEntry(buf: Buffer, entrySuffix: string): Buffer | undefined {
|
|
188
|
+
const eocd = findEndOfCentralDirectory(buf);
|
|
189
|
+
const entryCount = buf.readUInt16LE(eocd + 10);
|
|
190
|
+
let offset = buf.readUInt32LE(eocd + 16);
|
|
191
|
+
for (let i = 0; i < entryCount; i++) {
|
|
192
|
+
if (buf.readUInt32LE(offset) !== 0x02014b50) throw new Error("invalid zip: bad central entry");
|
|
193
|
+
const method = buf.readUInt16LE(offset + 10);
|
|
194
|
+
const compressedSize = buf.readUInt32LE(offset + 20);
|
|
195
|
+
const nameLen = buf.readUInt16LE(offset + 28);
|
|
196
|
+
const extraLen = buf.readUInt16LE(offset + 30);
|
|
197
|
+
const commentLen = buf.readUInt16LE(offset + 32);
|
|
198
|
+
const localOffset = buf.readUInt32LE(offset + 42);
|
|
199
|
+
const name = buf.subarray(offset + 46, offset + 46 + nameLen).toString("utf8");
|
|
200
|
+
offset += 46 + nameLen + extraLen + commentLen;
|
|
201
|
+
if (name !== entrySuffix && !name.endsWith(`/${entrySuffix}`)) continue;
|
|
202
|
+
|
|
203
|
+
if (buf.readUInt32LE(localOffset) !== 0x04034b50)
|
|
204
|
+
throw new Error("invalid zip: bad local header");
|
|
205
|
+
const localNameLen = buf.readUInt16LE(localOffset + 26);
|
|
206
|
+
const localExtraLen = buf.readUInt16LE(localOffset + 28);
|
|
207
|
+
const dataStart = localOffset + 30 + localNameLen + localExtraLen;
|
|
208
|
+
const data = buf.subarray(dataStart, dataStart + compressedSize);
|
|
209
|
+
if (method === 0) return Buffer.from(data);
|
|
210
|
+
if (method === 8) return inflateRawSync(data);
|
|
211
|
+
throw new Error(`unsupported zip compression method: ${method}`);
|
|
212
|
+
}
|
|
213
|
+
return undefined;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// ---------------------------------------------------------------------------
|
|
217
|
+
// Parse the datasource bin script: source enum + capability description.
|
|
218
|
+
// ---------------------------------------------------------------------------
|
|
219
|
+
|
|
220
|
+
export function parseDatasourceBin(script: string): {
|
|
221
|
+
sources: string[];
|
|
222
|
+
capabilities?: string;
|
|
223
|
+
} {
|
|
224
|
+
const enumMatch = /enum:\s*\[((?:\s*'[^']*',?\s*)+)\]/.exec(script);
|
|
225
|
+
if (!enumMatch) throw new Error("datasource bin: enum block not found");
|
|
226
|
+
const sources = [...enumMatch[1].matchAll(/'([^']+)'/g)]
|
|
227
|
+
.map((m) => m[1])
|
|
228
|
+
.filter((s) => s.length > 0);
|
|
229
|
+
if (sources.length === 0) throw new Error("datasource bin: enum block is empty");
|
|
230
|
+
|
|
231
|
+
let capabilities: string | undefined;
|
|
232
|
+
const descMatch = /description:\s*((?:\s*'[^']*'\s*\+?\s*)+)/.exec(script.slice(enumMatch.index));
|
|
233
|
+
if (descMatch) {
|
|
234
|
+
const text = [...descMatch[1].matchAll(/'([^']*)'/g)].map((m) => m[1]).join("");
|
|
235
|
+
if (text.length > 0) capabilities = text;
|
|
236
|
+
}
|
|
237
|
+
return { sources, capabilities };
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// ---------------------------------------------------------------------------
|
|
241
|
+
// Marketplace fetch
|
|
242
|
+
// ---------------------------------------------------------------------------
|
|
243
|
+
|
|
244
|
+
interface MarketplaceEntry {
|
|
245
|
+
readonly version?: unknown;
|
|
246
|
+
readonly source?: unknown;
|
|
247
|
+
readonly sha256sum?: unknown;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
async function fetchMarketplaceEntry(
|
|
251
|
+
fetchImpl: typeof fetch,
|
|
252
|
+
signal: AbortSignal,
|
|
253
|
+
): Promise<{ version: string; source: string; sha256sum?: string } | undefined> {
|
|
254
|
+
const url = `${getKimiBaseV1()}/plugins/marketplace`;
|
|
255
|
+
const response = await fetchImpl(url, { signal });
|
|
256
|
+
if (!response.ok) throw new Error(`marketplace fetch failed: HTTP ${response.status}`);
|
|
257
|
+
const body = (await response.json()) as { plugins?: MarketplaceEntry[] };
|
|
258
|
+
const entry = (body.plugins ?? []).find((p) => {
|
|
259
|
+
const id = (p as Record<string, unknown>).id;
|
|
260
|
+
return id === MARKETPLACE_PLUGIN_ID;
|
|
261
|
+
});
|
|
262
|
+
if (!entry || typeof entry.source !== "string" || entry.source.length === 0) return undefined;
|
|
263
|
+
return {
|
|
264
|
+
version: typeof entry.version === "string" ? entry.version : "unknown",
|
|
265
|
+
source: entry.source,
|
|
266
|
+
sha256sum: typeof entry.sha256sum === "string" ? entry.sha256sum : undefined,
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
async function fetchCatalogFromNetwork(
|
|
271
|
+
fetchImpl: typeof fetch,
|
|
272
|
+
cached: DatasourceCatalog | undefined,
|
|
273
|
+
now: number,
|
|
274
|
+
signal?: AbortSignal,
|
|
275
|
+
): Promise<DatasourceCatalog | undefined> {
|
|
276
|
+
const timeout = AbortSignal.timeout(CATALOG_FETCH_TIMEOUT_MS);
|
|
277
|
+
const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
278
|
+
const entry = await fetchMarketplaceEntry(fetchImpl, combined);
|
|
279
|
+
if (!entry) return undefined;
|
|
280
|
+
|
|
281
|
+
// Same marketplace version as the cache: the catalog cannot have changed,
|
|
282
|
+
// so just renew the TTL without downloading the archive again.
|
|
283
|
+
if (cached && entry.version === cached.version) {
|
|
284
|
+
const renewed: DatasourceCatalog = { ...cached, fetchedAt: now, origin: "cache" };
|
|
285
|
+
writeCatalogCache(renewed);
|
|
286
|
+
return renewed;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
const zipResponse = await fetchImpl(entry.source, { signal: combined });
|
|
290
|
+
if (!zipResponse.ok) throw new Error(`archive fetch failed: HTTP ${zipResponse.status}`);
|
|
291
|
+
const zipBuf = Buffer.from(await zipResponse.arrayBuffer());
|
|
292
|
+
if (entry.sha256sum) {
|
|
293
|
+
const digest = createHash("sha256").update(zipBuf).digest("hex");
|
|
294
|
+
if (digest !== entry.sha256sum.toLowerCase()) {
|
|
295
|
+
throw new Error("archive sha256 mismatch");
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
const bin = extractZipEntry(zipBuf, BIN_ENTRY_SUFFIX);
|
|
299
|
+
if (!bin) throw new Error("archive is missing the datasource bin entry");
|
|
300
|
+
const parsed = parseDatasourceBin(bin.toString("utf8"));
|
|
301
|
+
const catalog: DatasourceCatalog = {
|
|
302
|
+
version: entry.version,
|
|
303
|
+
sources: parsed.sources,
|
|
304
|
+
capabilities: parsed.capabilities ?? BUNDLED_DATASOURCE_CAPABILITIES,
|
|
305
|
+
fetchedAt: now,
|
|
306
|
+
origin: "network",
|
|
307
|
+
};
|
|
308
|
+
writeCatalogCache(catalog);
|
|
309
|
+
return catalog;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// ---------------------------------------------------------------------------
|
|
313
|
+
// Public lazy accessor with 24h caching. Never throws.
|
|
314
|
+
// ---------------------------------------------------------------------------
|
|
315
|
+
|
|
316
|
+
let memoryCatalog: DatasourceCatalog | undefined;
|
|
317
|
+
let inFlight: Promise<DatasourceCatalog> | undefined;
|
|
318
|
+
|
|
319
|
+
export interface EnsureCatalogOptions {
|
|
320
|
+
readonly fetchImpl?: typeof fetch;
|
|
321
|
+
/** Skip the TTL check and hit the network (used to re-validate unknown names). */
|
|
322
|
+
readonly forceRefresh?: boolean;
|
|
323
|
+
readonly signal?: AbortSignal;
|
|
324
|
+
readonly now?: () => number;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
function bundledCatalog(): DatasourceCatalog {
|
|
328
|
+
return {
|
|
329
|
+
version: BUNDLED_DATASOURCE_VERSION,
|
|
330
|
+
sources: BUNDLED_DATASOURCE_SOURCES,
|
|
331
|
+
capabilities: BUNDLED_DATASOURCE_CAPABILITIES,
|
|
332
|
+
fetchedAt: 0,
|
|
333
|
+
origin: "bundled",
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Synchronous best-known catalog for validating a call without blocking on
|
|
339
|
+
* the network: memory, then disk (any age), then the bundled snapshot.
|
|
340
|
+
*/
|
|
341
|
+
export function peekDatasourceCatalog(): DatasourceCatalog {
|
|
342
|
+
return memoryCatalog ?? readCatalogCacheSync() ?? bundledCatalog();
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/** Fire-and-forget refresh when the best-known catalog is past its TTL. */
|
|
346
|
+
export function refreshDatasourceCatalogInBackground(fetchImpl?: typeof fetch): void {
|
|
347
|
+
const now = Date.now();
|
|
348
|
+
if (memoryCatalog && now - memoryCatalog.fetchedAt < CATALOG_TTL_MS) return;
|
|
349
|
+
const disk = readCatalogCacheSync();
|
|
350
|
+
if (disk && now - disk.fetchedAt < CATALOG_TTL_MS) {
|
|
351
|
+
memoryCatalog = disk;
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
void refreshCatalog(fetchImpl ?? fetch, disk, now);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
export async function ensureDatasourceCatalog(
|
|
358
|
+
options: EnsureCatalogOptions = {},
|
|
359
|
+
): Promise<DatasourceCatalog> {
|
|
360
|
+
const now = options.now?.() ?? Date.now();
|
|
361
|
+
const fetchImpl = options.fetchImpl ?? fetch;
|
|
362
|
+
|
|
363
|
+
if (!options.forceRefresh) {
|
|
364
|
+
if (memoryCatalog && now - memoryCatalog.fetchedAt < CATALOG_TTL_MS) return memoryCatalog;
|
|
365
|
+
const disk = readCatalogCacheSync();
|
|
366
|
+
if (disk && now - disk.fetchedAt < CATALOG_TTL_MS) {
|
|
367
|
+
memoryCatalog = disk;
|
|
368
|
+
return disk;
|
|
369
|
+
}
|
|
370
|
+
return refreshCatalog(fetchImpl, disk, now, options.signal);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// Forced re-validation: network only, keep the previous catalog on failure.
|
|
374
|
+
const previous = memoryCatalog ?? readCatalogCacheSync();
|
|
375
|
+
try {
|
|
376
|
+
const fresh = await fetchCatalogFromNetwork(fetchImpl, previous, now, options.signal);
|
|
377
|
+
if (fresh) {
|
|
378
|
+
memoryCatalog = fresh;
|
|
379
|
+
return fresh;
|
|
380
|
+
}
|
|
381
|
+
} catch {
|
|
382
|
+
// fall through to whatever we had
|
|
383
|
+
}
|
|
384
|
+
return previous ?? bundledCatalog();
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function refreshCatalog(
|
|
388
|
+
fetchImpl: typeof fetch,
|
|
389
|
+
disk: DatasourceCatalog | undefined,
|
|
390
|
+
now: number,
|
|
391
|
+
signal?: AbortSignal,
|
|
392
|
+
): Promise<DatasourceCatalog> {
|
|
393
|
+
if (inFlight) return inFlight;
|
|
394
|
+
inFlight = (async () => {
|
|
395
|
+
try {
|
|
396
|
+
const fresh = await fetchCatalogFromNetwork(fetchImpl, disk, now, signal);
|
|
397
|
+
if (fresh) {
|
|
398
|
+
memoryCatalog = fresh;
|
|
399
|
+
return fresh;
|
|
400
|
+
}
|
|
401
|
+
} catch {
|
|
402
|
+
// fall through to stale/bundled
|
|
403
|
+
} finally {
|
|
404
|
+
inFlight = undefined;
|
|
405
|
+
}
|
|
406
|
+
const fallback = disk ?? bundledCatalog();
|
|
407
|
+
// Mark the fallback with the attempt time (minus most of the TTL) so a
|
|
408
|
+
// network outage does not trigger a fetch on every tool call, but a
|
|
409
|
+
// transient failure still retries after the backoff window.
|
|
410
|
+
memoryCatalog = { ...fallback, fetchedAt: now - (CATALOG_TTL_MS - CATALOG_RETRY_BACKOFF_MS) };
|
|
411
|
+
return memoryCatalog;
|
|
412
|
+
})();
|
|
413
|
+
return inFlight;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/** Test hook: drop the in-memory catalog and in-flight refresh. */
|
|
417
|
+
export function resetDatasourceCatalogForTests(): void {
|
|
418
|
+
memoryCatalog = undefined;
|
|
419
|
+
inFlight = undefined;
|
|
420
|
+
}
|
package/src/tools/datasource.ts
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
import { defineTool } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import {
|
|
2
|
+
import { map, obj, opt, str, unknown, type Static } from "@earendil-works/pi-schema";
|
|
3
|
+
|
|
4
|
+
import {
|
|
5
|
+
ensureDatasourceCatalog,
|
|
6
|
+
peekDatasourceCatalog,
|
|
7
|
+
readCatalogCacheSync,
|
|
8
|
+
refreshDatasourceCatalogInBackground,
|
|
9
|
+
} from "./datasource-catalog.ts";
|
|
3
10
|
|
|
4
11
|
import {
|
|
5
12
|
type BuildKimiToolOptions,
|
|
@@ -15,72 +22,21 @@ import {
|
|
|
15
22
|
textComponent,
|
|
16
23
|
} from "./common.ts";
|
|
17
24
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
{
|
|
24
|
-
name: "yahoo_finance",
|
|
25
|
-
description: "Global finance: quotes, analyst ratings, options, indexes, historical prices.",
|
|
26
|
-
},
|
|
27
|
-
{
|
|
28
|
-
name: "world_bank_open_data",
|
|
29
|
-
description: "World Bank macro data: GDP, inflation, population, country comparisons.",
|
|
30
|
-
},
|
|
31
|
-
{
|
|
32
|
-
name: "tianyancha",
|
|
33
|
-
description: "Chinese enterprise registry: shareholders, legal representative, risks, patents.",
|
|
34
|
-
},
|
|
35
|
-
{ name: "arxiv", description: "arXiv preprints: paper search, abstracts, arXiv IDs." },
|
|
36
|
-
{
|
|
37
|
-
name: "scholar",
|
|
38
|
-
description: "Google Scholar: academic papers, authors, citations, high-citation surveys.",
|
|
39
|
-
},
|
|
40
|
-
{
|
|
41
|
-
name: "yuandian_law",
|
|
42
|
-
description: "Chinese laws, regulations, judicial cases; labor contracts and legal clauses.",
|
|
43
|
-
},
|
|
44
|
-
{
|
|
45
|
-
name: "wind",
|
|
25
|
+
// Source names are validated at call time against the synced catalog
|
|
26
|
+
// (datasource-catalog.ts) instead of a static enum, so sources added
|
|
27
|
+
// server-side never get rejected by a stale client schema.
|
|
28
|
+
export const kimiDatasourceSchema = obj({
|
|
29
|
+
data_source_name: str({
|
|
46
30
|
description:
|
|
47
|
-
"
|
|
48
|
-
},
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
description: "IMF macro data: FX rates, CPI, GDP forecasts, balance of payments.",
|
|
52
|
-
},
|
|
53
|
-
{
|
|
54
|
-
name: "gildata",
|
|
55
|
-
description: "Natural-language stock/fund screening.",
|
|
56
|
-
},
|
|
57
|
-
{
|
|
58
|
-
name: "sec_edgar",
|
|
59
|
-
description: "US SEC filings: 10-K/10-Q, S-1, Form 4, 13F, 8-K.",
|
|
60
|
-
},
|
|
61
|
-
{
|
|
62
|
-
name: "sp_data",
|
|
63
|
-
description: "S&P fundamentals: consensus estimates, valuation ratios, transcripts.",
|
|
64
|
-
},
|
|
65
|
-
] as const;
|
|
66
|
-
|
|
67
|
-
const DATASOURCE_NAMES = DATASOURCE_DEFINITIONS.map((source) => source.name);
|
|
68
|
-
|
|
69
|
-
export const kimiDatasourceSchema = Type.Object({
|
|
70
|
-
data_source_name: Type.Union(
|
|
71
|
-
DATASOURCE_NAMES.map((name) => Type.Literal(name)),
|
|
72
|
-
{
|
|
73
|
-
description:
|
|
74
|
-
"Source: stock_finance_data stocks/reports; yahoo_finance quotes/options; world_bank_open_data GDP/population; imf FX/CPI/GDP forecasts; tianyancha Chinese company registry/legal rep; arxiv preprints; scholar citations; yuandian_law Chinese laws/cases/contracts; wind A-share intraday minute series/funds/bonds (map field names via wind_search_fields first); gildata natural-language stock/fund screening; sec_edgar US filings (10-K/10-Q, S-1, Form 4, 13F, 8-K); sp_data S&P fundamentals (estimates, valuation, transcripts).",
|
|
75
|
-
},
|
|
76
|
-
),
|
|
77
|
-
api_name: Type.Optional(
|
|
78
|
-
Type.String({
|
|
31
|
+
"Data source name. The set of available sources is synced from the official kimi-datasource plugin and cached for 24h; if a call fails with an unknown-source error, pick one of the names listed in that error.",
|
|
32
|
+
}),
|
|
33
|
+
api_name: opt(
|
|
34
|
+
str({
|
|
79
35
|
description: "API name from the datasource docs. Omit to get docs first.",
|
|
80
36
|
}),
|
|
81
37
|
),
|
|
82
|
-
params:
|
|
83
|
-
|
|
38
|
+
params: opt(
|
|
39
|
+
map(unknown(), {
|
|
84
40
|
description: "Parameters matching the selected datasource API docs.",
|
|
85
41
|
}),
|
|
86
42
|
),
|
|
@@ -164,6 +120,16 @@ function buildTraceLine(toolCallId: string, requestId: string | undefined): stri
|
|
|
164
120
|
return `\n\n[kimi-datasource] ${parts.join(" · ")}`;
|
|
165
121
|
}
|
|
166
122
|
|
|
123
|
+
function buildDescription(): string {
|
|
124
|
+
const base =
|
|
125
|
+
"Call Kimi datasource APIs: finance/quotes, macro & government data, papers, Chinese law, chemistry/bio/materials, and more. First omit api_name to get docs, then call api_name with documented params.";
|
|
126
|
+
// Registration-time snapshot: embed the cached catalog so the model sees the
|
|
127
|
+
// current source list without any network I/O. Refreshed lazily on use.
|
|
128
|
+
const cached = readCatalogCacheSync();
|
|
129
|
+
if (!cached) return base;
|
|
130
|
+
return `${base} Known sources (kimi-datasource v${cached.version}): ${cached.capabilities}`;
|
|
131
|
+
}
|
|
132
|
+
|
|
167
133
|
export function buildKimiDatasourceTool(options: BuildKimiToolOptions = {}) {
|
|
168
134
|
const deps = buildKimiToolDeps(options);
|
|
169
135
|
const defaultCollapsed = options.defaultCollapsed ?? true;
|
|
@@ -171,12 +137,11 @@ export function buildKimiDatasourceTool(options: BuildKimiToolOptions = {}) {
|
|
|
171
137
|
return defineTool({
|
|
172
138
|
name: "kimi_datasource",
|
|
173
139
|
label: "Kimi Datasource",
|
|
174
|
-
description:
|
|
175
|
-
"Call Kimi datasource APIs. Covers stocks/finance, macro data, Chinese company registry, papers, and Chinese law/cases/contracts. First omit api_name to get docs, then call api_name with documented params.",
|
|
140
|
+
description: buildDescription(),
|
|
176
141
|
promptSnippet: "Query Kimi datasource APIs; get docs first, then call documented api_name",
|
|
177
142
|
parameters: kimiDatasourceSchema,
|
|
178
143
|
|
|
179
|
-
async execute(toolCallId, params, signal) {
|
|
144
|
+
async execute(toolCallId, params: KimiDatasourceParams, signal) {
|
|
180
145
|
const accessToken = deps.getAccessToken();
|
|
181
146
|
if (!accessToken) {
|
|
182
147
|
return errorResult<string>(
|
|
@@ -186,19 +151,40 @@ export function buildKimiDatasourceTool(options: BuildKimiToolOptions = {}) {
|
|
|
186
151
|
|
|
187
152
|
const timeout = buildTimeoutSignal(signal);
|
|
188
153
|
try {
|
|
154
|
+
const sourceName = params.data_source_name.trim();
|
|
155
|
+
|
|
156
|
+
// Lazy catalog sync: validate against the best-known catalog without
|
|
157
|
+
// blocking, and refresh in the background when it is stale. Unknown
|
|
158
|
+
// names get one forced refresh before we reject them, so newly added
|
|
159
|
+
// server-side sources work even when the local cache is old.
|
|
160
|
+
let catalog = peekDatasourceCatalog();
|
|
161
|
+
if (!catalog.sources.includes(sourceName)) {
|
|
162
|
+
catalog = await ensureDatasourceCatalog({
|
|
163
|
+
fetchImpl: deps.fetch,
|
|
164
|
+
forceRefresh: true,
|
|
165
|
+
});
|
|
166
|
+
if (!catalog.sources.includes(sourceName)) {
|
|
167
|
+
return errorResult<string>(
|
|
168
|
+
`Unknown data source "${sourceName}". Available sources (kimi-datasource v${catalog.version}): ${catalog.sources.join(", ")}`,
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
} else {
|
|
172
|
+
refreshDatasourceCatalogInBackground(deps.fetch);
|
|
173
|
+
}
|
|
174
|
+
|
|
189
175
|
const apiName = params.api_name?.trim();
|
|
190
176
|
const body = apiName
|
|
191
177
|
? {
|
|
192
178
|
method: "call_data_source_tool",
|
|
193
179
|
params: {
|
|
194
|
-
data_source_name:
|
|
180
|
+
data_source_name: sourceName,
|
|
195
181
|
api_name: apiName,
|
|
196
182
|
params: params.params ?? {},
|
|
197
183
|
},
|
|
198
184
|
}
|
|
199
185
|
: {
|
|
200
186
|
method: "get_data_source_desc",
|
|
201
|
-
params: { name:
|
|
187
|
+
params: { name: sourceName },
|
|
202
188
|
};
|
|
203
189
|
|
|
204
190
|
const response = await fetchWithAuthRetry(deps, accessToken, (token) =>
|
package/src/tools/moonshot.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { defineTool, type AgentToolResult } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import {
|
|
2
|
+
import { obj, str, type Static } from "@earendil-works/pi-schema";
|
|
3
3
|
|
|
4
4
|
import {
|
|
5
5
|
type BuildKimiToolOptions,
|
|
@@ -15,12 +15,12 @@ import {
|
|
|
15
15
|
textComponent,
|
|
16
16
|
} from "./common.ts";
|
|
17
17
|
|
|
18
|
-
export const moonshotSearchSchema =
|
|
19
|
-
query:
|
|
18
|
+
export const moonshotSearchSchema = obj({
|
|
19
|
+
query: str({ description: "The query text to search for." }),
|
|
20
20
|
});
|
|
21
21
|
|
|
22
|
-
export const moonshotFetchSchema =
|
|
23
|
-
url:
|
|
22
|
+
export const moonshotFetchSchema = obj({
|
|
23
|
+
url: str({ description: "The URL to fetch content from." }),
|
|
24
24
|
});
|
|
25
25
|
|
|
26
26
|
export type MoonshotSearchParams = Static<typeof moonshotSearchSchema>;
|
|
@@ -119,7 +119,7 @@ export function buildMoonshotSearchTool(options: BuildKimiToolOptions = {}) {
|
|
|
119
119
|
"Search the web with Kimi Coding, then fetch the few relevant result URLs before answering",
|
|
120
120
|
parameters: moonshotSearchSchema,
|
|
121
121
|
|
|
122
|
-
async execute(toolCallId, params, signal) {
|
|
122
|
+
async execute(toolCallId, params: MoonshotSearchParams, signal) {
|
|
123
123
|
const accessToken = deps.getAccessToken();
|
|
124
124
|
if (!accessToken) {
|
|
125
125
|
return errorResult<MoonshotSearchResult[]>(
|
|
@@ -176,7 +176,7 @@ export function buildMoonshotFetchTool(options: BuildKimiToolOptions = {}) {
|
|
|
176
176
|
promptSnippet: "Fetch a web page with Kimi Coding's Moonshot fetch service",
|
|
177
177
|
parameters: moonshotFetchSchema,
|
|
178
178
|
|
|
179
|
-
async execute(toolCallId, params, signal) {
|
|
179
|
+
async execute(toolCallId, params: MoonshotFetchParams, signal) {
|
|
180
180
|
const accessToken = deps.getAccessToken();
|
|
181
181
|
if (!accessToken) {
|
|
182
182
|
return errorResult<MoonshotFetchResult>(
|