tradeblocks-mcp 3.0.2 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -80
- package/dist/{chunk-LDKTV7GW.js → chunk-27S67XW3.js} +24 -52
- package/dist/chunk-27S67XW3.js.map +1 -0
- package/dist/{chunk-FGZH632F.js → chunk-2E63THNI.js} +13 -11
- package/dist/chunk-2E63THNI.js.map +1 -0
- package/dist/{chunk-QTTR7AAW.js → chunk-2I3S2ZLT.js} +325 -84
- package/dist/chunk-2I3S2ZLT.js.map +1 -0
- package/dist/{chunk-4BLCXNQ6.js → chunk-4HZLVUOZ.js} +179 -183
- package/dist/chunk-4HZLVUOZ.js.map +1 -0
- package/dist/{chunk-XXYOUIZY.js → chunk-NWNSKALN.js} +63 -80
- package/dist/chunk-NWNSKALN.js.map +1 -0
- package/dist/{chunk-PRAYH3RT.js → chunk-WNTYA7ER.js} +29 -12
- package/dist/chunk-WNTYA7ER.js.map +1 -0
- package/dist/{chunk-BKQ4PM4Y.js → chunk-YZA4RS76.js} +24 -18
- package/dist/chunk-YZA4RS76.js.map +1 -0
- package/dist/{chunk-W2PP3LEH.js → chunk-ZTJXC3LS.js} +17 -8
- package/dist/chunk-ZTJXC3LS.js.map +1 -0
- package/dist/daily-log-processor-MZW2XBZY.js +9 -0
- package/dist/{daily-logs-store-D3QC2XPV.js → daily-logs-store-3C3OTGPK.js} +2 -2
- package/dist/iv-solver-worker.js +9 -1
- package/dist/market-provider-6U33TQUT.js +16 -0
- package/dist/{proto-3HZTCWK4.js → proto-V23GIZG2.js} +2 -2
- package/dist/{sync-PO4IPCYV.js → sync-3UBC37VW.js} +7 -7
- package/dist/test-exports.js +876 -483
- package/dist/test-exports.js.map +1 -1
- package/dist/trade-processor-DKDCWGKD.js +9 -0
- package/dist/{trades-store-LOL3FQSF.js → trades-store-R3NL25T2.js} +2 -2
- package/package.json +2 -6
- package/server/{chunk-CSDVJPBB.js → chunk-536NSFED.js} +29 -12
- package/server/chunk-536NSFED.js.map +1 -0
- package/server/{chunk-WA5AAPCH.js → chunk-A3UKD64A.js} +17 -8
- package/server/chunk-A3UKD64A.js.map +1 -0
- package/server/{chunk-PNKG7RY7.js → chunk-OMUDNJBJ.js} +63 -80
- package/server/chunk-OMUDNJBJ.js.map +1 -0
- package/server/{chunk-72GKJE2U.js → chunk-RJDJHZ5Y.js} +934 -335
- package/server/chunk-RJDJHZ5Y.js.map +1 -0
- package/server/{chunk-NZO6PT64.js → chunk-RM4B2VKS.js} +167 -177
- package/server/chunk-RM4B2VKS.js.map +1 -0
- package/server/{chunk-5EBXHT6C.js → chunk-SY6GDNVI.js} +24 -18
- package/server/chunk-SY6GDNVI.js.map +1 -0
- package/server/{chunk-FBNDMCT5.js → chunk-T66KH2XH.js} +13 -11
- package/server/chunk-T66KH2XH.js.map +1 -0
- package/server/{chunk-NRFXAJF7.js → chunk-W5E7FHC4.js} +24 -52
- package/server/chunk-W5E7FHC4.js.map +1 -0
- package/server/{config-6IZXEFEX.js → config-DK7KOMNL.js} +1 -1
- package/server/config-DK7KOMNL.js.map +1 -0
- package/server/daily-log-processor-B2VPU2FQ.js +10 -0
- package/server/{daily-logs-store-NU7M4L5C.js → daily-logs-store-YFNXTB7J.js} +2 -2
- package/server/http-server.js +2 -5
- package/server/index.js +2497 -2029
- package/server/index.js.map +1 -1
- package/server/iv-solver-worker.js +9 -1
- package/server/market-provider-L56RFHAC.js +17 -0
- package/server/{proto-DWRZJO4E.js → proto-YZ4TLX2N.js} +2 -2
- package/server/{sync-QFI5L7S7.js → sync-CENKDN53.js} +7 -7
- package/server/trade-processor-3PI4LHZI.js +10 -0
- package/server/{trades-store-VY26MWER.js → trades-store-RTTEYMXH.js} +2 -2
- package/src/auth/clients-store.ts +4 -4
- package/src/auth/code-store.ts +1 -1
- package/src/auth/config.ts +11 -12
- package/src/auth/login-page.ts +8 -10
- package/src/auth/provider.ts +35 -30
- package/src/auth/token.ts +17 -15
- package/src/db/backtest-schemas.ts +38 -8
- package/src/db/connection.ts +107 -40
- package/src/db/data-root.ts +22 -0
- package/src/db/index.ts +37 -5
- package/src/db/json-adapters.ts +22 -31
- package/src/db/json-migration.ts +11 -15
- package/src/db/json-store.ts +5 -3
- package/src/db/market-datasets.ts +48 -15
- package/src/db/market-schemas.ts +11 -5
- package/src/db/market-views.ts +119 -32
- package/src/db/parquet-writer.ts +7 -7
- package/src/db/profile-schemas.ts +39 -22
- package/src/db/schemas.ts +2 -2
- package/src/http-server.ts +7 -12
- package/src/index.ts +8 -9
- package/src/market/ingestor/index.ts +1 -4
- package/src/market/ingestor/market-ingestor.ts +140 -67
- package/src/market/ingestor/types.ts +1 -3
- package/src/market/stores/chain-sql.ts +2 -8
- package/src/market/stores/chain-store.ts +1 -4
- package/src/market/stores/duckdb-chain-store.ts +3 -14
- package/src/market/stores/duckdb-enriched-store.ts +3 -13
- package/src/market/stores/duckdb-quote-store.ts +10 -27
- package/src/market/stores/duckdb-spot-store.ts +4 -20
- package/src/market/stores/enriched-sql.ts +3 -9
- package/src/market/stores/enriched-store.ts +2 -2
- package/src/market/stores/parquet-chain-store.ts +5 -22
- package/src/market/stores/parquet-enriched-store.ts +4 -16
- package/src/market/stores/parquet-oi-daily-store.ts +3 -14
- package/src/market/stores/parquet-quote-store.ts +9 -29
- package/src/market/stores/parquet-spot-store.ts +14 -33
- package/src/market/stores/quote-store.ts +2 -10
- package/src/market/stores/spot-sql.ts +3 -15
- package/src/market/stores/spot-store.ts +1 -1
- package/src/market/tickers/defaults.json +6 -6
- package/src/market/tickers/loader.ts +2 -7
- package/src/market/tickers/registry.ts +5 -15
- package/src/market/tickers/schemas.ts +2 -6
- package/src/models/strategy-profile.ts +39 -39
- package/src/sync/block-sync.ts +44 -65
- package/src/sync/index.ts +1 -4
- package/src/sync/metadata.ts +14 -20
- package/src/test-exports.ts +139 -123
- package/src/tools/analysis.ts +180 -159
- package/src/tools/batch-exit-analysis.ts +100 -59
- package/src/tools/blocks/analysis.ts +116 -101
- package/src/tools/blocks/comparison.ts +281 -364
- package/src/tools/blocks/core.ts +208 -253
- package/src/tools/blocks/health.ts +571 -563
- package/src/tools/blocks/index.ts +2 -0
- package/src/tools/blocks/paired-comparison.ts +425 -0
- package/src/tools/blocks/similarity.ts +316 -283
- package/src/tools/edge-decay.ts +124 -197
- package/src/tools/exit-analysis.ts +130 -77
- package/src/tools/greeks-attribution.ts +84 -35
- package/src/tools/guides.ts +4 -6
- package/src/tools/imports.ts +11 -14
- package/src/tools/market-data.ts +1067 -757
- package/src/tools/market-enrichment.ts +3 -3
- package/src/tools/market-fetch.ts +148 -67
- package/src/tools/market-imports.ts +12 -12
- package/src/tools/middleware/sync-middleware.ts +5 -6
- package/src/tools/performance.ts +185 -302
- package/src/tools/profile-analysis.ts +52 -66
- package/src/tools/profiles.ts +106 -69
- package/src/tools/regime-advisor.ts +20 -45
- package/src/tools/replay.ts +81 -77
- package/src/tools/reports/discrepancies.ts +298 -328
- package/src/tools/reports/fields.ts +7 -25
- package/src/tools/reports/helpers.ts +18 -49
- package/src/tools/reports/predictive.ts +27 -70
- package/src/tools/reports/slippage-trends.ts +315 -345
- package/src/tools/reports/slippage.ts +1 -4
- package/src/tools/reports/strategy-matches.ts +399 -441
- package/src/tools/schema.ts +43 -40
- package/src/tools/shared/filters.ts +3 -9
- package/src/tools/snapshot.ts +9 -30
- package/src/tools/sql.ts +15 -14
- package/src/tools/tickers.ts +1 -4
- package/src/utils/batch-exit-analysis.ts +31 -42
- package/src/utils/black-scholes.ts +39 -29
- package/src/utils/block-loader.ts +69 -83
- package/src/utils/calibration-probe.ts +3 -4
- package/src/utils/chain-loader.ts +3 -3
- package/src/utils/csv-discovery.ts +16 -22
- package/src/utils/data-quality.ts +24 -36
- package/src/utils/exit-triggers.ts +91 -96
- package/src/utils/field-timing.ts +94 -79
- package/src/utils/filter-predicates.ts +13 -9
- package/src/utils/flatfile-importer.ts +94 -64
- package/src/utils/greeks-decomposition.ts +152 -100
- package/src/utils/iv-solver-pool.ts +55 -25
- package/src/utils/iv-solver-worker.ts +5 -5
- package/src/utils/market-enricher.ts +528 -497
- package/src/utils/market-importer.ts +31 -12
- package/src/utils/market-provider.ts +21 -23
- package/src/utils/massive-tier.ts +5 -7
- package/src/utils/migrate-option-data-helpers.ts +2 -8
- package/src/utils/option-quote-greeks.ts +25 -31
- package/src/utils/option-time.ts +4 -8
- package/src/utils/output-formatter.ts +1 -4
- package/src/utils/provider-capabilities.ts +1 -4
- package/src/utils/providers/massive.ts +59 -93
- package/src/utils/providers/thetadata/backfill.ts +14 -23
- package/src/utils/providers/thetadata/client.ts +12 -8
- package/src/utils/providers/thetadata/decode.ts +2 -20
- package/src/utils/providers/thetadata/endpoints.ts +32 -40
- package/src/utils/providers/thetadata/join.ts +11 -10
- package/src/utils/providers/thetadata/proto.ts +12 -10
- package/src/utils/providers/thetadata/quote-mid-greeks.ts +5 -5
- package/src/utils/providers/thetadata.ts +11 -10
- package/src/utils/quote-enricher.ts +4 -4
- package/src/utils/quote-parquet-projection.ts +3 -11
- package/src/utils/sample-date-selector.ts +3 -5
- package/src/utils/schema-metadata.ts +102 -70
- package/src/utils/ticker.ts +5 -9
- package/src/utils/trade-replay.ts +77 -68
- package/dist/chunk-4BLCXNQ6.js.map +0 -1
- package/dist/chunk-BKQ4PM4Y.js.map +0 -1
- package/dist/chunk-FGZH632F.js.map +0 -1
- package/dist/chunk-LDKTV7GW.js.map +0 -1
- package/dist/chunk-PRAYH3RT.js.map +0 -1
- package/dist/chunk-QTTR7AAW.js.map +0 -1
- package/dist/chunk-W2PP3LEH.js.map +0 -1
- package/dist/chunk-XXYOUIZY.js.map +0 -1
- package/dist/daily-log-processor-6MWJ23JK.js +0 -9
- package/dist/market-provider-VDRJUEF2.js +0 -16
- package/dist/trade-processor-NHU4VWRX.js +0 -9
- package/manifest.json +0 -141
- package/server/chunk-5EBXHT6C.js.map +0 -1
- package/server/chunk-72GKJE2U.js.map +0 -1
- package/server/chunk-CSDVJPBB.js.map +0 -1
- package/server/chunk-FBNDMCT5.js.map +0 -1
- package/server/chunk-NRFXAJF7.js.map +0 -1
- package/server/chunk-NZO6PT64.js.map +0 -1
- package/server/chunk-PNKG7RY7.js.map +0 -1
- package/server/chunk-WA5AAPCH.js.map +0 -1
- package/server/config-6IZXEFEX.js.map +0 -1
- package/server/daily-log-processor-Y3PVSVBM.js +0 -10
- package/server/market-provider-VOYYVYWT.js +0 -17
- package/server/trade-processor-JWVS37KM.js +0 -10
- /package/dist/{daily-log-processor-6MWJ23JK.js.map → daily-log-processor-MZW2XBZY.js.map} +0 -0
- /package/dist/{daily-logs-store-D3QC2XPV.js.map → daily-logs-store-3C3OTGPK.js.map} +0 -0
- /package/dist/{market-provider-VDRJUEF2.js.map → market-provider-6U33TQUT.js.map} +0 -0
- /package/dist/{proto-3HZTCWK4.js.map → proto-V23GIZG2.js.map} +0 -0
- /package/dist/{sync-PO4IPCYV.js.map → sync-3UBC37VW.js.map} +0 -0
- /package/dist/{trade-processor-NHU4VWRX.js.map → trade-processor-DKDCWGKD.js.map} +0 -0
- /package/dist/{trades-store-LOL3FQSF.js.map → trades-store-R3NL25T2.js.map} +0 -0
- /package/server/{daily-log-processor-Y3PVSVBM.js.map → daily-log-processor-B2VPU2FQ.js.map} +0 -0
- /package/server/{daily-logs-store-NU7M4L5C.js.map → daily-logs-store-YFNXTB7J.js.map} +0 -0
- /package/server/{market-provider-VOYYVYWT.js.map → market-provider-L56RFHAC.js.map} +0 -0
- /package/server/{proto-DWRZJO4E.js.map → proto-YZ4TLX2N.js.map} +0 -0
- /package/server/{sync-QFI5L7S7.js.map → sync-CENKDN53.js.map} +0 -0
- /package/server/{trade-processor-JWVS37KM.js.map → trade-processor-3PI4LHZI.js.map} +0 -0
- /package/server/{trades-store-VY26MWER.js.map → trades-store-RTTEYMXH.js.map} +0 -0
package/src/index.ts
CHANGED
|
@@ -209,9 +209,7 @@ export function registerTradeBlocksCoreTools(
|
|
|
209
209
|
}
|
|
210
210
|
|
|
211
211
|
// Main entry point - handles both skill CLI commands and MCP server mode
|
|
212
|
-
export async function startTradeBlocksMcp(
|
|
213
|
-
options: StartTradeBlocksMcpOptions = {}
|
|
214
|
-
): Promise<void> {
|
|
212
|
+
export async function startTradeBlocksMcp(options: StartTradeBlocksMcpOptions = {}): Promise<void> {
|
|
215
213
|
const plugins = options.plugins ?? [];
|
|
216
214
|
const command = process.argv[2];
|
|
217
215
|
|
|
@@ -304,7 +302,9 @@ export async function startTradeBlocksMcp(
|
|
|
304
302
|
// the module-level handle — a captured reference would go stale the moment a
|
|
305
303
|
// tool upgrades for a write, surfacing as "connection disconnected".
|
|
306
304
|
const storeContext: StoreContext = {
|
|
307
|
-
get conn() {
|
|
305
|
+
get conn() {
|
|
306
|
+
return getCurrentConnection();
|
|
307
|
+
},
|
|
308
308
|
dataDir: resolvedDir,
|
|
309
309
|
parquetMode,
|
|
310
310
|
tickers: tickerRegistry,
|
|
@@ -321,9 +321,7 @@ export async function startTradeBlocksMcp(
|
|
|
321
321
|
// stdio transport reserves stdout for the MCP JSON protocol — anything else
|
|
322
322
|
// on stdout (even diagnostics) makes the client throw "not valid JSON".
|
|
323
323
|
// Diagnostic output must go to stderr.
|
|
324
|
-
console.error(
|
|
325
|
-
`[market-stores] Constructed: ${parquetMode ? "parquet" : "duckdb"} backend`,
|
|
326
|
-
);
|
|
324
|
+
console.error(`[market-stores] Constructed: ${parquetMode ? "parquet" : "duckdb"} backend`);
|
|
327
325
|
|
|
328
326
|
// Factory function to create configured MCP server instances
|
|
329
327
|
// Used by HTTP transport which needs fresh instances per request (stateless mode)
|
|
@@ -332,8 +330,9 @@ export async function startTradeBlocksMcp(
|
|
|
332
330
|
{ name: "tradeblocks-mcp", version: "2.0.0" },
|
|
333
331
|
{
|
|
334
332
|
capabilities: { tools: {} },
|
|
335
|
-
instructions:
|
|
336
|
-
|
|
333
|
+
instructions:
|
|
334
|
+
"Call list_blocks first to discover available block IDs. All other block tools require a blockId returned by list_blocks. For SQL queries, call describe_database first to discover block_ids and column names, then filter trades with WHERE block_id = '...'.",
|
|
335
|
+
},
|
|
337
336
|
);
|
|
338
337
|
registerTradeBlocksCoreTools(server, pluginContext);
|
|
339
338
|
for (const plugin of plugins) {
|
|
@@ -20,7 +20,10 @@ import type {
|
|
|
20
20
|
BulkProgressReporter,
|
|
21
21
|
} from "./types.ts";
|
|
22
22
|
import type { OiDailyRow, QuoteRow } from "../stores/types.ts";
|
|
23
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
applyQuoteGreeksParallel,
|
|
25
|
+
type QuoteGreeksStats,
|
|
26
|
+
} from "../../utils/option-quote-greeks.ts";
|
|
24
27
|
|
|
25
28
|
export interface MarketIngestorDeps {
|
|
26
29
|
stores: MarketStores;
|
|
@@ -201,7 +204,13 @@ export class MarketIngestor {
|
|
|
201
204
|
assetClass,
|
|
202
205
|
});
|
|
203
206
|
} catch (error) {
|
|
204
|
-
const mapped = this.mapProviderFailure(
|
|
207
|
+
const mapped = this.mapProviderFailure(
|
|
208
|
+
provider,
|
|
209
|
+
"bars",
|
|
210
|
+
normalizedTicker,
|
|
211
|
+
error,
|
|
212
|
+
assetClass,
|
|
213
|
+
);
|
|
205
214
|
if (mapped) return mapped;
|
|
206
215
|
throw error;
|
|
207
216
|
}
|
|
@@ -217,9 +226,10 @@ export class MarketIngestor {
|
|
|
217
226
|
}
|
|
218
227
|
}
|
|
219
228
|
|
|
220
|
-
const enrichment =
|
|
221
|
-
|
|
222
|
-
|
|
229
|
+
const enrichment =
|
|
230
|
+
opts.skipEnrichment || !minDate
|
|
231
|
+
? null
|
|
232
|
+
: await this.triggerPerTickerEnrichment(opts.tickers, minDate, maxDate!);
|
|
223
233
|
|
|
224
234
|
return {
|
|
225
235
|
status: "ok",
|
|
@@ -238,14 +248,23 @@ export class MarketIngestor {
|
|
|
238
248
|
return getProvider();
|
|
239
249
|
}
|
|
240
250
|
|
|
241
|
-
private timespanToProviderArgs(timespan: string): {
|
|
251
|
+
private timespanToProviderArgs(timespan: string): {
|
|
252
|
+
timespan: "day" | "minute" | "hour";
|
|
253
|
+
multiplier: number;
|
|
254
|
+
} {
|
|
242
255
|
switch (timespan) {
|
|
243
|
-
case "1d":
|
|
244
|
-
|
|
245
|
-
case "
|
|
246
|
-
|
|
247
|
-
case "
|
|
248
|
-
|
|
256
|
+
case "1d":
|
|
257
|
+
return { timespan: "day", multiplier: 1 };
|
|
258
|
+
case "1m":
|
|
259
|
+
return { timespan: "minute", multiplier: 1 };
|
|
260
|
+
case "5m":
|
|
261
|
+
return { timespan: "minute", multiplier: 5 };
|
|
262
|
+
case "15m":
|
|
263
|
+
return { timespan: "minute", multiplier: 15 };
|
|
264
|
+
case "1h":
|
|
265
|
+
return { timespan: "hour", multiplier: 1 };
|
|
266
|
+
default:
|
|
267
|
+
throw new Error(`Unknown timespan: ${timespan}`);
|
|
249
268
|
}
|
|
250
269
|
}
|
|
251
270
|
|
|
@@ -290,15 +309,17 @@ export class MarketIngestor {
|
|
|
290
309
|
assetClass === "index" &&
|
|
291
310
|
/(TimeoutError|aborted due to timeout|timed out)/i.test(message)
|
|
292
311
|
) {
|
|
293
|
-
const reason =
|
|
294
|
-
|
|
295
|
-
|
|
312
|
+
const reason =
|
|
313
|
+
operation === "chain"
|
|
314
|
+
? "current Massive provider path does not reliably support index option-chain refreshes"
|
|
315
|
+
: "current Massive provider path does not reliably support index data for this request";
|
|
296
316
|
return unsupportedProviderResult(provider, operation, target, reason, message);
|
|
297
317
|
}
|
|
298
318
|
if (provider.name === "massive" && /HTTP 403 Forbidden/.test(message)) {
|
|
299
|
-
const reason =
|
|
300
|
-
|
|
301
|
-
|
|
319
|
+
const reason =
|
|
320
|
+
assetClass === "index"
|
|
321
|
+
? "current Massive account/tier does not permit index data for this request"
|
|
322
|
+
: "current Massive account/tier does not permit this request";
|
|
302
323
|
return unsupportedProviderResult(provider, operation, target, reason, message);
|
|
303
324
|
}
|
|
304
325
|
return null;
|
|
@@ -367,9 +388,10 @@ export class MarketIngestor {
|
|
|
367
388
|
}
|
|
368
389
|
|
|
369
390
|
try {
|
|
370
|
-
const coverage =
|
|
371
|
-
|
|
372
|
-
|
|
391
|
+
const coverage =
|
|
392
|
+
dataset === "spot"
|
|
393
|
+
? await this.deps.stores.spot.getCoverage(symbol.toUpperCase(), asOf, asOf)
|
|
394
|
+
: await this.deps.stores.chain.getCoverage(symbol.toUpperCase(), asOf, asOf);
|
|
373
395
|
if (coverage.totalDates <= 0) return result;
|
|
374
396
|
|
|
375
397
|
return {
|
|
@@ -533,11 +555,7 @@ export class MarketIngestor {
|
|
|
533
555
|
if (mapped) return mapped;
|
|
534
556
|
throw error;
|
|
535
557
|
}
|
|
536
|
-
const written = await this.writeQuotesForTicker(
|
|
537
|
-
provider,
|
|
538
|
-
ticker,
|
|
539
|
-
quotes,
|
|
540
|
-
);
|
|
558
|
+
const written = await this.writeQuotesForTicker(provider, ticker, quotes);
|
|
541
559
|
totalRows += written.rowsWritten;
|
|
542
560
|
if (written.minDate && (!minDate || written.minDate < minDate)) minDate = written.minDate;
|
|
543
561
|
if (written.maxDate && (!maxDate || written.maxDate > maxDate)) maxDate = written.maxDate;
|
|
@@ -583,12 +601,7 @@ export class MarketIngestor {
|
|
|
583
601
|
for (const underlying of underlyings) {
|
|
584
602
|
const upperUnderlying = underlying.toUpperCase();
|
|
585
603
|
for (const date of dates) {
|
|
586
|
-
const drain = await this.drainBulkQuotes(
|
|
587
|
-
provider,
|
|
588
|
-
upperUnderlying,
|
|
589
|
-
date,
|
|
590
|
-
onProgress,
|
|
591
|
-
);
|
|
604
|
+
const drain = await this.drainBulkQuotes(provider, upperUnderlying, date, onProgress);
|
|
592
605
|
if (drain.rowsWritten > 0) {
|
|
593
606
|
totalRows += drain.rowsWritten;
|
|
594
607
|
if (!minDate || date < minDate) minDate = date;
|
|
@@ -688,7 +701,11 @@ export class MarketIngestor {
|
|
|
688
701
|
}
|
|
689
702
|
: undefined;
|
|
690
703
|
|
|
691
|
-
const stream = provider.fetchBulkQuotes!({
|
|
704
|
+
const stream = provider.fetchBulkQuotes!({
|
|
705
|
+
underlying: upperUnderlying,
|
|
706
|
+
date,
|
|
707
|
+
onGroupComplete,
|
|
708
|
+
});
|
|
692
709
|
for await (const chunk of stream) {
|
|
693
710
|
for (const row of chunk) {
|
|
694
711
|
const root = extractRoot(row.ticker);
|
|
@@ -752,15 +769,12 @@ export class MarketIngestor {
|
|
|
752
769
|
const message = error instanceof Error ? error.message : String(error);
|
|
753
770
|
// Warn is still emitted for live tail-following — the load-bearing
|
|
754
771
|
// signal is `result.skipped[]` / `status: "partial"`.
|
|
755
|
-
console.warn(
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
error: message,
|
|
762
|
-
},
|
|
763
|
-
);
|
|
772
|
+
console.warn("[drainBulkQuotes] enrichQuoteRows failed; skipping batch", {
|
|
773
|
+
underlying: resolvedUnderlying,
|
|
774
|
+
date,
|
|
775
|
+
rows: rows.length,
|
|
776
|
+
error: message,
|
|
777
|
+
});
|
|
764
778
|
skipped.push({
|
|
765
779
|
underlying: resolvedUnderlying,
|
|
766
780
|
date,
|
|
@@ -809,7 +823,12 @@ export class MarketIngestor {
|
|
|
809
823
|
provider: MarketDataProvider,
|
|
810
824
|
ticker: string,
|
|
811
825
|
quotes: Map<string, MinuteQuote>,
|
|
812
|
-
): Promise<{
|
|
826
|
+
): Promise<{
|
|
827
|
+
rowsWritten: number;
|
|
828
|
+
minDate?: string;
|
|
829
|
+
maxDate?: string;
|
|
830
|
+
skipped: IngestSkippedBatch[];
|
|
831
|
+
}> {
|
|
813
832
|
const root = extractRoot(ticker);
|
|
814
833
|
const underlying = this.deps.stores.quote.tickers.resolve(root);
|
|
815
834
|
|
|
@@ -856,16 +875,13 @@ export class MarketIngestor {
|
|
|
856
875
|
const message = error instanceof Error ? error.message : String(error);
|
|
857
876
|
// Warn is still emitted for live tail-following — the load-bearing
|
|
858
877
|
// signal is `result.skipped[]` / `status: "partial"`.
|
|
859
|
-
console.warn(
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
error: message,
|
|
867
|
-
},
|
|
868
|
-
);
|
|
878
|
+
console.warn("[writeQuotesForTicker] enrichQuoteRows failed; skipping batch", {
|
|
879
|
+
underlying,
|
|
880
|
+
date,
|
|
881
|
+
ticker,
|
|
882
|
+
rows: rows.length,
|
|
883
|
+
error: message,
|
|
884
|
+
});
|
|
869
885
|
skipped.push({
|
|
870
886
|
underlying,
|
|
871
887
|
date,
|
|
@@ -937,7 +953,12 @@ export class MarketIngestor {
|
|
|
937
953
|
for (const underlying of opts.underlyings) {
|
|
938
954
|
const upperUnderlying = underlying.toUpperCase();
|
|
939
955
|
const assetClass = this.detectAssetClass(upperUnderlying);
|
|
940
|
-
const unsupported = this.preflightProviderSupport(
|
|
956
|
+
const unsupported = this.preflightProviderSupport(
|
|
957
|
+
provider,
|
|
958
|
+
"chain",
|
|
959
|
+
upperUnderlying,
|
|
960
|
+
assetClass,
|
|
961
|
+
);
|
|
941
962
|
if (unsupported) return unsupported;
|
|
942
963
|
// Enumerate trading dates in [from, to] and fetch the chain as-of each date.
|
|
943
964
|
const dates = this.enumerateDates(opts.from, opts.to);
|
|
@@ -950,7 +971,13 @@ export class MarketIngestor {
|
|
|
950
971
|
expired: true,
|
|
951
972
|
});
|
|
952
973
|
} catch (error) {
|
|
953
|
-
const mapped = this.mapProviderFailure(
|
|
974
|
+
const mapped = this.mapProviderFailure(
|
|
975
|
+
provider,
|
|
976
|
+
"chain",
|
|
977
|
+
upperUnderlying,
|
|
978
|
+
error,
|
|
979
|
+
assetClass,
|
|
980
|
+
);
|
|
954
981
|
if (mapped) return mapped;
|
|
955
982
|
throw error;
|
|
956
983
|
}
|
|
@@ -1119,43 +1146,75 @@ export class MarketIngestor {
|
|
|
1119
1146
|
case "spot_bars": {
|
|
1120
1147
|
const ticker = opts.partition.ticker;
|
|
1121
1148
|
if (!ticker) {
|
|
1122
|
-
return {
|
|
1149
|
+
return {
|
|
1150
|
+
status: "error",
|
|
1151
|
+
rowsWritten: 0,
|
|
1152
|
+
error: "partition.ticker is required for datasetType='spot_bars'",
|
|
1153
|
+
};
|
|
1123
1154
|
}
|
|
1124
1155
|
const { rowCount } = await this.deps.stores.spot.writeFromSelect(
|
|
1125
1156
|
{ ticker: ticker.toUpperCase(), date: partitionDate },
|
|
1126
1157
|
opts.selectSql,
|
|
1127
1158
|
);
|
|
1128
|
-
return {
|
|
1159
|
+
return {
|
|
1160
|
+
status: "ok",
|
|
1161
|
+
rowsWritten: rowCount,
|
|
1162
|
+
dateRange: { from: partitionDate, to: partitionDate },
|
|
1163
|
+
};
|
|
1129
1164
|
}
|
|
1130
1165
|
case "option_quotes": {
|
|
1131
1166
|
const underlying = opts.partition.underlying;
|
|
1132
1167
|
if (!underlying) {
|
|
1133
|
-
return {
|
|
1168
|
+
return {
|
|
1169
|
+
status: "error",
|
|
1170
|
+
rowsWritten: 0,
|
|
1171
|
+
error: "partition.underlying is required for datasetType='option_quotes'",
|
|
1172
|
+
};
|
|
1134
1173
|
}
|
|
1135
1174
|
const { rowCount } = await this.deps.stores.quote.writeFromSelect(
|
|
1136
1175
|
{ underlying: underlying.toUpperCase(), date: partitionDate },
|
|
1137
1176
|
opts.selectSql,
|
|
1138
1177
|
);
|
|
1139
|
-
return {
|
|
1178
|
+
return {
|
|
1179
|
+
status: "ok",
|
|
1180
|
+
rowsWritten: rowCount,
|
|
1181
|
+
dateRange: { from: partitionDate, to: partitionDate },
|
|
1182
|
+
};
|
|
1140
1183
|
}
|
|
1141
1184
|
case "option_chain": {
|
|
1142
1185
|
const underlying = opts.partition.underlying;
|
|
1143
1186
|
if (!underlying) {
|
|
1144
|
-
return {
|
|
1187
|
+
return {
|
|
1188
|
+
status: "error",
|
|
1189
|
+
rowsWritten: 0,
|
|
1190
|
+
error: "partition.underlying is required for datasetType='option_chain'",
|
|
1191
|
+
};
|
|
1145
1192
|
}
|
|
1146
1193
|
const { rowCount } = await this.deps.stores.chain.writeFromSelect(
|
|
1147
1194
|
{ underlying: underlying.toUpperCase(), date: partitionDate },
|
|
1148
1195
|
opts.selectSql,
|
|
1149
1196
|
);
|
|
1150
|
-
return {
|
|
1197
|
+
return {
|
|
1198
|
+
status: "ok",
|
|
1199
|
+
rowsWritten: rowCount,
|
|
1200
|
+
dateRange: { from: partitionDate, to: partitionDate },
|
|
1201
|
+
};
|
|
1151
1202
|
}
|
|
1152
1203
|
default: {
|
|
1153
1204
|
const _exhaustive: never = opts.datasetType;
|
|
1154
|
-
return {
|
|
1205
|
+
return {
|
|
1206
|
+
status: "error",
|
|
1207
|
+
rowsWritten: 0,
|
|
1208
|
+
error: `Unknown datasetType: ${String(_exhaustive)}`,
|
|
1209
|
+
};
|
|
1155
1210
|
}
|
|
1156
1211
|
}
|
|
1157
1212
|
} catch (err) {
|
|
1158
|
-
return {
|
|
1213
|
+
return {
|
|
1214
|
+
status: "error",
|
|
1215
|
+
rowsWritten: 0,
|
|
1216
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1217
|
+
};
|
|
1159
1218
|
}
|
|
1160
1219
|
}
|
|
1161
1220
|
|
|
@@ -1271,13 +1330,21 @@ export class MarketIngestor {
|
|
|
1271
1330
|
}
|
|
1272
1331
|
|
|
1273
1332
|
// Coverage report — shallow summary per ticker
|
|
1274
|
-
const coverage: Record<
|
|
1333
|
+
const coverage: Record<
|
|
1334
|
+
string,
|
|
1335
|
+
{ totalDates: number; dateRange?: { from: string; to: string } }
|
|
1336
|
+
> = {};
|
|
1275
1337
|
for (const ticker of opts.spotTickers) {
|
|
1276
1338
|
try {
|
|
1277
|
-
const cov = await this.deps.stores.spot.getCoverage(
|
|
1339
|
+
const cov = await this.deps.stores.spot.getCoverage(
|
|
1340
|
+
ticker.toUpperCase(),
|
|
1341
|
+
opts.asOf,
|
|
1342
|
+
opts.asOf,
|
|
1343
|
+
);
|
|
1278
1344
|
coverage[ticker] = {
|
|
1279
1345
|
totalDates: cov.totalDates,
|
|
1280
|
-
dateRange:
|
|
1346
|
+
dateRange:
|
|
1347
|
+
cov.earliest && cov.latest ? { from: cov.earliest, to: cov.latest } : undefined,
|
|
1281
1348
|
};
|
|
1282
1349
|
} catch {
|
|
1283
1350
|
coverage[ticker] = { totalDates: 0 };
|
|
@@ -1304,7 +1371,13 @@ export class MarketIngestor {
|
|
|
1304
1371
|
|
|
1305
1372
|
return {
|
|
1306
1373
|
status,
|
|
1307
|
-
perOperation: {
|
|
1374
|
+
perOperation: {
|
|
1375
|
+
spot: spotResults,
|
|
1376
|
+
chain: chainResults,
|
|
1377
|
+
quotes: quoteResults,
|
|
1378
|
+
openInterest: openInterestResults,
|
|
1379
|
+
vixContext,
|
|
1380
|
+
},
|
|
1308
1381
|
coverage,
|
|
1309
1382
|
errors,
|
|
1310
1383
|
...(aggregateSkipped.length > 0 ? { skipped: aggregateSkipped } : {}),
|
|
@@ -147,9 +147,7 @@ export type BulkProgressEvent =
|
|
|
147
147
|
rowsWritten: number;
|
|
148
148
|
};
|
|
149
149
|
|
|
150
|
-
export type BulkProgressReporter = (
|
|
151
|
-
event: BulkProgressEvent,
|
|
152
|
-
) => void | Promise<void>;
|
|
150
|
+
export type BulkProgressReporter = (event: BulkProgressEvent) => void | Promise<void>;
|
|
153
151
|
|
|
154
152
|
export interface IngestChainOptions {
|
|
155
153
|
underlyings: string[];
|
|
@@ -21,10 +21,7 @@ function lit(value: string): string {
|
|
|
21
21
|
* date partition. Results are ordered by `ticker` so consumer iteration is
|
|
22
22
|
* deterministic across backends.
|
|
23
23
|
*/
|
|
24
|
-
export function buildReadChainSQL(
|
|
25
|
-
underlying: string,
|
|
26
|
-
date: string,
|
|
27
|
-
): BuiltSQL {
|
|
24
|
+
export function buildReadChainSQL(underlying: string, date: string): BuiltSQL {
|
|
28
25
|
return {
|
|
29
26
|
sql: `SELECT underlying, date, ticker, contract_type, strike, expiration, dte, exercise_style
|
|
30
27
|
FROM market.option_chain
|
|
@@ -43,10 +40,7 @@ export function buildReadChainSQL(
|
|
|
43
40
|
*
|
|
44
41
|
* Throws when `dates` is empty (prevents `IN ()` which DuckDB rejects).
|
|
45
42
|
*/
|
|
46
|
-
export function buildReadChainDatesSQL(
|
|
47
|
-
underlying: string,
|
|
48
|
-
dates: string[],
|
|
49
|
-
): BuiltSQL {
|
|
43
|
+
export function buildReadChainDatesSQL(underlying: string, dates: string[]): BuiltSQL {
|
|
50
44
|
if (dates.length === 0) {
|
|
51
45
|
throw new Error("buildReadChainDatesSQL: dates must not be empty");
|
|
52
46
|
}
|
|
@@ -58,10 +58,7 @@ export abstract class ChainStore {
|
|
|
58
58
|
* with identical columns. Use this instead of N per-date `readChain` calls —
|
|
59
59
|
* per-call glob-expansion / planning overhead dominates for view reads.
|
|
60
60
|
*/
|
|
61
|
-
async readChainDates(
|
|
62
|
-
underlying: string,
|
|
63
|
-
dates: string[],
|
|
64
|
-
): Promise<ContractRow[]> {
|
|
61
|
+
async readChainDates(underlying: string, dates: string[]): Promise<ContractRow[]> {
|
|
65
62
|
if (dates.length === 0) return [];
|
|
66
63
|
// Builder inlines values; unbound runAndReadAll(sql) bypasses extract_statements.
|
|
67
64
|
const { sql } = buildReadChainDatesSQL(underlying, dates);
|
|
@@ -14,11 +14,7 @@ import type { ContractRow, CoverageReport } from "./types.ts";
|
|
|
14
14
|
import { buildReadChainSQL } from "./chain-sql.ts";
|
|
15
15
|
|
|
16
16
|
export class DuckdbChainStore extends ChainStore {
|
|
17
|
-
async writeChain(
|
|
18
|
-
underlying: string,
|
|
19
|
-
date: string,
|
|
20
|
-
rows: ContractRow[],
|
|
21
|
-
): Promise<void> {
|
|
17
|
+
async writeChain(underlying: string, date: string, rows: ContractRow[]): Promise<void> {
|
|
22
18
|
if (rows.length === 0) return;
|
|
23
19
|
const placeholders = rows
|
|
24
20
|
.map((_, i) => {
|
|
@@ -56,10 +52,7 @@ export class DuckdbChainStore extends ChainStore {
|
|
|
56
52
|
return { rowCount: Number(result.rowsChanged) };
|
|
57
53
|
}
|
|
58
54
|
|
|
59
|
-
async readChain(
|
|
60
|
-
underlying: string,
|
|
61
|
-
date: string,
|
|
62
|
-
): Promise<ContractRow[]> {
|
|
55
|
+
async readChain(underlying: string, date: string): Promise<ContractRow[]> {
|
|
63
56
|
// Builder inlines values; unbound runAndReadAll(sql) bypasses
|
|
64
57
|
// extract_statements (chain-sql.ts header).
|
|
65
58
|
const { sql } = buildReadChainSQL(underlying, date);
|
|
@@ -76,11 +69,7 @@ export class DuckdbChainStore extends ChainStore {
|
|
|
76
69
|
}));
|
|
77
70
|
}
|
|
78
71
|
|
|
79
|
-
async getCoverage(
|
|
80
|
-
underlying: string,
|
|
81
|
-
from: string,
|
|
82
|
-
to: string,
|
|
83
|
-
): Promise<CoverageReport> {
|
|
72
|
+
async getCoverage(underlying: string, from: string, to: string): Promise<CoverageReport> {
|
|
84
73
|
// Inline literals — same leak rationale as readChain.
|
|
85
74
|
const underlyingLit = underlying.replace(/'/g, "''");
|
|
86
75
|
const fromLit = from.replace(/'/g, "''");
|
|
@@ -15,17 +15,11 @@ import { SpotStore } from "./spot-store.ts";
|
|
|
15
15
|
import type { StoreContext, CoverageReport } from "./types.ts";
|
|
16
16
|
import { buildReadEnrichedSQL } from "./enriched-sql.ts";
|
|
17
17
|
import { runEnrichment } from "../../utils/market-enricher.ts";
|
|
18
|
-
import {
|
|
19
|
-
getEnrichedThrough,
|
|
20
|
-
upsertEnrichedThrough,
|
|
21
|
-
} from "../../db/json-adapters.ts";
|
|
18
|
+
import { getEnrichedThrough, upsertEnrichedThrough } from "../../db/json-adapters.ts";
|
|
22
19
|
|
|
23
20
|
export class DuckdbEnrichedStore extends EnrichedStore {
|
|
24
21
|
private readonly spotStore: SpotStore;
|
|
25
|
-
constructor(
|
|
26
|
-
ctx: StoreContext,
|
|
27
|
-
spotStore: SpotStore,
|
|
28
|
-
) {
|
|
22
|
+
constructor(ctx: StoreContext, spotStore: SpotStore) {
|
|
29
23
|
super(ctx);
|
|
30
24
|
this.spotStore = spotStore;
|
|
31
25
|
}
|
|
@@ -74,11 +68,7 @@ export class DuckdbEnrichedStore extends EnrichedStore {
|
|
|
74
68
|
});
|
|
75
69
|
const reader = await this.ctx.conn.runAndReadAll(sql);
|
|
76
70
|
const names = reader.columnNames();
|
|
77
|
-
return reader
|
|
78
|
-
.getRows()
|
|
79
|
-
.map((row) =>
|
|
80
|
-
Object.fromEntries(names.map((n, i) => [n, row[i]])),
|
|
81
|
-
);
|
|
71
|
+
return reader.getRows().map((row) => Object.fromEntries(names.map((n, i) => [n, row[i]])));
|
|
82
72
|
}
|
|
83
73
|
|
|
84
74
|
async getCoverage(ticker: string): Promise<CoverageReport> {
|
|
@@ -11,12 +11,7 @@
|
|
|
11
11
|
* D-02 reminder: no method body inspects `ctx.parquetMode`.
|
|
12
12
|
*/
|
|
13
13
|
import { QuoteStore } from "./quote-store.ts";
|
|
14
|
-
import type {
|
|
15
|
-
QuoteRow,
|
|
16
|
-
CoverageReport,
|
|
17
|
-
ReadWindowParams,
|
|
18
|
-
WindowQuoteRow,
|
|
19
|
-
} from "./types.ts";
|
|
14
|
+
import type { QuoteRow, CoverageReport, ReadWindowParams, WindowQuoteRow } from "./types.ts";
|
|
20
15
|
import { extractRoot } from "../tickers/resolver.ts";
|
|
21
16
|
import {
|
|
22
17
|
describeQueryColumns,
|
|
@@ -43,7 +38,7 @@ function parseQuoteRow(row: unknown[]): QuoteRow {
|
|
|
43
38
|
theta: row[11] == null ? null : Number(row[11]),
|
|
44
39
|
vega: row[12] == null ? null : Number(row[12]),
|
|
45
40
|
iv: row[13] == null ? null : Number(row[13]),
|
|
46
|
-
greeks_source: row[14] == null ? null : String(row[14]) as QuoteRow["greeks_source"],
|
|
41
|
+
greeks_source: row[14] == null ? null : (String(row[14]) as QuoteRow["greeks_source"]),
|
|
47
42
|
greeks_revision: row[15] == null ? null : Number(row[15]),
|
|
48
43
|
rate_type: row[16] == null ? null : String(row[16]),
|
|
49
44
|
rate_value: row[17] == null ? null : Number(row[17]),
|
|
@@ -85,11 +80,7 @@ export class DuckdbQuoteStore extends QuoteStore {
|
|
|
85
80
|
this.quoteTableColumns = null;
|
|
86
81
|
}
|
|
87
82
|
|
|
88
|
-
async writeQuotes(
|
|
89
|
-
underlying: string,
|
|
90
|
-
date: string,
|
|
91
|
-
quotes: QuoteRow[],
|
|
92
|
-
): Promise<void> {
|
|
83
|
+
async writeQuotes(underlying: string, date: string, quotes: QuoteRow[]): Promise<void> {
|
|
93
84
|
if (quotes.length === 0) return;
|
|
94
85
|
await this.ensureWritableQuoteSchema();
|
|
95
86
|
const placeholders = quotes
|
|
@@ -109,8 +100,8 @@ export class DuckdbQuoteStore extends QuoteStore {
|
|
|
109
100
|
q.bid,
|
|
110
101
|
q.ask,
|
|
111
102
|
mid,
|
|
112
|
-
null,
|
|
113
|
-
q.source ?? null,
|
|
103
|
+
null, // last_updated_ns — not tracked in QuoteRow
|
|
104
|
+
q.source ?? null, // source — populated when provider tags rows (Task 6)
|
|
114
105
|
q.delta ?? null,
|
|
115
106
|
q.gamma ?? null,
|
|
116
107
|
q.theta ?? null,
|
|
@@ -157,9 +148,7 @@ export class DuckdbQuoteStore extends QuoteStore {
|
|
|
157
148
|
to: string,
|
|
158
149
|
): Promise<Map<string, QuoteRow[]>> {
|
|
159
150
|
if (occTickers.length === 0) return new Map();
|
|
160
|
-
const firstUnderlying = this.ctx.tickers.resolve(
|
|
161
|
-
extractRoot(occTickers[0]),
|
|
162
|
-
);
|
|
151
|
+
const firstUnderlying = this.ctx.tickers.resolve(extractRoot(occTickers[0]));
|
|
163
152
|
for (const t of occTickers) {
|
|
164
153
|
const u = this.ctx.tickers.resolve(extractRoot(t));
|
|
165
154
|
if (u !== firstUnderlying) {
|
|
@@ -180,9 +169,7 @@ export class DuckdbQuoteStore extends QuoteStore {
|
|
|
180
169
|
const underlyingLit = `'${escapeSqlLiteral(firstUnderlying)}'`;
|
|
181
170
|
const fromLit = `'${escapeSqlLiteral(from)}'`;
|
|
182
171
|
const toLit = `'${escapeSqlLiteral(to)}'`;
|
|
183
|
-
const tickerList = occTickers
|
|
184
|
-
.map((t) => `'${escapeSqlLiteral(t)}'`)
|
|
185
|
-
.join(", ");
|
|
172
|
+
const tickerList = occTickers.map((t) => `'${escapeSqlLiteral(t)}'`).join(", ");
|
|
186
173
|
const reader = await this.ctx.conn.runAndReadAll(
|
|
187
174
|
`SELECT ${projection}
|
|
188
175
|
FROM market.option_quote_minutes AS q
|
|
@@ -256,7 +243,7 @@ export class DuckdbQuoteStore extends QuoteStore {
|
|
|
256
243
|
if (perf) {
|
|
257
244
|
console.log(
|
|
258
245
|
` [P] readQuotesBulk underlying=${underlying} dates=${perDate.size} ` +
|
|
259
|
-
|
|
246
|
+
`tickers=${occUnion.size} rows=${rows.length} queryMs=${Date.now() - queryStart}`,
|
|
260
247
|
);
|
|
261
248
|
}
|
|
262
249
|
for (const row of rows) {
|
|
@@ -355,15 +342,11 @@ export class DuckdbQuoteStore extends QuoteStore {
|
|
|
355
342
|
theta: r[10] == null ? null : Number(r[10]),
|
|
356
343
|
vega: r[11] == null ? null : Number(r[11]),
|
|
357
344
|
iv: r[12] == null ? null : Number(r[12]),
|
|
358
|
-
greeks_source: r[13] == null ? null : String(r[13]) as WindowQuoteRow["greeks_source"],
|
|
345
|
+
greeks_source: r[13] == null ? null : (String(r[13]) as WindowQuoteRow["greeks_source"]),
|
|
359
346
|
}));
|
|
360
347
|
}
|
|
361
348
|
|
|
362
|
-
async getCoverage(
|
|
363
|
-
underlying: string,
|
|
364
|
-
from: string,
|
|
365
|
-
to: string,
|
|
366
|
-
): Promise<CoverageReport> {
|
|
349
|
+
async getCoverage(underlying: string, from: string, to: string): Promise<CoverageReport> {
|
|
367
350
|
// Inline literal path — same leak rationale (see spot-sql.ts header).
|
|
368
351
|
const underlyingLit = underlying.replace(/'/g, "''");
|
|
369
352
|
const fromLit = from.replace(/'/g, "''");
|