tradeblocks-mcp 3.0.3 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -12
- package/dist/{chunk-C6LL746C.js → chunk-2I3S2ZLT.js} +259 -1
- package/dist/chunk-2I3S2ZLT.js.map +1 -0
- package/dist/{chunk-UBUC5A66.js → chunk-4HZLVUOZ.js} +81 -76
- package/dist/chunk-4HZLVUOZ.js.map +1 -0
- package/dist/{chunk-VDU25Z6X.js → chunk-NWNSKALN.js} +2 -2
- package/dist/{chunk-AP7IHUUR.js → chunk-WNTYA7ER.js} +2 -2
- package/dist/{chunk-6S37CXUA.js → chunk-YZA4RS76.js} +14 -6
- package/dist/chunk-YZA4RS76.js.map +1 -0
- package/dist/{chunk-SEUZYQGQ.js → chunk-ZTJXC3LS.js} +2 -2
- package/dist/daily-log-processor-MZW2XBZY.js +9 -0
- package/dist/market-provider-6U33TQUT.js +16 -0
- package/dist/{sync-EVLKB3ZJ.js → sync-3UBC37VW.js} +6 -6
- package/dist/test-exports.js +560 -271
- package/dist/test-exports.js.map +1 -1
- package/dist/trade-processor-DKDCWGKD.js +9 -0
- package/package.json +2 -6
- package/server/{chunk-YUCOAJ4Z.js → chunk-536NSFED.js} +2 -2
- package/server/{chunk-GH2552SE.js → chunk-A3UKD64A.js} +2 -2
- package/server/{chunk-ZBJCF4ZG.js → chunk-OMUDNJBJ.js} +2 -2
- package/server/{chunk-BOPHW5M6.js → chunk-RJDJHZ5Y.js} +414 -101
- package/server/chunk-RJDJHZ5Y.js.map +1 -0
- package/server/{chunk-4P7D7YZP.js → chunk-RM4B2VKS.js} +72 -71
- package/server/chunk-RM4B2VKS.js.map +1 -0
- package/server/{chunk-OBYKFW2B.js → chunk-SY6GDNVI.js} +14 -6
- package/server/chunk-SY6GDNVI.js.map +1 -0
- package/server/daily-log-processor-B2VPU2FQ.js +10 -0
- package/server/index.js +787 -552
- package/server/index.js.map +1 -1
- package/server/market-provider-L56RFHAC.js +17 -0
- package/server/{sync-V25UQJA3.js → sync-CENKDN53.js} +6 -6
- package/server/trade-processor-3PI4LHZI.js +10 -0
- package/src/db/backtest-schemas.ts +26 -4
- package/src/db/data-root.ts +22 -0
- package/src/test-exports.ts +12 -0
- package/src/tools/blocks/index.ts +2 -0
- package/src/tools/blocks/paired-comparison.ts +425 -0
- package/src/tools/imports.ts +2 -2
- package/src/utils/block-loader.ts +1 -1
- package/src/utils/calibration-probe.ts +2 -2
- package/dist/chunk-6S37CXUA.js.map +0 -1
- package/dist/chunk-C6LL746C.js.map +0 -1
- package/dist/chunk-UBUC5A66.js.map +0 -1
- package/dist/daily-log-processor-BY3ISY6K.js +0 -9
- package/dist/market-provider-H3ARIVZ4.js +0 -16
- package/dist/trade-processor-EYA3I3XB.js +0 -9
- package/manifest.json +0 -133
- package/server/chunk-4P7D7YZP.js.map +0 -1
- package/server/chunk-BOPHW5M6.js.map +0 -1
- package/server/chunk-OBYKFW2B.js.map +0 -1
- package/server/daily-log-processor-ENEUT22A.js +0 -10
- package/server/market-provider-B437HKLW.js +0 -17
- package/server/trade-processor-L3PIQ5TG.js +0 -10
- /package/dist/{chunk-VDU25Z6X.js.map → chunk-NWNSKALN.js.map} +0 -0
- /package/dist/{chunk-AP7IHUUR.js.map → chunk-WNTYA7ER.js.map} +0 -0
- /package/dist/{chunk-SEUZYQGQ.js.map → chunk-ZTJXC3LS.js.map} +0 -0
- /package/dist/{daily-log-processor-BY3ISY6K.js.map → daily-log-processor-MZW2XBZY.js.map} +0 -0
- /package/dist/{market-provider-H3ARIVZ4.js.map → market-provider-6U33TQUT.js.map} +0 -0
- /package/dist/{sync-EVLKB3ZJ.js.map → sync-3UBC37VW.js.map} +0 -0
- /package/dist/{trade-processor-EYA3I3XB.js.map → trade-processor-DKDCWGKD.js.map} +0 -0
- /package/server/{chunk-YUCOAJ4Z.js.map → chunk-536NSFED.js.map} +0 -0
- /package/server/{chunk-GH2552SE.js.map → chunk-A3UKD64A.js.map} +0 -0
- /package/server/{chunk-ZBJCF4ZG.js.map → chunk-OMUDNJBJ.js.map} +0 -0
- /package/server/{daily-log-processor-ENEUT22A.js.map → daily-log-processor-B2VPU2FQ.js.map} +0 -0
- /package/server/{market-provider-B437HKLW.js.map → market-provider-L56RFHAC.js.map} +0 -0
- /package/server/{sync-V25UQJA3.js.map → sync-CENKDN53.js.map} +0 -0
- /package/server/{trade-processor-L3PIQ5TG.js.map → trade-processor-3PI4LHZI.js.map} +0 -0
|
@@ -2,19 +2,19 @@ import {
|
|
|
2
2
|
REPORTING_TRADE_COLUMN_ALIASES,
|
|
3
3
|
convertTatRowToReportingTrade,
|
|
4
4
|
isTatFormat
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-2I3S2ZLT.js";
|
|
6
6
|
|
|
7
7
|
// src/sync/index.ts
|
|
8
8
|
import { existsSync as existsSync2 } from "fs";
|
|
9
9
|
import * as fs9 from "fs/promises";
|
|
10
|
-
import * as
|
|
10
|
+
import * as path12 from "path";
|
|
11
11
|
|
|
12
12
|
// src/db/connection.ts
|
|
13
13
|
import { DuckDBInstance } from "@duckdb/node-api";
|
|
14
14
|
import { execFile } from "child_process";
|
|
15
15
|
import * as fs4 from "fs/promises";
|
|
16
16
|
import * as os from "os";
|
|
17
|
-
import * as
|
|
17
|
+
import * as path8 from "path";
|
|
18
18
|
import { promisify } from "util";
|
|
19
19
|
|
|
20
20
|
// src/db/schemas.ts
|
|
@@ -210,13 +210,17 @@ async function ensureMarketDataTables(conn) {
|
|
|
210
210
|
|
|
211
211
|
// src/db/parquet-writer.ts
|
|
212
212
|
import * as fs from "fs/promises";
|
|
213
|
-
import * as
|
|
213
|
+
import * as path3 from "path";
|
|
214
214
|
|
|
215
215
|
// src/db/market-datasets.ts
|
|
216
|
-
import * as
|
|
216
|
+
import * as path2 from "path";
|
|
217
217
|
|
|
218
218
|
// src/db/data-root.ts
|
|
219
|
+
import path from "path";
|
|
219
220
|
var KEY = "__tradeblocks_data_root__";
|
|
221
|
+
function resolveDbPath(dataRoot, kind) {
|
|
222
|
+
return path.join(dataRoot, "database", `${kind}.duckdb`);
|
|
223
|
+
}
|
|
220
224
|
function setDataRoot(dir) {
|
|
221
225
|
globalThis[KEY] = dir;
|
|
222
226
|
}
|
|
@@ -238,19 +242,19 @@ var PARTITIONED_DATASETS = {
|
|
|
238
242
|
option_quote_minutes: "option_quote_minutes"
|
|
239
243
|
};
|
|
240
244
|
function resolveMarketDir(dataDir) {
|
|
241
|
-
return
|
|
245
|
+
return path2.join(getDataRoot(dataDir), "market");
|
|
242
246
|
}
|
|
243
247
|
function resolveCanonicalMarketFile(dataDir, dataset) {
|
|
244
|
-
return
|
|
248
|
+
return path2.join(resolveMarketDir(dataDir), SINGLE_FILE_DATASETS[dataset]);
|
|
245
249
|
}
|
|
246
250
|
function resolveCanonicalMarketPartitionDir(dataDir, dataset) {
|
|
247
|
-
return
|
|
251
|
+
return path2.join(resolveMarketDir(dataDir), PARTITIONED_DATASETS[dataset]);
|
|
248
252
|
}
|
|
249
253
|
function resolveCanonicalMarketPartitionPath(dataDir, dataset, date) {
|
|
250
|
-
return
|
|
254
|
+
return path2.join(resolveCanonicalMarketPartitionDir(dataDir, dataset), `date=${date}`);
|
|
251
255
|
}
|
|
252
256
|
function resolveCanonicalMarketPartitionFile(dataDir, dataset, date) {
|
|
253
|
-
return
|
|
257
|
+
return path2.join(resolveCanonicalMarketPartitionPath(dataDir, dataset, date), "data.parquet");
|
|
254
258
|
}
|
|
255
259
|
function canonicalMarketTableName(dataset) {
|
|
256
260
|
return `market.${dataset}`;
|
|
@@ -278,7 +282,7 @@ var DATASETS_V3 = {
|
|
|
278
282
|
async function writeSpotPartition(conn, args) {
|
|
279
283
|
const def = DATASETS_V3.spot;
|
|
280
284
|
return writeParquetPartition(conn, {
|
|
281
|
-
baseDir:
|
|
285
|
+
baseDir: path2.join(resolveMarketDir(args.dataDir), def.subdir),
|
|
282
286
|
partitions: { ticker: args.ticker, date: args.date },
|
|
283
287
|
// order matches def.partitionKeys
|
|
284
288
|
selectQuery: args.selectQuery,
|
|
@@ -289,7 +293,7 @@ async function writeSpotPartition(conn, args) {
|
|
|
289
293
|
async function writeChainPartition(conn, args) {
|
|
290
294
|
const def = DATASETS_V3.option_chain;
|
|
291
295
|
return writeParquetPartition(conn, {
|
|
292
|
-
baseDir:
|
|
296
|
+
baseDir: path2.join(resolveMarketDir(args.dataDir), def.subdir),
|
|
293
297
|
partitions: { underlying: args.underlying, date: args.date },
|
|
294
298
|
selectQuery: args.selectQuery,
|
|
295
299
|
compression: args.compression,
|
|
@@ -300,7 +304,7 @@ async function writeQuoteMinutesPartition(conn, args) {
|
|
|
300
304
|
const def = DATASETS_V3.option_quote_minutes;
|
|
301
305
|
const sortedSelect = `SELECT * FROM (${args.selectQuery}) AS q ORDER BY q.ticker, q.time`;
|
|
302
306
|
return writeParquetPartition(conn, {
|
|
303
|
-
baseDir:
|
|
307
|
+
baseDir: path2.join(resolveMarketDir(args.dataDir), def.subdir),
|
|
304
308
|
partitions: { underlying: args.underlying, date: args.date },
|
|
305
309
|
selectQuery: sortedSelect,
|
|
306
310
|
compression: args.compression,
|
|
@@ -311,7 +315,7 @@ async function writeOiDailyPartition(conn, args) {
|
|
|
311
315
|
const def = DATASETS_V3.option_oi_daily;
|
|
312
316
|
const sortedSelect = `SELECT * FROM (${args.selectQuery}) AS q ORDER BY q.ticker`;
|
|
313
317
|
return writeParquetPartition(conn, {
|
|
314
|
-
baseDir:
|
|
318
|
+
baseDir: path2.join(resolveMarketDir(args.dataDir), def.subdir),
|
|
315
319
|
partitions: { underlying: args.underlying, date: args.date },
|
|
316
320
|
selectQuery: sortedSelect,
|
|
317
321
|
compression: args.compression,
|
|
@@ -321,7 +325,7 @@ async function writeOiDailyPartition(conn, args) {
|
|
|
321
325
|
async function writeEnrichedTickerFile(conn, args) {
|
|
322
326
|
const def = DATASETS_V3.enriched;
|
|
323
327
|
return writeParquetPartition(conn, {
|
|
324
|
-
baseDir:
|
|
328
|
+
baseDir: path2.join(resolveMarketDir(args.dataDir), def.subdir),
|
|
325
329
|
partitions: { ticker: args.ticker },
|
|
326
330
|
selectQuery: args.selectQuery,
|
|
327
331
|
compression: args.compression,
|
|
@@ -330,7 +334,7 @@ async function writeEnrichedTickerFile(conn, args) {
|
|
|
330
334
|
}
|
|
331
335
|
async function writeEnrichedContext(conn, args) {
|
|
332
336
|
const def = DATASETS_V3.enriched_context;
|
|
333
|
-
const targetPath =
|
|
337
|
+
const targetPath = path2.join(resolveMarketDir(args.dataDir), def.subdir, def.filename);
|
|
334
338
|
return writeParquetAtomic(conn, {
|
|
335
339
|
targetPath,
|
|
336
340
|
selectQuery: args.selectQuery,
|
|
@@ -352,7 +356,7 @@ async function writeParquetAtomic(conn, opts) {
|
|
|
352
356
|
const tempPath = `${targetPath}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
353
357
|
try {
|
|
354
358
|
await conn.run(`CREATE TEMP TABLE "${stagingName}" AS ${selectQuery}`);
|
|
355
|
-
await fs.mkdir(
|
|
359
|
+
await fs.mkdir(path3.dirname(targetPath), { recursive: true });
|
|
356
360
|
await conn.run(
|
|
357
361
|
`COPY "${stagingName}" TO '${tempPath}' (FORMAT PARQUET, COMPRESSION ${compression})`
|
|
358
362
|
);
|
|
@@ -392,7 +396,7 @@ async function writeParquetPartition(conn, opts) {
|
|
|
392
396
|
}
|
|
393
397
|
}
|
|
394
398
|
const partitionSegments = Object.entries(partitions).map(([k, v]) => `${k}=${v}`);
|
|
395
|
-
const targetPath =
|
|
399
|
+
const targetPath = path3.join(opts.baseDir, ...partitionSegments, filename);
|
|
396
400
|
return writeParquetAtomic(conn, {
|
|
397
401
|
targetPath,
|
|
398
402
|
selectQuery: opts.selectQuery,
|
|
@@ -402,18 +406,18 @@ async function writeParquetPartition(conn, opts) {
|
|
|
402
406
|
|
|
403
407
|
// src/db/json-adapters.ts
|
|
404
408
|
import * as fs3 from "fs/promises";
|
|
405
|
-
import * as
|
|
409
|
+
import * as path5 from "path";
|
|
406
410
|
import { z } from "zod";
|
|
407
411
|
|
|
408
412
|
// src/db/json-store.ts
|
|
409
413
|
import * as fs2 from "fs/promises";
|
|
410
|
-
import * as
|
|
414
|
+
import * as path4 from "path";
|
|
411
415
|
function isEnoent(err) {
|
|
412
416
|
return typeof err === "object" && err !== null && err.code === "ENOENT";
|
|
413
417
|
}
|
|
414
418
|
async function writeJsonFile(filePath, data) {
|
|
415
419
|
const tmpPath = filePath + ".tmp";
|
|
416
|
-
await fs2.mkdir(
|
|
420
|
+
await fs2.mkdir(path4.dirname(filePath), { recursive: true });
|
|
417
421
|
await fs2.writeFile(tmpPath, JSON.stringify(data, null, 2) + "\n", "utf-8");
|
|
418
422
|
await fs2.rename(tmpPath, filePath);
|
|
419
423
|
}
|
|
@@ -442,7 +446,7 @@ async function deleteJsonFile(filePath) {
|
|
|
442
446
|
async function listJsonFiles(dirPath, suffix = ".json") {
|
|
443
447
|
try {
|
|
444
448
|
const entries = await fs2.readdir(dirPath, { withFileTypes: true });
|
|
445
|
-
return entries.filter((e) => e.isFile() && e.name.endsWith(suffix)).map((e) =>
|
|
449
|
+
return entries.filter((e) => e.isFile() && e.name.endsWith(suffix)).map((e) => path4.join(dirPath, e.name)).sort();
|
|
446
450
|
} catch (err) {
|
|
447
451
|
if (isEnoent(err)) {
|
|
448
452
|
return [];
|
|
@@ -470,7 +474,7 @@ function jsonToProfile(json) {
|
|
|
470
474
|
};
|
|
471
475
|
}
|
|
472
476
|
function profilePath(blockId, strategyName, blocksDir) {
|
|
473
|
-
return
|
|
477
|
+
return path5.join(blocksDir, blockId, "profiles", toFileSlug(strategyName) + ".json");
|
|
474
478
|
}
|
|
475
479
|
async function upsertProfileJson(profile, blocksDir) {
|
|
476
480
|
const filePath = profilePath(profile.blockId, profile.strategyName, blocksDir);
|
|
@@ -492,7 +496,7 @@ async function getProfileJson(blockId, strategyName, blocksDir) {
|
|
|
492
496
|
}
|
|
493
497
|
async function listProfilesJson(blocksDir, blockId) {
|
|
494
498
|
if (blockId) {
|
|
495
|
-
const profileDir =
|
|
499
|
+
const profileDir = path5.join(blocksDir, blockId, "profiles");
|
|
496
500
|
const files = await listJsonFiles(profileDir);
|
|
497
501
|
const profiles2 = [];
|
|
498
502
|
for (const file of files) {
|
|
@@ -532,7 +536,7 @@ function jsonToSyncMeta(json) {
|
|
|
532
536
|
};
|
|
533
537
|
}
|
|
534
538
|
function syncMetaPath(blockId, blocksDir) {
|
|
535
|
-
return
|
|
539
|
+
return path5.join(blocksDir, blockId, ".sync-meta.json");
|
|
536
540
|
}
|
|
537
541
|
async function getSyncMetadataJson(blockId, blocksDir) {
|
|
538
542
|
const filePath = syncMetaPath(blockId, blocksDir);
|
|
@@ -570,7 +574,7 @@ function marketMetaKey(source, ticker, targetTable) {
|
|
|
570
574
|
return `${source}|${ticker}|${targetTable}`;
|
|
571
575
|
}
|
|
572
576
|
function marketMetaFilePath(dataDir) {
|
|
573
|
-
return
|
|
577
|
+
return path5.join(getDataRoot(dataDir), "market-meta", "sync-metadata.json");
|
|
574
578
|
}
|
|
575
579
|
async function getMarketImportMetadataJson(source, ticker, targetTable, dataDir) {
|
|
576
580
|
const filePath = marketMetaFilePath(dataDir);
|
|
@@ -598,7 +602,7 @@ function flatLogKey(entry) {
|
|
|
598
602
|
return `${entry.date}|${entry.asset_class}|${entry.underlying}`;
|
|
599
603
|
}
|
|
600
604
|
function flatLogFilePath(dataDir) {
|
|
601
|
-
return
|
|
605
|
+
return path5.join(getDataRoot(dataDir), "market-meta", "flat-import-log.json");
|
|
602
606
|
}
|
|
603
607
|
async function getFlatImportLogJson(assetClass, underlying, from, to, dataDir) {
|
|
604
608
|
const filePath = flatLogFilePath(dataDir);
|
|
@@ -636,7 +640,7 @@ var EnrichmentWatermarksSchema = z.object({
|
|
|
636
640
|
watermarks: z.record(z.string().regex(/^[A-Z0-9._-]+$/), TickerEntrySchema)
|
|
637
641
|
});
|
|
638
642
|
function watermarksFilePath(dataDir) {
|
|
639
|
-
return
|
|
643
|
+
return path5.join(getDataRoot(dataDir), "market-meta", "enrichment-watermarks.json");
|
|
640
644
|
}
|
|
641
645
|
async function loadEnrichmentWatermarks(dataDir) {
|
|
642
646
|
const raw = await readJsonFile(watermarksFilePath(dataDir));
|
|
@@ -943,7 +947,7 @@ async function deleteProfile(conn, blockId, strategyName, baseDir) {
|
|
|
943
947
|
|
|
944
948
|
// src/db/market-views.ts
|
|
945
949
|
import { existsSync, readdirSync } from "fs";
|
|
946
|
-
import * as
|
|
950
|
+
import * as path6 from "path";
|
|
947
951
|
|
|
948
952
|
// src/utils/quote-parquet-projection.ts
|
|
949
953
|
var describeCache = /* @__PURE__ */ new Map();
|
|
@@ -1052,12 +1056,12 @@ function hasParquetPartitions(dir, partitionKey = "date") {
|
|
|
1052
1056
|
const prefix = `${partitionKey}=`;
|
|
1053
1057
|
return readdirSync(dir).some((entry) => {
|
|
1054
1058
|
if (!entry.startsWith(prefix)) return false;
|
|
1055
|
-
const partDir =
|
|
1059
|
+
const partDir = path6.join(dir, entry);
|
|
1056
1060
|
try {
|
|
1057
1061
|
const entries = readdirSync(partDir);
|
|
1058
1062
|
for (const sub of entries) {
|
|
1059
1063
|
if (sub.endsWith(".parquet")) return true;
|
|
1060
|
-
const nested =
|
|
1064
|
+
const nested = path6.join(partDir, sub);
|
|
1061
1065
|
try {
|
|
1062
1066
|
if (readdirSync(nested).some((f) => f.endsWith(".parquet"))) return true;
|
|
1063
1067
|
} catch {
|
|
@@ -1077,14 +1081,14 @@ function hasEnrichedTickerFiles(dir) {
|
|
|
1077
1081
|
try {
|
|
1078
1082
|
return readdirSync(dir).some((entry) => {
|
|
1079
1083
|
if (!entry.startsWith("ticker=")) return false;
|
|
1080
|
-
return existsSync(
|
|
1084
|
+
return existsSync(path6.join(dir, entry, "data.parquet"));
|
|
1081
1085
|
});
|
|
1082
1086
|
} catch {
|
|
1083
1087
|
return false;
|
|
1084
1088
|
}
|
|
1085
1089
|
}
|
|
1086
1090
|
function hasEnrichedContextFile(dir) {
|
|
1087
|
-
return existsSync(
|
|
1091
|
+
return existsSync(path6.join(dir, "context", "data.parquet"));
|
|
1088
1092
|
}
|
|
1089
1093
|
async function createMarketParquetViews(conn, dataDir) {
|
|
1090
1094
|
const viewsCreated = [];
|
|
@@ -1165,7 +1169,7 @@ async function createMarketParquetViews(conn, dataDir) {
|
|
|
1165
1169
|
await conn.run("DROP TABLE IF EXISTS market.option_delta_index");
|
|
1166
1170
|
} catch {
|
|
1167
1171
|
}
|
|
1168
|
-
const spotDir =
|
|
1172
|
+
const spotDir = path6.join(resolveMarketDir(dataDir), "spot");
|
|
1169
1173
|
if (hasParquetPartitions(spotDir, "ticker")) {
|
|
1170
1174
|
try {
|
|
1171
1175
|
await conn.run("DROP VIEW IF EXISTS market.spot");
|
|
@@ -1187,7 +1191,7 @@ async function createMarketParquetViews(conn, dataDir) {
|
|
|
1187
1191
|
}
|
|
1188
1192
|
tablesKept.push("spot");
|
|
1189
1193
|
}
|
|
1190
|
-
const enrichedDir =
|
|
1194
|
+
const enrichedDir = path6.join(resolveMarketDir(dataDir), "enriched");
|
|
1191
1195
|
if (hasEnrichedTickerFiles(enrichedDir)) {
|
|
1192
1196
|
try {
|
|
1193
1197
|
await conn.run("DROP VIEW IF EXISTS market.enriched");
|
|
@@ -1220,7 +1224,7 @@ async function createMarketParquetViews(conn, dataDir) {
|
|
|
1220
1224
|
}
|
|
1221
1225
|
await conn.run(
|
|
1222
1226
|
`CREATE OR REPLACE VIEW market.enriched_context AS
|
|
1223
|
-
SELECT * FROM read_parquet('${
|
|
1227
|
+
SELECT * FROM read_parquet('${path6.join(enrichedDir, "context", "data.parquet")}')`
|
|
1224
1228
|
);
|
|
1225
1229
|
viewsCreated.push("enriched_context");
|
|
1226
1230
|
} else {
|
|
@@ -1285,7 +1289,7 @@ async function createMarketParquetViews(conn, dataDir) {
|
|
|
1285
1289
|
}
|
|
1286
1290
|
|
|
1287
1291
|
// src/db/json-migration.ts
|
|
1288
|
-
import * as
|
|
1292
|
+
import * as path7 from "path";
|
|
1289
1293
|
function toDate2(value) {
|
|
1290
1294
|
if (value instanceof Date) return value;
|
|
1291
1295
|
if (typeof value === "object" && value !== null && "micros" in value) {
|
|
@@ -1362,7 +1366,7 @@ async function migrateProfiles(conn, blocksDir) {
|
|
|
1362
1366
|
createdAt: createdAt.toISOString(),
|
|
1363
1367
|
updatedAt: updatedAt.toISOString()
|
|
1364
1368
|
};
|
|
1365
|
-
const filePath =
|
|
1369
|
+
const filePath = path7.join(blocksDir, blockId, "profiles", toFileSlug(strategyName) + ".json");
|
|
1366
1370
|
await writeJsonFile(filePath, profileJson);
|
|
1367
1371
|
}
|
|
1368
1372
|
return rows.length;
|
|
@@ -1395,7 +1399,7 @@ async function migrateSyncMetadata(conn, blocksDir) {
|
|
|
1395
1399
|
return rows.length;
|
|
1396
1400
|
}
|
|
1397
1401
|
async function migrateMarketImportMetadata(conn, dataDir) {
|
|
1398
|
-
const existingFile = await readJsonFile(
|
|
1402
|
+
const existingFile = await readJsonFile(path7.join(dataDir, "market-meta", "sync-metadata.json"));
|
|
1399
1403
|
if (existingFile !== null) return 0;
|
|
1400
1404
|
let rows;
|
|
1401
1405
|
try {
|
|
@@ -1422,7 +1426,7 @@ async function migrateMarketImportMetadata(conn, dataDir) {
|
|
|
1422
1426
|
}
|
|
1423
1427
|
async function migrateFlatImportLog(conn, dataDir) {
|
|
1424
1428
|
const existingFile = await readJsonFile(
|
|
1425
|
-
|
|
1429
|
+
path7.join(dataDir, "market-meta", "flat-import-log.json")
|
|
1426
1430
|
);
|
|
1427
1431
|
if (existingFile !== null) return 0;
|
|
1428
1432
|
let rows;
|
|
@@ -1567,7 +1571,7 @@ async function tryRecoverLockByTerminatingStaleProcess(errorMessage, dbPath, for
|
|
|
1567
1571
|
return false;
|
|
1568
1572
|
}
|
|
1569
1573
|
const normalizedDbPath = dbPath.replace(/\\/g, "/");
|
|
1570
|
-
const normalizedDbDir =
|
|
1574
|
+
const normalizedDbDir = path8.dirname(normalizedDbPath);
|
|
1571
1575
|
const isTradeblocksProcess = command.includes("tradeblocks-mcp") || command.includes("/mcp-server/server/index.js") || command.includes("packages/mcp-server/server/index.js") || command.includes("\\mcp-server\\server\\index.js") || command.includes("packages\\mcp-server\\server\\index.js");
|
|
1572
1576
|
const normalizedCommand = command.replace(/\\/g, "/");
|
|
1573
1577
|
const targetsSameDb = normalizedCommand.includes(normalizedDbPath) || normalizedCommand.includes(normalizedDbDir);
|
|
@@ -1601,16 +1605,16 @@ function resolveMarketDbPath(dataDir) {
|
|
|
1601
1605
|
const args = process.argv;
|
|
1602
1606
|
for (let i = 0; i < args.length; i++) {
|
|
1603
1607
|
if (args[i] === "--market-db" && args[i + 1]) {
|
|
1604
|
-
return
|
|
1608
|
+
return path8.resolve(args[i + 1]);
|
|
1605
1609
|
}
|
|
1606
1610
|
}
|
|
1607
1611
|
if (process.env.MARKET_DB_PATH) {
|
|
1608
|
-
return
|
|
1612
|
+
return path8.resolve(process.env.MARKET_DB_PATH);
|
|
1609
1613
|
}
|
|
1610
|
-
return
|
|
1614
|
+
return path8.join(dataDir, "market.duckdb");
|
|
1611
1615
|
}
|
|
1612
1616
|
async function attachMarketDb(conn, marketDbPath, mode) {
|
|
1613
|
-
await fs4.mkdir(
|
|
1617
|
+
await fs4.mkdir(path8.dirname(marketDbPath), { recursive: true });
|
|
1614
1618
|
const readOnlyClause = mode === "read_only" ? " (READ_ONLY)" : "";
|
|
1615
1619
|
const escapedPath = marketDbPath.replace(/'/g, "''");
|
|
1616
1620
|
try {
|
|
@@ -1657,12 +1661,12 @@ async function openReadWriteConnection(dbPath, threads, memoryLimit) {
|
|
|
1657
1661
|
await ensureReportingDataTable(connection);
|
|
1658
1662
|
await ensureMutableMarketTables(connection);
|
|
1659
1663
|
await ensureProfilesSchema(connection);
|
|
1660
|
-
const dataDir =
|
|
1664
|
+
const dataDir = path8.dirname(dbPath);
|
|
1661
1665
|
const dataRoot = getDataRoot(dataDir);
|
|
1662
1666
|
await createMarketParquetViews(connection, dataRoot);
|
|
1663
1667
|
await ensureMarketDataTables(connection);
|
|
1664
1668
|
try {
|
|
1665
|
-
const blocksDir = (await import("./sync-
|
|
1669
|
+
const blocksDir = (await import("./sync-3UBC37VW.js")).getBlocksDir(dataRoot);
|
|
1666
1670
|
await migrateMetadataToJson(connection, dataRoot, blocksDir);
|
|
1667
1671
|
} catch (err) {
|
|
1668
1672
|
console.warn(
|
|
@@ -1708,7 +1712,7 @@ async function getConnection(dataDir) {
|
|
|
1708
1712
|
if (connection) {
|
|
1709
1713
|
return connection;
|
|
1710
1714
|
}
|
|
1711
|
-
const dbPath =
|
|
1715
|
+
const dbPath = path8.join(dataDir, "analytics.duckdb");
|
|
1712
1716
|
const threads = process.env.DUCKDB_THREADS || defaultThreads();
|
|
1713
1717
|
const memoryLimit = process.env.DUCKDB_MEMORY_LIMIT || defaultMemoryLimit();
|
|
1714
1718
|
storedDbPath = dbPath;
|
|
@@ -1785,7 +1789,7 @@ async function closeConnection() {
|
|
|
1785
1789
|
async function upgradeToReadWrite(dataDir, options) {
|
|
1786
1790
|
if (connectionMode === "read_write" && connection) return connection;
|
|
1787
1791
|
await closeConnection();
|
|
1788
|
-
const dbPath = storedDbPath ||
|
|
1792
|
+
const dbPath = storedDbPath || path8.join(dataDir, "analytics.duckdb");
|
|
1789
1793
|
const threads = storedThreads || process.env.DUCKDB_THREADS || defaultThreads();
|
|
1790
1794
|
const memoryLimit = storedMemoryLimit || process.env.DUCKDB_MEMORY_LIMIT || defaultMemoryLimit();
|
|
1791
1795
|
if (!storedMarketDbPath) {
|
|
@@ -1885,7 +1889,7 @@ function openMarketReadOnlyConnection(baseDir) {
|
|
|
1885
1889
|
}
|
|
1886
1890
|
async function getReadOnlyConnection(dataDir) {
|
|
1887
1891
|
if (connection) return connection;
|
|
1888
|
-
const dbPath =
|
|
1892
|
+
const dbPath = path8.join(dataDir, "analytics.duckdb");
|
|
1889
1893
|
const threads = process.env.DUCKDB_THREADS || defaultThreads();
|
|
1890
1894
|
const memoryLimit = process.env.DUCKDB_MEMORY_LIMIT || defaultMemoryLimit();
|
|
1891
1895
|
storedDbPath = dbPath;
|
|
@@ -1912,7 +1916,7 @@ function getCurrentConnection() {
|
|
|
1912
1916
|
|
|
1913
1917
|
// src/sync/block-sync.ts
|
|
1914
1918
|
import * as fs8 from "fs/promises";
|
|
1915
|
-
import * as
|
|
1919
|
+
import * as path11 from "path";
|
|
1916
1920
|
|
|
1917
1921
|
// src/sync/hasher.ts
|
|
1918
1922
|
import * as crypto from "crypto";
|
|
@@ -2069,11 +2073,11 @@ function marketTickerDateKey(ticker, date) {
|
|
|
2069
2073
|
|
|
2070
2074
|
// src/utils/block-loader.ts
|
|
2071
2075
|
import * as fs7 from "fs/promises";
|
|
2072
|
-
import * as
|
|
2076
|
+
import * as path10 from "path";
|
|
2073
2077
|
|
|
2074
2078
|
// src/utils/csv-discovery.ts
|
|
2075
2079
|
import * as fs6 from "fs/promises";
|
|
2076
|
-
import * as
|
|
2080
|
+
import * as path9 from "path";
|
|
2077
2081
|
function parseCSVLine(line) {
|
|
2078
2082
|
const result = [];
|
|
2079
2083
|
let current = "";
|
|
@@ -2198,7 +2202,7 @@ async function discoverCsvFiles(folderPath) {
|
|
|
2198
2202
|
if (csvFile === "tradelog.csv" || csvFile === "dailylog.csv" || csvFile === "reportinglog.csv" || csvFile === mappings.reportinglog) {
|
|
2199
2203
|
continue;
|
|
2200
2204
|
}
|
|
2201
|
-
const csvPath =
|
|
2205
|
+
const csvPath = path9.join(folderPath, csvFile);
|
|
2202
2206
|
const detectedType = await detectCsvType(csvPath);
|
|
2203
2207
|
if (detectedType) {
|
|
2204
2208
|
if (detectedType === "tradelog" && !mappings.tradelog) {
|
|
@@ -2394,7 +2398,7 @@ function convertToDailyLogEntry(raw, blockId) {
|
|
|
2394
2398
|
}
|
|
2395
2399
|
}
|
|
2396
2400
|
async function loadTrades(blockPath, filename = "tradelog.csv", blockId) {
|
|
2397
|
-
const tradelogPath =
|
|
2401
|
+
const tradelogPath = path10.join(blockPath, filename);
|
|
2398
2402
|
const content = await fs7.readFile(tradelogPath, "utf-8");
|
|
2399
2403
|
const records = parseCSV(content);
|
|
2400
2404
|
const trades = [];
|
|
@@ -2412,7 +2416,7 @@ async function loadTrades(blockPath, filename = "tradelog.csv", blockId) {
|
|
|
2412
2416
|
return trades;
|
|
2413
2417
|
}
|
|
2414
2418
|
async function loadDailyLogs(blockPath, blockId, filename = "dailylog.csv") {
|
|
2415
|
-
const dailylogPath =
|
|
2419
|
+
const dailylogPath = path10.join(blockPath, filename);
|
|
2416
2420
|
try {
|
|
2417
2421
|
await fs7.access(dailylogPath);
|
|
2418
2422
|
const content = await fs7.readFile(dailylogPath, "utf-8");
|
|
@@ -2432,10 +2436,10 @@ async function loadDailyLogs(blockPath, blockId, filename = "dailylog.csv") {
|
|
|
2432
2436
|
}
|
|
2433
2437
|
async function loadBlock(baseDir, blockId) {
|
|
2434
2438
|
const blocksDir = resolveBlocksBaseDir(baseDir);
|
|
2435
|
-
const blockPath =
|
|
2439
|
+
const blockPath = path10.join(blocksDir, blockId);
|
|
2436
2440
|
const { mappings } = await discoverCsvFiles(blockPath);
|
|
2437
2441
|
const tradelogFilename = mappings.tradelog || "tradelog.csv";
|
|
2438
|
-
const tradelogPath =
|
|
2442
|
+
const tradelogPath = path10.join(blockPath, tradelogFilename);
|
|
2439
2443
|
try {
|
|
2440
2444
|
await fs7.access(tradelogPath);
|
|
2441
2445
|
} catch {
|
|
@@ -2605,7 +2609,7 @@ async function listBlocks(baseDir) {
|
|
|
2605
2609
|
}
|
|
2606
2610
|
blocks.push(info);
|
|
2607
2611
|
} else if (!stats) {
|
|
2608
|
-
const blockPath =
|
|
2612
|
+
const blockPath = path10.join(blocksDir, blockId);
|
|
2609
2613
|
try {
|
|
2610
2614
|
const { mappings } = await discoverCsvFiles(blockPath);
|
|
2611
2615
|
if (mappings.tradelog) {
|
|
@@ -2673,10 +2677,10 @@ function convertToReportingTrade(raw) {
|
|
|
2673
2677
|
}
|
|
2674
2678
|
async function loadReportingLog(baseDir, blockId) {
|
|
2675
2679
|
const blocksDir = resolveBlocksBaseDir(baseDir);
|
|
2676
|
-
const blockPath =
|
|
2680
|
+
const blockPath = path10.join(blocksDir, blockId);
|
|
2677
2681
|
const { mappings } = await discoverCsvFiles(blockPath);
|
|
2678
2682
|
const filename = mappings.reportinglog || "reportinglog.csv";
|
|
2679
|
-
const reportingLogPath =
|
|
2683
|
+
const reportingLogPath = path10.join(blockPath, filename);
|
|
2680
2684
|
try {
|
|
2681
2685
|
await fs7.access(reportingLogPath);
|
|
2682
2686
|
} catch {
|
|
@@ -2773,7 +2777,7 @@ async function importCsv(baseDir, options) {
|
|
|
2773
2777
|
if (!blockId) {
|
|
2774
2778
|
throw new Error("Could not derive a valid block ID from the filename or provided name");
|
|
2775
2779
|
}
|
|
2776
|
-
const blockPath =
|
|
2780
|
+
const blockPath = path10.join(blocksDir, blockId);
|
|
2777
2781
|
try {
|
|
2778
2782
|
await fs7.access(blockPath);
|
|
2779
2783
|
throw new Error(
|
|
@@ -2786,7 +2790,7 @@ async function importCsv(baseDir, options) {
|
|
|
2786
2790
|
}
|
|
2787
2791
|
await fs7.mkdir(blockPath, { recursive: true });
|
|
2788
2792
|
const targetFilename = csvType === "tradelog" ? "tradelog.csv" : csvType === "dailylog" ? "dailylog.csv" : "reportinglog.csv";
|
|
2789
|
-
const targetPath =
|
|
2793
|
+
const targetPath = path10.join(blockPath, targetFilename);
|
|
2790
2794
|
await fs7.copyFile(csvPath, targetPath);
|
|
2791
2795
|
let dateRange = {
|
|
2792
2796
|
start: null,
|
|
@@ -3032,7 +3036,7 @@ function versionedHash(hash) {
|
|
|
3032
3036
|
return `${hash}:${PARSE_VERSION}`;
|
|
3033
3037
|
}
|
|
3034
3038
|
async function syncBlockInternal(conn, blockId, blockPath) {
|
|
3035
|
-
const blocksDir =
|
|
3039
|
+
const blocksDir = path11.dirname(blockPath);
|
|
3036
3040
|
try {
|
|
3037
3041
|
const existingMetadata = await getSyncMetadata(conn, blockId, blocksDir);
|
|
3038
3042
|
const tradelogFilename = await findTradelogFile(blockPath);
|
|
@@ -3056,12 +3060,12 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3056
3060
|
error: "No tradelog CSV found in block"
|
|
3057
3061
|
};
|
|
3058
3062
|
}
|
|
3059
|
-
const tradelogPath =
|
|
3063
|
+
const tradelogPath = path11.join(blockPath, tradelogFilename);
|
|
3060
3064
|
const tradelogHash = versionedHash(await hashFileContent(tradelogPath));
|
|
3061
3065
|
if (existingMetadata && existingMetadata.tradelog_hash === tradelogHash) {
|
|
3062
3066
|
const optionalLogs = await findOptionalLogFiles(blockPath);
|
|
3063
3067
|
if (optionalLogs.reportinglog) {
|
|
3064
|
-
const reportinglogPath =
|
|
3068
|
+
const reportinglogPath = path11.join(blockPath, optionalLogs.reportinglog);
|
|
3065
3069
|
const reportinglogHash = versionedHash(await hashFileContent(reportinglogPath));
|
|
3066
3070
|
if (existingMetadata.reportinglog_hash !== reportinglogHash) {
|
|
3067
3071
|
} else {
|
|
@@ -3089,7 +3093,7 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3089
3093
|
if (optionalLogs.dailylog) {
|
|
3090
3094
|
try {
|
|
3091
3095
|
dailylogHash = versionedHash(
|
|
3092
|
-
await hashFileContent(
|
|
3096
|
+
await hashFileContent(path11.join(blockPath, optionalLogs.dailylog))
|
|
3093
3097
|
);
|
|
3094
3098
|
} catch {
|
|
3095
3099
|
}
|
|
@@ -3097,14 +3101,14 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3097
3101
|
if (optionalLogs.reportinglog) {
|
|
3098
3102
|
try {
|
|
3099
3103
|
reportinglogHash = versionedHash(
|
|
3100
|
-
await hashFileContent(
|
|
3104
|
+
await hashFileContent(path11.join(blockPath, optionalLogs.reportinglog))
|
|
3101
3105
|
);
|
|
3102
3106
|
} catch {
|
|
3103
3107
|
}
|
|
3104
3108
|
}
|
|
3105
3109
|
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
3106
3110
|
if (optionalLogs.reportinglog && reportinglogHash) {
|
|
3107
|
-
const reportingPath =
|
|
3111
|
+
const reportingPath = path11.join(blockPath, optionalLogs.reportinglog);
|
|
3108
3112
|
const reportingContent = await fs8.readFile(reportingPath, "utf-8");
|
|
3109
3113
|
const reportingRecords = parseCSV2(reportingContent);
|
|
3110
3114
|
const reportingTrades = [];
|
|
@@ -3182,7 +3186,7 @@ async function detectBlockChanges(conn, baseDir) {
|
|
|
3182
3186
|
continue;
|
|
3183
3187
|
const blockId = entry.name;
|
|
3184
3188
|
folderNames.add(blockId);
|
|
3185
|
-
const blockPath =
|
|
3189
|
+
const blockPath = path11.join(baseDir, blockId);
|
|
3186
3190
|
if (!syncedBlockIds.has(blockId)) {
|
|
3187
3191
|
const tradelog = await findTradelogFile(blockPath);
|
|
3188
3192
|
if (tradelog) {
|
|
@@ -3196,7 +3200,7 @@ async function detectBlockChanges(conn, baseDir) {
|
|
|
3196
3200
|
continue;
|
|
3197
3201
|
}
|
|
3198
3202
|
try {
|
|
3199
|
-
const tradelogPath =
|
|
3203
|
+
const tradelogPath = path11.join(blockPath, tradelogFilename);
|
|
3200
3204
|
const currentHash = versionedHash(await hashFileContent(tradelogPath));
|
|
3201
3205
|
const metadata = await getSyncMetadata(conn, blockId, baseDir);
|
|
3202
3206
|
if (!metadata || metadata.tradelog_hash !== currentHash) {
|
|
@@ -3204,7 +3208,7 @@ async function detectBlockChanges(conn, baseDir) {
|
|
|
3204
3208
|
} else {
|
|
3205
3209
|
const optionalLogs = await findOptionalLogFiles(blockPath);
|
|
3206
3210
|
if (optionalLogs.reportinglog) {
|
|
3207
|
-
const reportinglogPath =
|
|
3211
|
+
const reportinglogPath = path11.join(blockPath, optionalLogs.reportinglog);
|
|
3208
3212
|
const reportingHash = versionedHash(await hashFileContent(reportinglogPath));
|
|
3209
3213
|
if (metadata.reportinglog_hash !== reportingHash) {
|
|
3210
3214
|
toSync.push(blockId);
|
|
@@ -3246,7 +3250,7 @@ function setBlocksDir(dir) {
|
|
|
3246
3250
|
}
|
|
3247
3251
|
function getBlocksDir(baseDir) {
|
|
3248
3252
|
if (_blocksDir) return _blocksDir;
|
|
3249
|
-
const nestedBlocksDir =
|
|
3253
|
+
const nestedBlocksDir = path12.join(baseDir, "blocks");
|
|
3250
3254
|
return existsSync2(nestedBlocksDir) ? nestedBlocksDir : baseDir;
|
|
3251
3255
|
}
|
|
3252
3256
|
async function syncAllBlocks(baseDir) {
|
|
@@ -3265,7 +3269,7 @@ async function syncAllBlocks(baseDir) {
|
|
|
3265
3269
|
}
|
|
3266
3270
|
}
|
|
3267
3271
|
for (const blockId of toSync) {
|
|
3268
|
-
const blockPath =
|
|
3272
|
+
const blockPath = path12.join(blocksDir, blockId);
|
|
3269
3273
|
const result = await syncBlockInternal(conn, blockId, blockPath);
|
|
3270
3274
|
results.push(result);
|
|
3271
3275
|
if (result.status === "error" && result.error) {
|
|
@@ -3284,7 +3288,7 @@ async function syncAllBlocks(baseDir) {
|
|
|
3284
3288
|
async function syncBlock(blockId, baseDir) {
|
|
3285
3289
|
const conn = await getConnection(baseDir);
|
|
3286
3290
|
const blocksDir = getBlocksDir(baseDir);
|
|
3287
|
-
const blockPath =
|
|
3291
|
+
const blockPath = path12.join(blocksDir, blockId);
|
|
3288
3292
|
try {
|
|
3289
3293
|
await fs9.access(blockPath);
|
|
3290
3294
|
} catch {
|
|
@@ -3301,6 +3305,7 @@ async function syncBlock(blockId, baseDir) {
|
|
|
3301
3305
|
export {
|
|
3302
3306
|
ensureMutableMarketTables,
|
|
3303
3307
|
ensureMarketDataTables,
|
|
3308
|
+
resolveDbPath,
|
|
3304
3309
|
setDataRoot,
|
|
3305
3310
|
getDataRoot,
|
|
3306
3311
|
resetDataRoot,
|
|
@@ -3388,4 +3393,4 @@ export {
|
|
|
3388
3393
|
loadReportingLog,
|
|
3389
3394
|
importCsv
|
|
3390
3395
|
};
|
|
3391
|
-
//# sourceMappingURL=chunk-
|
|
3396
|
+
//# sourceMappingURL=chunk-4HZLVUOZ.js.map
|