tradeblocks-mcp 3.0.2 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -80
- package/dist/{chunk-LDKTV7GW.js → chunk-27S67XW3.js} +24 -52
- package/dist/chunk-27S67XW3.js.map +1 -0
- package/dist/{chunk-FGZH632F.js → chunk-2E63THNI.js} +13 -11
- package/dist/chunk-2E63THNI.js.map +1 -0
- package/dist/{chunk-QTTR7AAW.js → chunk-2I3S2ZLT.js} +325 -84
- package/dist/chunk-2I3S2ZLT.js.map +1 -0
- package/dist/{chunk-4BLCXNQ6.js → chunk-4HZLVUOZ.js} +179 -183
- package/dist/chunk-4HZLVUOZ.js.map +1 -0
- package/dist/{chunk-XXYOUIZY.js → chunk-NWNSKALN.js} +63 -80
- package/dist/chunk-NWNSKALN.js.map +1 -0
- package/dist/{chunk-PRAYH3RT.js → chunk-WNTYA7ER.js} +29 -12
- package/dist/chunk-WNTYA7ER.js.map +1 -0
- package/dist/{chunk-BKQ4PM4Y.js → chunk-YZA4RS76.js} +24 -18
- package/dist/chunk-YZA4RS76.js.map +1 -0
- package/dist/{chunk-W2PP3LEH.js → chunk-ZTJXC3LS.js} +17 -8
- package/dist/chunk-ZTJXC3LS.js.map +1 -0
- package/dist/daily-log-processor-MZW2XBZY.js +9 -0
- package/dist/{daily-logs-store-D3QC2XPV.js → daily-logs-store-3C3OTGPK.js} +2 -2
- package/dist/iv-solver-worker.js +9 -1
- package/dist/market-provider-6U33TQUT.js +16 -0
- package/dist/{proto-3HZTCWK4.js → proto-V23GIZG2.js} +2 -2
- package/dist/{sync-PO4IPCYV.js → sync-3UBC37VW.js} +7 -7
- package/dist/test-exports.js +876 -483
- package/dist/test-exports.js.map +1 -1
- package/dist/trade-processor-DKDCWGKD.js +9 -0
- package/dist/{trades-store-LOL3FQSF.js → trades-store-R3NL25T2.js} +2 -2
- package/package.json +2 -6
- package/server/{chunk-CSDVJPBB.js → chunk-536NSFED.js} +29 -12
- package/server/chunk-536NSFED.js.map +1 -0
- package/server/{chunk-WA5AAPCH.js → chunk-A3UKD64A.js} +17 -8
- package/server/chunk-A3UKD64A.js.map +1 -0
- package/server/{chunk-PNKG7RY7.js → chunk-OMUDNJBJ.js} +63 -80
- package/server/chunk-OMUDNJBJ.js.map +1 -0
- package/server/{chunk-72GKJE2U.js → chunk-RJDJHZ5Y.js} +934 -335
- package/server/chunk-RJDJHZ5Y.js.map +1 -0
- package/server/{chunk-NZO6PT64.js → chunk-RM4B2VKS.js} +167 -177
- package/server/chunk-RM4B2VKS.js.map +1 -0
- package/server/{chunk-5EBXHT6C.js → chunk-SY6GDNVI.js} +24 -18
- package/server/chunk-SY6GDNVI.js.map +1 -0
- package/server/{chunk-FBNDMCT5.js → chunk-T66KH2XH.js} +13 -11
- package/server/chunk-T66KH2XH.js.map +1 -0
- package/server/{chunk-NRFXAJF7.js → chunk-W5E7FHC4.js} +24 -52
- package/server/chunk-W5E7FHC4.js.map +1 -0
- package/server/{config-6IZXEFEX.js → config-DK7KOMNL.js} +1 -1
- package/server/config-DK7KOMNL.js.map +1 -0
- package/server/daily-log-processor-B2VPU2FQ.js +10 -0
- package/server/{daily-logs-store-NU7M4L5C.js → daily-logs-store-YFNXTB7J.js} +2 -2
- package/server/http-server.js +2 -5
- package/server/index.js +2497 -2029
- package/server/index.js.map +1 -1
- package/server/iv-solver-worker.js +9 -1
- package/server/market-provider-L56RFHAC.js +17 -0
- package/server/{proto-DWRZJO4E.js → proto-YZ4TLX2N.js} +2 -2
- package/server/{sync-QFI5L7S7.js → sync-CENKDN53.js} +7 -7
- package/server/trade-processor-3PI4LHZI.js +10 -0
- package/server/{trades-store-VY26MWER.js → trades-store-RTTEYMXH.js} +2 -2
- package/src/auth/clients-store.ts +4 -4
- package/src/auth/code-store.ts +1 -1
- package/src/auth/config.ts +11 -12
- package/src/auth/login-page.ts +8 -10
- package/src/auth/provider.ts +35 -30
- package/src/auth/token.ts +17 -15
- package/src/db/backtest-schemas.ts +38 -8
- package/src/db/connection.ts +107 -40
- package/src/db/data-root.ts +22 -0
- package/src/db/index.ts +37 -5
- package/src/db/json-adapters.ts +22 -31
- package/src/db/json-migration.ts +11 -15
- package/src/db/json-store.ts +5 -3
- package/src/db/market-datasets.ts +48 -15
- package/src/db/market-schemas.ts +11 -5
- package/src/db/market-views.ts +119 -32
- package/src/db/parquet-writer.ts +7 -7
- package/src/db/profile-schemas.ts +39 -22
- package/src/db/schemas.ts +2 -2
- package/src/http-server.ts +7 -12
- package/src/index.ts +8 -9
- package/src/market/ingestor/index.ts +1 -4
- package/src/market/ingestor/market-ingestor.ts +140 -67
- package/src/market/ingestor/types.ts +1 -3
- package/src/market/stores/chain-sql.ts +2 -8
- package/src/market/stores/chain-store.ts +1 -4
- package/src/market/stores/duckdb-chain-store.ts +3 -14
- package/src/market/stores/duckdb-enriched-store.ts +3 -13
- package/src/market/stores/duckdb-quote-store.ts +10 -27
- package/src/market/stores/duckdb-spot-store.ts +4 -20
- package/src/market/stores/enriched-sql.ts +3 -9
- package/src/market/stores/enriched-store.ts +2 -2
- package/src/market/stores/parquet-chain-store.ts +5 -22
- package/src/market/stores/parquet-enriched-store.ts +4 -16
- package/src/market/stores/parquet-oi-daily-store.ts +3 -14
- package/src/market/stores/parquet-quote-store.ts +9 -29
- package/src/market/stores/parquet-spot-store.ts +14 -33
- package/src/market/stores/quote-store.ts +2 -10
- package/src/market/stores/spot-sql.ts +3 -15
- package/src/market/stores/spot-store.ts +1 -1
- package/src/market/tickers/defaults.json +6 -6
- package/src/market/tickers/loader.ts +2 -7
- package/src/market/tickers/registry.ts +5 -15
- package/src/market/tickers/schemas.ts +2 -6
- package/src/models/strategy-profile.ts +39 -39
- package/src/sync/block-sync.ts +44 -65
- package/src/sync/index.ts +1 -4
- package/src/sync/metadata.ts +14 -20
- package/src/test-exports.ts +139 -123
- package/src/tools/analysis.ts +180 -159
- package/src/tools/batch-exit-analysis.ts +100 -59
- package/src/tools/blocks/analysis.ts +116 -101
- package/src/tools/blocks/comparison.ts +281 -364
- package/src/tools/blocks/core.ts +208 -253
- package/src/tools/blocks/health.ts +571 -563
- package/src/tools/blocks/index.ts +2 -0
- package/src/tools/blocks/paired-comparison.ts +425 -0
- package/src/tools/blocks/similarity.ts +316 -283
- package/src/tools/edge-decay.ts +124 -197
- package/src/tools/exit-analysis.ts +130 -77
- package/src/tools/greeks-attribution.ts +84 -35
- package/src/tools/guides.ts +4 -6
- package/src/tools/imports.ts +11 -14
- package/src/tools/market-data.ts +1067 -757
- package/src/tools/market-enrichment.ts +3 -3
- package/src/tools/market-fetch.ts +148 -67
- package/src/tools/market-imports.ts +12 -12
- package/src/tools/middleware/sync-middleware.ts +5 -6
- package/src/tools/performance.ts +185 -302
- package/src/tools/profile-analysis.ts +52 -66
- package/src/tools/profiles.ts +106 -69
- package/src/tools/regime-advisor.ts +20 -45
- package/src/tools/replay.ts +81 -77
- package/src/tools/reports/discrepancies.ts +298 -328
- package/src/tools/reports/fields.ts +7 -25
- package/src/tools/reports/helpers.ts +18 -49
- package/src/tools/reports/predictive.ts +27 -70
- package/src/tools/reports/slippage-trends.ts +315 -345
- package/src/tools/reports/slippage.ts +1 -4
- package/src/tools/reports/strategy-matches.ts +399 -441
- package/src/tools/schema.ts +43 -40
- package/src/tools/shared/filters.ts +3 -9
- package/src/tools/snapshot.ts +9 -30
- package/src/tools/sql.ts +15 -14
- package/src/tools/tickers.ts +1 -4
- package/src/utils/batch-exit-analysis.ts +31 -42
- package/src/utils/black-scholes.ts +39 -29
- package/src/utils/block-loader.ts +69 -83
- package/src/utils/calibration-probe.ts +3 -4
- package/src/utils/chain-loader.ts +3 -3
- package/src/utils/csv-discovery.ts +16 -22
- package/src/utils/data-quality.ts +24 -36
- package/src/utils/exit-triggers.ts +91 -96
- package/src/utils/field-timing.ts +94 -79
- package/src/utils/filter-predicates.ts +13 -9
- package/src/utils/flatfile-importer.ts +94 -64
- package/src/utils/greeks-decomposition.ts +152 -100
- package/src/utils/iv-solver-pool.ts +55 -25
- package/src/utils/iv-solver-worker.ts +5 -5
- package/src/utils/market-enricher.ts +528 -497
- package/src/utils/market-importer.ts +31 -12
- package/src/utils/market-provider.ts +21 -23
- package/src/utils/massive-tier.ts +5 -7
- package/src/utils/migrate-option-data-helpers.ts +2 -8
- package/src/utils/option-quote-greeks.ts +25 -31
- package/src/utils/option-time.ts +4 -8
- package/src/utils/output-formatter.ts +1 -4
- package/src/utils/provider-capabilities.ts +1 -4
- package/src/utils/providers/massive.ts +59 -93
- package/src/utils/providers/thetadata/backfill.ts +14 -23
- package/src/utils/providers/thetadata/client.ts +12 -8
- package/src/utils/providers/thetadata/decode.ts +2 -20
- package/src/utils/providers/thetadata/endpoints.ts +32 -40
- package/src/utils/providers/thetadata/join.ts +11 -10
- package/src/utils/providers/thetadata/proto.ts +12 -10
- package/src/utils/providers/thetadata/quote-mid-greeks.ts +5 -5
- package/src/utils/providers/thetadata.ts +11 -10
- package/src/utils/quote-enricher.ts +4 -4
- package/src/utils/quote-parquet-projection.ts +3 -11
- package/src/utils/sample-date-selector.ts +3 -5
- package/src/utils/schema-metadata.ts +102 -70
- package/src/utils/ticker.ts +5 -9
- package/src/utils/trade-replay.ts +77 -68
- package/dist/chunk-4BLCXNQ6.js.map +0 -1
- package/dist/chunk-BKQ4PM4Y.js.map +0 -1
- package/dist/chunk-FGZH632F.js.map +0 -1
- package/dist/chunk-LDKTV7GW.js.map +0 -1
- package/dist/chunk-PRAYH3RT.js.map +0 -1
- package/dist/chunk-QTTR7AAW.js.map +0 -1
- package/dist/chunk-W2PP3LEH.js.map +0 -1
- package/dist/chunk-XXYOUIZY.js.map +0 -1
- package/dist/daily-log-processor-6MWJ23JK.js +0 -9
- package/dist/market-provider-VDRJUEF2.js +0 -16
- package/dist/trade-processor-NHU4VWRX.js +0 -9
- package/manifest.json +0 -141
- package/server/chunk-5EBXHT6C.js.map +0 -1
- package/server/chunk-72GKJE2U.js.map +0 -1
- package/server/chunk-CSDVJPBB.js.map +0 -1
- package/server/chunk-FBNDMCT5.js.map +0 -1
- package/server/chunk-NRFXAJF7.js.map +0 -1
- package/server/chunk-NZO6PT64.js.map +0 -1
- package/server/chunk-PNKG7RY7.js.map +0 -1
- package/server/chunk-WA5AAPCH.js.map +0 -1
- package/server/config-6IZXEFEX.js.map +0 -1
- package/server/daily-log-processor-Y3PVSVBM.js +0 -10
- package/server/market-provider-VOYYVYWT.js +0 -17
- package/server/trade-processor-JWVS37KM.js +0 -10
- /package/dist/{daily-log-processor-6MWJ23JK.js.map → daily-log-processor-MZW2XBZY.js.map} +0 -0
- /package/dist/{daily-logs-store-D3QC2XPV.js.map → daily-logs-store-3C3OTGPK.js.map} +0 -0
- /package/dist/{market-provider-VDRJUEF2.js.map → market-provider-6U33TQUT.js.map} +0 -0
- /package/dist/{proto-3HZTCWK4.js.map → proto-V23GIZG2.js.map} +0 -0
- /package/dist/{sync-PO4IPCYV.js.map → sync-3UBC37VW.js.map} +0 -0
- /package/dist/{trade-processor-NHU4VWRX.js.map → trade-processor-DKDCWGKD.js.map} +0 -0
- /package/dist/{trades-store-LOL3FQSF.js.map → trades-store-R3NL25T2.js.map} +0 -0
- /package/server/{daily-log-processor-Y3PVSVBM.js.map → daily-log-processor-B2VPU2FQ.js.map} +0 -0
- /package/server/{daily-logs-store-NU7M4L5C.js.map → daily-logs-store-YFNXTB7J.js.map} +0 -0
- /package/server/{market-provider-VOYYVYWT.js.map → market-provider-L56RFHAC.js.map} +0 -0
- /package/server/{proto-DWRZJO4E.js.map → proto-YZ4TLX2N.js.map} +0 -0
- /package/server/{sync-QFI5L7S7.js.map → sync-CENKDN53.js.map} +0 -0
- /package/server/{trade-processor-JWVS37KM.js.map → trade-processor-3PI4LHZI.js.map} +0 -0
- /package/server/{trades-store-VY26MWER.js.map → trades-store-RTTEYMXH.js.map} +0 -0
|
@@ -2,19 +2,19 @@ import {
|
|
|
2
2
|
REPORTING_TRADE_COLUMN_ALIASES,
|
|
3
3
|
convertTatRowToReportingTrade,
|
|
4
4
|
isTatFormat
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-2I3S2ZLT.js";
|
|
6
6
|
|
|
7
7
|
// src/sync/index.ts
|
|
8
8
|
import { existsSync as existsSync2 } from "fs";
|
|
9
9
|
import * as fs9 from "fs/promises";
|
|
10
|
-
import * as
|
|
10
|
+
import * as path12 from "path";
|
|
11
11
|
|
|
12
12
|
// src/db/connection.ts
|
|
13
13
|
import { DuckDBInstance } from "@duckdb/node-api";
|
|
14
14
|
import { execFile } from "child_process";
|
|
15
15
|
import * as fs4 from "fs/promises";
|
|
16
16
|
import * as os from "os";
|
|
17
|
-
import * as
|
|
17
|
+
import * as path8 from "path";
|
|
18
18
|
import { promisify } from "util";
|
|
19
19
|
|
|
20
20
|
// src/db/schemas.ts
|
|
@@ -210,13 +210,17 @@ async function ensureMarketDataTables(conn) {
|
|
|
210
210
|
|
|
211
211
|
// src/db/parquet-writer.ts
|
|
212
212
|
import * as fs from "fs/promises";
|
|
213
|
-
import * as
|
|
213
|
+
import * as path3 from "path";
|
|
214
214
|
|
|
215
215
|
// src/db/market-datasets.ts
|
|
216
|
-
import * as
|
|
216
|
+
import * as path2 from "path";
|
|
217
217
|
|
|
218
218
|
// src/db/data-root.ts
|
|
219
|
+
import path from "path";
|
|
219
220
|
var KEY = "__tradeblocks_data_root__";
|
|
221
|
+
function resolveDbPath(dataRoot, kind) {
|
|
222
|
+
return path.join(dataRoot, "database", `${kind}.duckdb`);
|
|
223
|
+
}
|
|
220
224
|
function setDataRoot(dir) {
|
|
221
225
|
globalThis[KEY] = dir;
|
|
222
226
|
}
|
|
@@ -238,19 +242,19 @@ var PARTITIONED_DATASETS = {
|
|
|
238
242
|
option_quote_minutes: "option_quote_minutes"
|
|
239
243
|
};
|
|
240
244
|
function resolveMarketDir(dataDir) {
|
|
241
|
-
return
|
|
245
|
+
return path2.join(getDataRoot(dataDir), "market");
|
|
242
246
|
}
|
|
243
247
|
function resolveCanonicalMarketFile(dataDir, dataset) {
|
|
244
|
-
return
|
|
248
|
+
return path2.join(resolveMarketDir(dataDir), SINGLE_FILE_DATASETS[dataset]);
|
|
245
249
|
}
|
|
246
250
|
function resolveCanonicalMarketPartitionDir(dataDir, dataset) {
|
|
247
|
-
return
|
|
251
|
+
return path2.join(resolveMarketDir(dataDir), PARTITIONED_DATASETS[dataset]);
|
|
248
252
|
}
|
|
249
253
|
function resolveCanonicalMarketPartitionPath(dataDir, dataset, date) {
|
|
250
|
-
return
|
|
254
|
+
return path2.join(resolveCanonicalMarketPartitionDir(dataDir, dataset), `date=${date}`);
|
|
251
255
|
}
|
|
252
256
|
function resolveCanonicalMarketPartitionFile(dataDir, dataset, date) {
|
|
253
|
-
return
|
|
257
|
+
return path2.join(resolveCanonicalMarketPartitionPath(dataDir, dataset, date), "data.parquet");
|
|
254
258
|
}
|
|
255
259
|
function canonicalMarketTableName(dataset) {
|
|
256
260
|
return `market.${dataset}`;
|
|
@@ -259,14 +263,26 @@ var DATASETS_V3 = {
|
|
|
259
263
|
spot: { subdir: "spot", partitionKeys: ["ticker", "date"], filename: "data.parquet" },
|
|
260
264
|
enriched: { subdir: "enriched", partitionKeys: ["ticker"], filename: "data.parquet" },
|
|
261
265
|
enriched_context: { subdir: "enriched/context", partitionKeys: [], filename: "data.parquet" },
|
|
262
|
-
option_chain: {
|
|
263
|
-
|
|
264
|
-
|
|
266
|
+
option_chain: {
|
|
267
|
+
subdir: "option_chain",
|
|
268
|
+
partitionKeys: ["underlying", "date"],
|
|
269
|
+
filename: "data.parquet"
|
|
270
|
+
},
|
|
271
|
+
option_quote_minutes: {
|
|
272
|
+
subdir: "option_quote_minutes",
|
|
273
|
+
partitionKeys: ["underlying", "date"],
|
|
274
|
+
filename: "data.parquet"
|
|
275
|
+
},
|
|
276
|
+
option_oi_daily: {
|
|
277
|
+
subdir: "option_oi_daily",
|
|
278
|
+
partitionKeys: ["underlying", "date"],
|
|
279
|
+
filename: "data.parquet"
|
|
280
|
+
}
|
|
265
281
|
};
|
|
266
282
|
async function writeSpotPartition(conn, args) {
|
|
267
283
|
const def = DATASETS_V3.spot;
|
|
268
284
|
return writeParquetPartition(conn, {
|
|
269
|
-
baseDir:
|
|
285
|
+
baseDir: path2.join(resolveMarketDir(args.dataDir), def.subdir),
|
|
270
286
|
partitions: { ticker: args.ticker, date: args.date },
|
|
271
287
|
// order matches def.partitionKeys
|
|
272
288
|
selectQuery: args.selectQuery,
|
|
@@ -277,7 +293,7 @@ async function writeSpotPartition(conn, args) {
|
|
|
277
293
|
async function writeChainPartition(conn, args) {
|
|
278
294
|
const def = DATASETS_V3.option_chain;
|
|
279
295
|
return writeParquetPartition(conn, {
|
|
280
|
-
baseDir:
|
|
296
|
+
baseDir: path2.join(resolveMarketDir(args.dataDir), def.subdir),
|
|
281
297
|
partitions: { underlying: args.underlying, date: args.date },
|
|
282
298
|
selectQuery: args.selectQuery,
|
|
283
299
|
compression: args.compression,
|
|
@@ -288,7 +304,7 @@ async function writeQuoteMinutesPartition(conn, args) {
|
|
|
288
304
|
const def = DATASETS_V3.option_quote_minutes;
|
|
289
305
|
const sortedSelect = `SELECT * FROM (${args.selectQuery}) AS q ORDER BY q.ticker, q.time`;
|
|
290
306
|
return writeParquetPartition(conn, {
|
|
291
|
-
baseDir:
|
|
307
|
+
baseDir: path2.join(resolveMarketDir(args.dataDir), def.subdir),
|
|
292
308
|
partitions: { underlying: args.underlying, date: args.date },
|
|
293
309
|
selectQuery: sortedSelect,
|
|
294
310
|
compression: args.compression,
|
|
@@ -299,7 +315,7 @@ async function writeOiDailyPartition(conn, args) {
|
|
|
299
315
|
const def = DATASETS_V3.option_oi_daily;
|
|
300
316
|
const sortedSelect = `SELECT * FROM (${args.selectQuery}) AS q ORDER BY q.ticker`;
|
|
301
317
|
return writeParquetPartition(conn, {
|
|
302
|
-
baseDir:
|
|
318
|
+
baseDir: path2.join(resolveMarketDir(args.dataDir), def.subdir),
|
|
303
319
|
partitions: { underlying: args.underlying, date: args.date },
|
|
304
320
|
selectQuery: sortedSelect,
|
|
305
321
|
compression: args.compression,
|
|
@@ -309,7 +325,7 @@ async function writeOiDailyPartition(conn, args) {
|
|
|
309
325
|
async function writeEnrichedTickerFile(conn, args) {
|
|
310
326
|
const def = DATASETS_V3.enriched;
|
|
311
327
|
return writeParquetPartition(conn, {
|
|
312
|
-
baseDir:
|
|
328
|
+
baseDir: path2.join(resolveMarketDir(args.dataDir), def.subdir),
|
|
313
329
|
partitions: { ticker: args.ticker },
|
|
314
330
|
selectQuery: args.selectQuery,
|
|
315
331
|
compression: args.compression,
|
|
@@ -318,7 +334,7 @@ async function writeEnrichedTickerFile(conn, args) {
|
|
|
318
334
|
}
|
|
319
335
|
async function writeEnrichedContext(conn, args) {
|
|
320
336
|
const def = DATASETS_V3.enriched_context;
|
|
321
|
-
const targetPath =
|
|
337
|
+
const targetPath = path2.join(resolveMarketDir(args.dataDir), def.subdir, def.filename);
|
|
322
338
|
return writeParquetAtomic(conn, {
|
|
323
339
|
targetPath,
|
|
324
340
|
selectQuery: args.selectQuery,
|
|
@@ -340,7 +356,7 @@ async function writeParquetAtomic(conn, opts) {
|
|
|
340
356
|
const tempPath = `${targetPath}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
341
357
|
try {
|
|
342
358
|
await conn.run(`CREATE TEMP TABLE "${stagingName}" AS ${selectQuery}`);
|
|
343
|
-
await fs.mkdir(
|
|
359
|
+
await fs.mkdir(path3.dirname(targetPath), { recursive: true });
|
|
344
360
|
await conn.run(
|
|
345
361
|
`COPY "${stagingName}" TO '${tempPath}' (FORMAT PARQUET, COMPRESSION ${compression})`
|
|
346
362
|
);
|
|
@@ -371,9 +387,7 @@ async function writeParquetPartition(conn, opts) {
|
|
|
371
387
|
const filename = isLegacy ? "data.parquet" : opts.filename ?? "data.parquet";
|
|
372
388
|
for (const [k, v] of Object.entries(partitions)) {
|
|
373
389
|
if (!PARTITION_KEY_RE.test(k)) {
|
|
374
|
-
throw new Error(
|
|
375
|
-
`writeParquetPartition: unsafe partition key: ${JSON.stringify(k)}`
|
|
376
|
-
);
|
|
390
|
+
throw new Error(`writeParquetPartition: unsafe partition key: ${JSON.stringify(k)}`);
|
|
377
391
|
}
|
|
378
392
|
if (!PARTITION_VALUE_RE.test(v)) {
|
|
379
393
|
throw new Error(
|
|
@@ -381,10 +395,8 @@ async function writeParquetPartition(conn, opts) {
|
|
|
381
395
|
);
|
|
382
396
|
}
|
|
383
397
|
}
|
|
384
|
-
const partitionSegments = Object.entries(partitions).map(
|
|
385
|
-
|
|
386
|
-
);
|
|
387
|
-
const targetPath = path2.join(opts.baseDir, ...partitionSegments, filename);
|
|
398
|
+
const partitionSegments = Object.entries(partitions).map(([k, v]) => `${k}=${v}`);
|
|
399
|
+
const targetPath = path3.join(opts.baseDir, ...partitionSegments, filename);
|
|
388
400
|
return writeParquetAtomic(conn, {
|
|
389
401
|
targetPath,
|
|
390
402
|
selectQuery: opts.selectQuery,
|
|
@@ -394,18 +406,18 @@ async function writeParquetPartition(conn, opts) {
|
|
|
394
406
|
|
|
395
407
|
// src/db/json-adapters.ts
|
|
396
408
|
import * as fs3 from "fs/promises";
|
|
397
|
-
import * as
|
|
409
|
+
import * as path5 from "path";
|
|
398
410
|
import { z } from "zod";
|
|
399
411
|
|
|
400
412
|
// src/db/json-store.ts
|
|
401
413
|
import * as fs2 from "fs/promises";
|
|
402
|
-
import * as
|
|
414
|
+
import * as path4 from "path";
|
|
403
415
|
function isEnoent(err) {
|
|
404
416
|
return typeof err === "object" && err !== null && err.code === "ENOENT";
|
|
405
417
|
}
|
|
406
418
|
async function writeJsonFile(filePath, data) {
|
|
407
419
|
const tmpPath = filePath + ".tmp";
|
|
408
|
-
await fs2.mkdir(
|
|
420
|
+
await fs2.mkdir(path4.dirname(filePath), { recursive: true });
|
|
409
421
|
await fs2.writeFile(tmpPath, JSON.stringify(data, null, 2) + "\n", "utf-8");
|
|
410
422
|
await fs2.rename(tmpPath, filePath);
|
|
411
423
|
}
|
|
@@ -434,7 +446,7 @@ async function deleteJsonFile(filePath) {
|
|
|
434
446
|
async function listJsonFiles(dirPath, suffix = ".json") {
|
|
435
447
|
try {
|
|
436
448
|
const entries = await fs2.readdir(dirPath, { withFileTypes: true });
|
|
437
|
-
return entries.filter((e) => e.isFile() && e.name.endsWith(suffix)).map((e) =>
|
|
449
|
+
return entries.filter((e) => e.isFile() && e.name.endsWith(suffix)).map((e) => path4.join(dirPath, e.name)).sort();
|
|
438
450
|
} catch (err) {
|
|
439
451
|
if (isEnoent(err)) {
|
|
440
452
|
return [];
|
|
@@ -462,7 +474,7 @@ function jsonToProfile(json) {
|
|
|
462
474
|
};
|
|
463
475
|
}
|
|
464
476
|
function profilePath(blockId, strategyName, blocksDir) {
|
|
465
|
-
return
|
|
477
|
+
return path5.join(blocksDir, blockId, "profiles", toFileSlug(strategyName) + ".json");
|
|
466
478
|
}
|
|
467
479
|
async function upsertProfileJson(profile, blocksDir) {
|
|
468
480
|
const filePath = profilePath(profile.blockId, profile.strategyName, blocksDir);
|
|
@@ -484,7 +496,7 @@ async function getProfileJson(blockId, strategyName, blocksDir) {
|
|
|
484
496
|
}
|
|
485
497
|
async function listProfilesJson(blocksDir, blockId) {
|
|
486
498
|
if (blockId) {
|
|
487
|
-
const profileDir =
|
|
499
|
+
const profileDir = path5.join(blocksDir, blockId, "profiles");
|
|
488
500
|
const files = await listJsonFiles(profileDir);
|
|
489
501
|
const profiles2 = [];
|
|
490
502
|
for (const file of files) {
|
|
@@ -524,7 +536,7 @@ function jsonToSyncMeta(json) {
|
|
|
524
536
|
};
|
|
525
537
|
}
|
|
526
538
|
function syncMetaPath(blockId, blocksDir) {
|
|
527
|
-
return
|
|
539
|
+
return path5.join(blocksDir, blockId, ".sync-meta.json");
|
|
528
540
|
}
|
|
529
541
|
async function getSyncMetadataJson(blockId, blocksDir) {
|
|
530
542
|
const filePath = syncMetaPath(blockId, blocksDir);
|
|
@@ -562,7 +574,7 @@ function marketMetaKey(source, ticker, targetTable) {
|
|
|
562
574
|
return `${source}|${ticker}|${targetTable}`;
|
|
563
575
|
}
|
|
564
576
|
function marketMetaFilePath(dataDir) {
|
|
565
|
-
return
|
|
577
|
+
return path5.join(getDataRoot(dataDir), "market-meta", "sync-metadata.json");
|
|
566
578
|
}
|
|
567
579
|
async function getMarketImportMetadataJson(source, ticker, targetTable, dataDir) {
|
|
568
580
|
const filePath = marketMetaFilePath(dataDir);
|
|
@@ -590,7 +602,7 @@ function flatLogKey(entry) {
|
|
|
590
602
|
return `${entry.date}|${entry.asset_class}|${entry.underlying}`;
|
|
591
603
|
}
|
|
592
604
|
function flatLogFilePath(dataDir) {
|
|
593
|
-
return
|
|
605
|
+
return path5.join(getDataRoot(dataDir), "market-meta", "flat-import-log.json");
|
|
594
606
|
}
|
|
595
607
|
async function getFlatImportLogJson(assetClass, underlying, from, to, dataDir) {
|
|
596
608
|
const filePath = flatLogFilePath(dataDir);
|
|
@@ -628,16 +640,14 @@ var EnrichmentWatermarksSchema = z.object({
|
|
|
628
640
|
watermarks: z.record(z.string().regex(/^[A-Z0-9._-]+$/), TickerEntrySchema)
|
|
629
641
|
});
|
|
630
642
|
function watermarksFilePath(dataDir) {
|
|
631
|
-
return
|
|
643
|
+
return path5.join(getDataRoot(dataDir), "market-meta", "enrichment-watermarks.json");
|
|
632
644
|
}
|
|
633
645
|
async function loadEnrichmentWatermarks(dataDir) {
|
|
634
646
|
const raw = await readJsonFile(watermarksFilePath(dataDir));
|
|
635
647
|
if (raw === null) return { version: 1, watermarks: {} };
|
|
636
648
|
const parsed = EnrichmentWatermarksSchema.safeParse(raw);
|
|
637
649
|
if (!parsed.success) {
|
|
638
|
-
throw new Error(
|
|
639
|
-
`enrichment-watermarks.json is malformed: ${parsed.error.message}`
|
|
640
|
-
);
|
|
650
|
+
throw new Error(`enrichment-watermarks.json is malformed: ${parsed.error.message}`);
|
|
641
651
|
}
|
|
642
652
|
return parsed.data;
|
|
643
653
|
}
|
|
@@ -678,13 +688,27 @@ async function ensureProfilesSchema(conn) {
|
|
|
678
688
|
await conn.run(`
|
|
679
689
|
ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS position_sizing JSON
|
|
680
690
|
`);
|
|
681
|
-
await conn.run(
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
await conn.run(
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
await conn.run(
|
|
691
|
+
await conn.run(
|
|
692
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS underlying VARCHAR`
|
|
693
|
+
);
|
|
694
|
+
await conn.run(
|
|
695
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS re_entry BOOLEAN`
|
|
696
|
+
);
|
|
697
|
+
await conn.run(
|
|
698
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS cap_profits BOOLEAN`
|
|
699
|
+
);
|
|
700
|
+
await conn.run(
|
|
701
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS cap_losses BOOLEAN`
|
|
702
|
+
);
|
|
703
|
+
await conn.run(
|
|
704
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS require_two_prices_pt BOOLEAN`
|
|
705
|
+
);
|
|
706
|
+
await conn.run(
|
|
707
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS close_on_completion BOOLEAN`
|
|
708
|
+
);
|
|
709
|
+
await conn.run(
|
|
710
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS ignore_margin_req BOOLEAN`
|
|
711
|
+
);
|
|
688
712
|
const strategyCols = [
|
|
689
713
|
{ name: "slippage_entry", type: "DOUBLE" },
|
|
690
714
|
{ name: "slippage_exit", type: "DOUBLE" },
|
|
@@ -698,7 +722,9 @@ async function ensureProfilesSchema(conn) {
|
|
|
698
722
|
{ name: "default_to_date", type: "VARCHAR" }
|
|
699
723
|
];
|
|
700
724
|
for (const col of strategyCols) {
|
|
701
|
-
await conn.run(
|
|
725
|
+
await conn.run(
|
|
726
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS ${col.name} ${col.type}`
|
|
727
|
+
);
|
|
702
728
|
}
|
|
703
729
|
await migrateExpectedRegimes(conn);
|
|
704
730
|
}
|
|
@@ -921,7 +947,7 @@ async function deleteProfile(conn, blockId, strategyName, baseDir) {
|
|
|
921
947
|
|
|
922
948
|
// src/db/market-views.ts
|
|
923
949
|
import { existsSync, readdirSync } from "fs";
|
|
924
|
-
import * as
|
|
950
|
+
import * as path6 from "path";
|
|
925
951
|
|
|
926
952
|
// src/utils/quote-parquet-projection.ts
|
|
927
953
|
var describeCache = /* @__PURE__ */ new Map();
|
|
@@ -1030,12 +1056,12 @@ function hasParquetPartitions(dir, partitionKey = "date") {
|
|
|
1030
1056
|
const prefix = `${partitionKey}=`;
|
|
1031
1057
|
return readdirSync(dir).some((entry) => {
|
|
1032
1058
|
if (!entry.startsWith(prefix)) return false;
|
|
1033
|
-
const partDir =
|
|
1059
|
+
const partDir = path6.join(dir, entry);
|
|
1034
1060
|
try {
|
|
1035
1061
|
const entries = readdirSync(partDir);
|
|
1036
1062
|
for (const sub of entries) {
|
|
1037
1063
|
if (sub.endsWith(".parquet")) return true;
|
|
1038
|
-
const nested =
|
|
1064
|
+
const nested = path6.join(partDir, sub);
|
|
1039
1065
|
try {
|
|
1040
1066
|
if (readdirSync(nested).some((f) => f.endsWith(".parquet"))) return true;
|
|
1041
1067
|
} catch {
|
|
@@ -1055,20 +1081,24 @@ function hasEnrichedTickerFiles(dir) {
|
|
|
1055
1081
|
try {
|
|
1056
1082
|
return readdirSync(dir).some((entry) => {
|
|
1057
1083
|
if (!entry.startsWith("ticker=")) return false;
|
|
1058
|
-
return existsSync(
|
|
1084
|
+
return existsSync(path6.join(dir, entry, "data.parquet"));
|
|
1059
1085
|
});
|
|
1060
1086
|
} catch {
|
|
1061
1087
|
return false;
|
|
1062
1088
|
}
|
|
1063
1089
|
}
|
|
1064
1090
|
function hasEnrichedContextFile(dir) {
|
|
1065
|
-
return existsSync(
|
|
1091
|
+
return existsSync(path6.join(dir, "context", "data.parquet"));
|
|
1066
1092
|
}
|
|
1067
1093
|
async function createMarketParquetViews(conn, dataDir) {
|
|
1068
1094
|
const viewsCreated = [];
|
|
1069
1095
|
const tablesKept = [];
|
|
1070
1096
|
const hiveViews = [
|
|
1071
|
-
{
|
|
1097
|
+
{
|
|
1098
|
+
name: "option_chain",
|
|
1099
|
+
subdir: resolveCanonicalMarketPartitionDir(dataDir, "option_chain"),
|
|
1100
|
+
partitionKey: "underlying"
|
|
1101
|
+
}
|
|
1072
1102
|
];
|
|
1073
1103
|
for (const { name, subdir, partitionKey } of hiveViews) {
|
|
1074
1104
|
const dirPath = subdir;
|
|
@@ -1139,7 +1169,7 @@ async function createMarketParquetViews(conn, dataDir) {
|
|
|
1139
1169
|
await conn.run("DROP TABLE IF EXISTS market.option_delta_index");
|
|
1140
1170
|
} catch {
|
|
1141
1171
|
}
|
|
1142
|
-
const spotDir =
|
|
1172
|
+
const spotDir = path6.join(resolveMarketDir(dataDir), "spot");
|
|
1143
1173
|
if (hasParquetPartitions(spotDir, "ticker")) {
|
|
1144
1174
|
try {
|
|
1145
1175
|
await conn.run("DROP VIEW IF EXISTS market.spot");
|
|
@@ -1161,7 +1191,7 @@ async function createMarketParquetViews(conn, dataDir) {
|
|
|
1161
1191
|
}
|
|
1162
1192
|
tablesKept.push("spot");
|
|
1163
1193
|
}
|
|
1164
|
-
const enrichedDir =
|
|
1194
|
+
const enrichedDir = path6.join(resolveMarketDir(dataDir), "enriched");
|
|
1165
1195
|
if (hasEnrichedTickerFiles(enrichedDir)) {
|
|
1166
1196
|
try {
|
|
1167
1197
|
await conn.run("DROP VIEW IF EXISTS market.enriched");
|
|
@@ -1194,7 +1224,7 @@ async function createMarketParquetViews(conn, dataDir) {
|
|
|
1194
1224
|
}
|
|
1195
1225
|
await conn.run(
|
|
1196
1226
|
`CREATE OR REPLACE VIEW market.enriched_context AS
|
|
1197
|
-
SELECT * FROM read_parquet('${
|
|
1227
|
+
SELECT * FROM read_parquet('${path6.join(enrichedDir, "context", "data.parquet")}')`
|
|
1198
1228
|
);
|
|
1199
1229
|
viewsCreated.push("enriched_context");
|
|
1200
1230
|
} else {
|
|
@@ -1259,7 +1289,7 @@ async function createMarketParquetViews(conn, dataDir) {
|
|
|
1259
1289
|
}
|
|
1260
1290
|
|
|
1261
1291
|
// src/db/json-migration.ts
|
|
1262
|
-
import * as
|
|
1292
|
+
import * as path7 from "path";
|
|
1263
1293
|
function toDate2(value) {
|
|
1264
1294
|
if (value instanceof Date) return value;
|
|
1265
1295
|
if (typeof value === "object" && value !== null && "micros" in value) {
|
|
@@ -1336,12 +1366,7 @@ async function migrateProfiles(conn, blocksDir) {
|
|
|
1336
1366
|
createdAt: createdAt.toISOString(),
|
|
1337
1367
|
updatedAt: updatedAt.toISOString()
|
|
1338
1368
|
};
|
|
1339
|
-
const filePath =
|
|
1340
|
-
blocksDir,
|
|
1341
|
-
blockId,
|
|
1342
|
-
"profiles",
|
|
1343
|
-
toFileSlug(strategyName) + ".json"
|
|
1344
|
-
);
|
|
1369
|
+
const filePath = path7.join(blocksDir, blockId, "profiles", toFileSlug(strategyName) + ".json");
|
|
1345
1370
|
await writeJsonFile(filePath, profileJson);
|
|
1346
1371
|
}
|
|
1347
1372
|
return rows.length;
|
|
@@ -1374,7 +1399,7 @@ async function migrateSyncMetadata(conn, blocksDir) {
|
|
|
1374
1399
|
return rows.length;
|
|
1375
1400
|
}
|
|
1376
1401
|
async function migrateMarketImportMetadata(conn, dataDir) {
|
|
1377
|
-
const existingFile = await readJsonFile(
|
|
1402
|
+
const existingFile = await readJsonFile(path7.join(dataDir, "market-meta", "sync-metadata.json"));
|
|
1378
1403
|
if (existingFile !== null) return 0;
|
|
1379
1404
|
let rows;
|
|
1380
1405
|
try {
|
|
@@ -1400,7 +1425,9 @@ async function migrateMarketImportMetadata(conn, dataDir) {
|
|
|
1400
1425
|
return rows.length;
|
|
1401
1426
|
}
|
|
1402
1427
|
async function migrateFlatImportLog(conn, dataDir) {
|
|
1403
|
-
const existingFile = await readJsonFile(
|
|
1428
|
+
const existingFile = await readJsonFile(
|
|
1429
|
+
path7.join(dataDir, "market-meta", "flat-import-log.json")
|
|
1430
|
+
);
|
|
1404
1431
|
if (existingFile !== null) return 0;
|
|
1405
1432
|
let rows;
|
|
1406
1433
|
try {
|
|
@@ -1544,7 +1571,7 @@ async function tryRecoverLockByTerminatingStaleProcess(errorMessage, dbPath, for
|
|
|
1544
1571
|
return false;
|
|
1545
1572
|
}
|
|
1546
1573
|
const normalizedDbPath = dbPath.replace(/\\/g, "/");
|
|
1547
|
-
const normalizedDbDir =
|
|
1574
|
+
const normalizedDbDir = path8.dirname(normalizedDbPath);
|
|
1548
1575
|
const isTradeblocksProcess = command.includes("tradeblocks-mcp") || command.includes("/mcp-server/server/index.js") || command.includes("packages/mcp-server/server/index.js") || command.includes("\\mcp-server\\server\\index.js") || command.includes("packages\\mcp-server\\server\\index.js");
|
|
1549
1576
|
const normalizedCommand = command.replace(/\\/g, "/");
|
|
1550
1577
|
const targetsSameDb = normalizedCommand.includes(normalizedDbPath) || normalizedCommand.includes(normalizedDbDir);
|
|
@@ -1578,16 +1605,16 @@ function resolveMarketDbPath(dataDir) {
|
|
|
1578
1605
|
const args = process.argv;
|
|
1579
1606
|
for (let i = 0; i < args.length; i++) {
|
|
1580
1607
|
if (args[i] === "--market-db" && args[i + 1]) {
|
|
1581
|
-
return
|
|
1608
|
+
return path8.resolve(args[i + 1]);
|
|
1582
1609
|
}
|
|
1583
1610
|
}
|
|
1584
1611
|
if (process.env.MARKET_DB_PATH) {
|
|
1585
|
-
return
|
|
1612
|
+
return path8.resolve(process.env.MARKET_DB_PATH);
|
|
1586
1613
|
}
|
|
1587
|
-
return
|
|
1614
|
+
return path8.join(dataDir, "market.duckdb");
|
|
1588
1615
|
}
|
|
1589
1616
|
async function attachMarketDb(conn, marketDbPath, mode) {
|
|
1590
|
-
await fs4.mkdir(
|
|
1617
|
+
await fs4.mkdir(path8.dirname(marketDbPath), { recursive: true });
|
|
1591
1618
|
const readOnlyClause = mode === "read_only" ? " (READ_ONLY)" : "";
|
|
1592
1619
|
const escapedPath = marketDbPath.replace(/'/g, "''");
|
|
1593
1620
|
try {
|
|
@@ -1634,15 +1661,18 @@ async function openReadWriteConnection(dbPath, threads, memoryLimit) {
|
|
|
1634
1661
|
await ensureReportingDataTable(connection);
|
|
1635
1662
|
await ensureMutableMarketTables(connection);
|
|
1636
1663
|
await ensureProfilesSchema(connection);
|
|
1637
|
-
const dataDir =
|
|
1664
|
+
const dataDir = path8.dirname(dbPath);
|
|
1638
1665
|
const dataRoot = getDataRoot(dataDir);
|
|
1639
1666
|
await createMarketParquetViews(connection, dataRoot);
|
|
1640
1667
|
await ensureMarketDataTables(connection);
|
|
1641
1668
|
try {
|
|
1642
|
-
const blocksDir = (await import("./sync-
|
|
1669
|
+
const blocksDir = (await import("./sync-3UBC37VW.js")).getBlocksDir(dataRoot);
|
|
1643
1670
|
await migrateMetadataToJson(connection, dataRoot, blocksDir);
|
|
1644
1671
|
} catch (err) {
|
|
1645
|
-
console.warn(
|
|
1672
|
+
console.warn(
|
|
1673
|
+
"[json-migration] Migration failed (non-fatal):",
|
|
1674
|
+
err instanceof Error ? err.message : err
|
|
1675
|
+
);
|
|
1646
1676
|
}
|
|
1647
1677
|
connectionMode = "read_write";
|
|
1648
1678
|
return connection;
|
|
@@ -1682,7 +1712,7 @@ async function getConnection(dataDir) {
|
|
|
1682
1712
|
if (connection) {
|
|
1683
1713
|
return connection;
|
|
1684
1714
|
}
|
|
1685
|
-
const dbPath =
|
|
1715
|
+
const dbPath = path8.join(dataDir, "analytics.duckdb");
|
|
1686
1716
|
const threads = process.env.DUCKDB_THREADS || defaultThreads();
|
|
1687
1717
|
const memoryLimit = process.env.DUCKDB_MEMORY_LIMIT || defaultMemoryLimit();
|
|
1688
1718
|
storedDbPath = dbPath;
|
|
@@ -1697,7 +1727,11 @@ async function getConnection(dataDir) {
|
|
|
1697
1727
|
} catch (error) {
|
|
1698
1728
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
1699
1729
|
if (isLockError(errorMessage)) {
|
|
1700
|
-
const recovered = await tryRecoverLockByTerminatingStaleProcess(
|
|
1730
|
+
const recovered = await tryRecoverLockByTerminatingStaleProcess(
|
|
1731
|
+
errorMessage,
|
|
1732
|
+
dbPath,
|
|
1733
|
+
forceRecovery
|
|
1734
|
+
);
|
|
1701
1735
|
if (recovered) {
|
|
1702
1736
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
1703
1737
|
await new Promise((r) => setTimeout(r, 500 * (attempt + 1)));
|
|
@@ -1755,7 +1789,7 @@ async function closeConnection() {
|
|
|
1755
1789
|
async function upgradeToReadWrite(dataDir, options) {
|
|
1756
1790
|
if (connectionMode === "read_write" && connection) return connection;
|
|
1757
1791
|
await closeConnection();
|
|
1758
|
-
const dbPath = storedDbPath ||
|
|
1792
|
+
const dbPath = storedDbPath || path8.join(dataDir, "analytics.duckdb");
|
|
1759
1793
|
const threads = storedThreads || process.env.DUCKDB_THREADS || defaultThreads();
|
|
1760
1794
|
const memoryLimit = storedMemoryLimit || process.env.DUCKDB_MEMORY_LIMIT || defaultMemoryLimit();
|
|
1761
1795
|
if (!storedMarketDbPath) {
|
|
@@ -1855,7 +1889,7 @@ function openMarketReadOnlyConnection(baseDir) {
|
|
|
1855
1889
|
}
|
|
1856
1890
|
async function getReadOnlyConnection(dataDir) {
|
|
1857
1891
|
if (connection) return connection;
|
|
1858
|
-
const dbPath =
|
|
1892
|
+
const dbPath = path8.join(dataDir, "analytics.duckdb");
|
|
1859
1893
|
const threads = process.env.DUCKDB_THREADS || defaultThreads();
|
|
1860
1894
|
const memoryLimit = process.env.DUCKDB_MEMORY_LIMIT || defaultMemoryLimit();
|
|
1861
1895
|
storedDbPath = dbPath;
|
|
@@ -1882,7 +1916,7 @@ function getCurrentConnection() {
|
|
|
1882
1916
|
|
|
1883
1917
|
// src/sync/block-sync.ts
|
|
1884
1918
|
import * as fs8 from "fs/promises";
|
|
1885
|
-
import * as
|
|
1919
|
+
import * as path11 from "path";
|
|
1886
1920
|
|
|
1887
1921
|
// src/sync/hasher.ts
|
|
1888
1922
|
import * as crypto from "crypto";
|
|
@@ -1940,18 +1974,13 @@ async function deleteSyncMetadata(conn, blockId, blocksDir) {
|
|
|
1940
1974
|
await deleteSyncMetadataJson(blockId, blocksDir);
|
|
1941
1975
|
return;
|
|
1942
1976
|
}
|
|
1943
|
-
await conn.run(
|
|
1944
|
-
`DELETE FROM trades._sync_metadata WHERE block_id = $1`,
|
|
1945
|
-
[blockId]
|
|
1946
|
-
);
|
|
1977
|
+
await conn.run(`DELETE FROM trades._sync_metadata WHERE block_id = $1`, [blockId]);
|
|
1947
1978
|
}
|
|
1948
1979
|
async function getAllSyncedBlockIds(conn, blocksDir) {
|
|
1949
1980
|
if (isParquetMode() && blocksDir) {
|
|
1950
1981
|
return getAllSyncedBlockIdsJson(blocksDir);
|
|
1951
1982
|
}
|
|
1952
|
-
const reader = await conn.runAndReadAll(
|
|
1953
|
-
`SELECT block_id FROM trades._sync_metadata`
|
|
1954
|
-
);
|
|
1983
|
+
const reader = await conn.runAndReadAll(`SELECT block_id FROM trades._sync_metadata`);
|
|
1955
1984
|
const rows = reader.getRows();
|
|
1956
1985
|
return rows.map((row) => row[0]);
|
|
1957
1986
|
}
|
|
@@ -2044,11 +2073,11 @@ function marketTickerDateKey(ticker, date) {
|
|
|
2044
2073
|
|
|
2045
2074
|
// src/utils/block-loader.ts
|
|
2046
2075
|
import * as fs7 from "fs/promises";
|
|
2047
|
-
import * as
|
|
2076
|
+
import * as path10 from "path";
|
|
2048
2077
|
|
|
2049
2078
|
// src/utils/csv-discovery.ts
|
|
2050
2079
|
import * as fs6 from "fs/promises";
|
|
2051
|
-
import * as
|
|
2080
|
+
import * as path9 from "path";
|
|
2052
2081
|
function parseCSVLine(line) {
|
|
2053
2082
|
const result = [];
|
|
2054
2083
|
let current = "";
|
|
@@ -2129,12 +2158,8 @@ async function detectCsvType(filePath) {
|
|
|
2129
2158
|
const reportingAliases = Object.keys(REPORTING_TRADE_COLUMN_ALIASES).map(
|
|
2130
2159
|
(k) => k.toLowerCase()
|
|
2131
2160
|
);
|
|
2132
|
-
const hasReportingColumns = reportingAliases.some(
|
|
2133
|
-
|
|
2134
|
-
);
|
|
2135
|
-
const hasActualPl = headers.some(
|
|
2136
|
-
(h) => h.includes("actual") && h.includes("p")
|
|
2137
|
-
);
|
|
2161
|
+
const hasReportingColumns = reportingAliases.some((alias) => headers.includes(alias));
|
|
2162
|
+
const hasActualPl = headers.some((h) => h.includes("actual") && h.includes("p"));
|
|
2138
2163
|
const hasReportedStyle = headers.includes("trade id") || headers.some((h) => h.includes("reported"));
|
|
2139
2164
|
if (hasActualPl || hasReportingColumns || hasReportedStyle) {
|
|
2140
2165
|
if (!hasPl || hasActualPl) {
|
|
@@ -2177,7 +2202,7 @@ async function discoverCsvFiles(folderPath) {
|
|
|
2177
2202
|
if (csvFile === "tradelog.csv" || csvFile === "dailylog.csv" || csvFile === "reportinglog.csv" || csvFile === mappings.reportinglog) {
|
|
2178
2203
|
continue;
|
|
2179
2204
|
}
|
|
2180
|
-
const csvPath =
|
|
2205
|
+
const csvPath = path9.join(folderPath, csvFile);
|
|
2181
2206
|
const detectedType = await detectCsvType(csvPath);
|
|
2182
2207
|
if (detectedType) {
|
|
2183
2208
|
if (detectedType === "tradelog" && !mappings.tradelog) {
|
|
@@ -2198,7 +2223,9 @@ async function discoverCsvFiles(folderPath) {
|
|
|
2198
2223
|
return { mappings, unrecognized };
|
|
2199
2224
|
}
|
|
2200
2225
|
function logCsvDiscoveryWarning(folderName, csvFiles) {
|
|
2201
|
-
console.error(
|
|
2226
|
+
console.error(
|
|
2227
|
+
`Warning: Folder '${folderName}' has CSV files but none match expected trade log format.`
|
|
2228
|
+
);
|
|
2202
2229
|
console.error(` Found: ${csvFiles.join(", ")}`);
|
|
2203
2230
|
console.error(` Expected columns: P/L, Date Opened, Date Closed, Symbol, Strategy`);
|
|
2204
2231
|
}
|
|
@@ -2371,7 +2398,7 @@ function convertToDailyLogEntry(raw, blockId) {
|
|
|
2371
2398
|
}
|
|
2372
2399
|
}
|
|
2373
2400
|
async function loadTrades(blockPath, filename = "tradelog.csv", blockId) {
|
|
2374
|
-
const tradelogPath =
|
|
2401
|
+
const tradelogPath = path10.join(blockPath, filename);
|
|
2375
2402
|
const content = await fs7.readFile(tradelogPath, "utf-8");
|
|
2376
2403
|
const records = parseCSV(content);
|
|
2377
2404
|
const trades = [];
|
|
@@ -2389,7 +2416,7 @@ async function loadTrades(blockPath, filename = "tradelog.csv", blockId) {
|
|
|
2389
2416
|
return trades;
|
|
2390
2417
|
}
|
|
2391
2418
|
async function loadDailyLogs(blockPath, blockId, filename = "dailylog.csv") {
|
|
2392
|
-
const dailylogPath =
|
|
2419
|
+
const dailylogPath = path10.join(blockPath, filename);
|
|
2393
2420
|
try {
|
|
2394
2421
|
await fs7.access(dailylogPath);
|
|
2395
2422
|
const content = await fs7.readFile(dailylogPath, "utf-8");
|
|
@@ -2401,9 +2428,7 @@ async function loadDailyLogs(blockPath, blockId, filename = "dailylog.csv") {
|
|
|
2401
2428
|
entries.push(entry);
|
|
2402
2429
|
}
|
|
2403
2430
|
}
|
|
2404
|
-
entries.sort(
|
|
2405
|
-
(a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()
|
|
2406
|
-
);
|
|
2431
|
+
entries.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime());
|
|
2407
2432
|
return entries.length > 0 ? entries : void 0;
|
|
2408
2433
|
} catch {
|
|
2409
2434
|
return void 0;
|
|
@@ -2411,10 +2436,10 @@ async function loadDailyLogs(blockPath, blockId, filename = "dailylog.csv") {
|
|
|
2411
2436
|
}
|
|
2412
2437
|
async function loadBlock(baseDir, blockId) {
|
|
2413
2438
|
const blocksDir = resolveBlocksBaseDir(baseDir);
|
|
2414
|
-
const blockPath =
|
|
2439
|
+
const blockPath = path10.join(blocksDir, blockId);
|
|
2415
2440
|
const { mappings } = await discoverCsvFiles(blockPath);
|
|
2416
2441
|
const tradelogFilename = mappings.tradelog || "tradelog.csv";
|
|
2417
|
-
const tradelogPath =
|
|
2442
|
+
const tradelogPath = path10.join(blockPath, tradelogFilename);
|
|
2418
2443
|
try {
|
|
2419
2444
|
await fs7.access(tradelogPath);
|
|
2420
2445
|
} catch {
|
|
@@ -2584,7 +2609,7 @@ async function listBlocks(baseDir) {
|
|
|
2584
2609
|
}
|
|
2585
2610
|
blocks.push(info);
|
|
2586
2611
|
} else if (!stats) {
|
|
2587
|
-
const blockPath =
|
|
2612
|
+
const blockPath = path10.join(blocksDir, blockId);
|
|
2588
2613
|
try {
|
|
2589
2614
|
const { mappings } = await discoverCsvFiles(blockPath);
|
|
2590
2615
|
if (mappings.tradelog) {
|
|
@@ -2612,14 +2637,12 @@ async function listBlocks(baseDir) {
|
|
|
2612
2637
|
}
|
|
2613
2638
|
function normalizeRecordHeaders(raw) {
|
|
2614
2639
|
const normalized = { ...raw };
|
|
2615
|
-
Object.entries(REPORTING_TRADE_COLUMN_ALIASES).forEach(
|
|
2616
|
-
([alias
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
delete normalized[alias];
|
|
2620
|
-
}
|
|
2640
|
+
Object.entries(REPORTING_TRADE_COLUMN_ALIASES).forEach(([alias, canonical]) => {
|
|
2641
|
+
if (normalized[alias] !== void 0) {
|
|
2642
|
+
normalized[canonical] = normalized[alias];
|
|
2643
|
+
delete normalized[alias];
|
|
2621
2644
|
}
|
|
2622
|
-
);
|
|
2645
|
+
});
|
|
2623
2646
|
return normalized;
|
|
2624
2647
|
}
|
|
2625
2648
|
function convertToReportingTrade(raw) {
|
|
@@ -2654,10 +2677,10 @@ function convertToReportingTrade(raw) {
|
|
|
2654
2677
|
}
|
|
2655
2678
|
async function loadReportingLog(baseDir, blockId) {
|
|
2656
2679
|
const blocksDir = resolveBlocksBaseDir(baseDir);
|
|
2657
|
-
const blockPath =
|
|
2680
|
+
const blockPath = path10.join(blocksDir, blockId);
|
|
2658
2681
|
const { mappings } = await discoverCsvFiles(blockPath);
|
|
2659
2682
|
const filename = mappings.reportinglog || "reportinglog.csv";
|
|
2660
|
-
const reportingLogPath =
|
|
2683
|
+
const reportingLogPath = path10.join(blockPath, filename);
|
|
2661
2684
|
try {
|
|
2662
2685
|
await fs7.access(reportingLogPath);
|
|
2663
2686
|
} catch {
|
|
@@ -2672,9 +2695,7 @@ async function loadReportingLog(baseDir, blockId) {
|
|
|
2672
2695
|
trades.push(trade);
|
|
2673
2696
|
}
|
|
2674
2697
|
}
|
|
2675
|
-
trades.sort(
|
|
2676
|
-
(a, b) => new Date(a.dateOpened).getTime() - new Date(b.dateOpened).getTime()
|
|
2677
|
-
);
|
|
2698
|
+
trades.sort((a, b) => new Date(a.dateOpened).getTime() - new Date(b.dateOpened).getTime());
|
|
2678
2699
|
return trades;
|
|
2679
2700
|
}
|
|
2680
2701
|
function toKebabCase(str) {
|
|
@@ -2714,9 +2735,7 @@ function validateCsvColumns(records, csvType) {
|
|
|
2714
2735
|
}
|
|
2715
2736
|
const dateOpenedAliases = ["Date Opened", "date_opened"];
|
|
2716
2737
|
const plAliases = ["P/L", "pl"];
|
|
2717
|
-
const hasDateOpened = dateOpenedAliases.some(
|
|
2718
|
-
(col) => headers.includes(col)
|
|
2719
|
-
);
|
|
2738
|
+
const hasDateOpened = dateOpenedAliases.some((col) => headers.includes(col));
|
|
2720
2739
|
const hasPl = plAliases.some((col) => headers.includes(col));
|
|
2721
2740
|
const missing = [];
|
|
2722
2741
|
if (!hasDateOpened) missing.push("Date Opened");
|
|
@@ -2756,11 +2775,9 @@ async function importCsv(baseDir, options) {
|
|
|
2756
2775
|
const name = blockName;
|
|
2757
2776
|
const blockId = toKebabCase(name);
|
|
2758
2777
|
if (!blockId) {
|
|
2759
|
-
throw new Error(
|
|
2760
|
-
"Could not derive a valid block ID from the filename or provided name"
|
|
2761
|
-
);
|
|
2778
|
+
throw new Error("Could not derive a valid block ID from the filename or provided name");
|
|
2762
2779
|
}
|
|
2763
|
-
const blockPath =
|
|
2780
|
+
const blockPath = path10.join(blocksDir, blockId);
|
|
2764
2781
|
try {
|
|
2765
2782
|
await fs7.access(blockPath);
|
|
2766
2783
|
throw new Error(
|
|
@@ -2773,7 +2790,7 @@ async function importCsv(baseDir, options) {
|
|
|
2773
2790
|
}
|
|
2774
2791
|
await fs7.mkdir(blockPath, { recursive: true });
|
|
2775
2792
|
const targetFilename = csvType === "tradelog" ? "tradelog.csv" : csvType === "dailylog" ? "dailylog.csv" : "reportinglog.csv";
|
|
2776
|
-
const targetPath =
|
|
2793
|
+
const targetPath = path10.join(blockPath, targetFilename);
|
|
2777
2794
|
await fs7.copyFile(csvPath, targetPath);
|
|
2778
2795
|
let dateRange = {
|
|
2779
2796
|
start: null,
|
|
@@ -2917,10 +2934,7 @@ async function insertTradeBatch(conn, blockId, records, startIdx, batchSize) {
|
|
|
2917
2934
|
for (let rowIdx = 0; rowIdx < batch.length; rowIdx++) {
|
|
2918
2935
|
const record = batch[rowIdx];
|
|
2919
2936
|
const baseParam = rowIdx * columnsPerRow + 1;
|
|
2920
|
-
const rowPlaceholders = Array.from(
|
|
2921
|
-
{ length: columnsPerRow },
|
|
2922
|
-
(_, i) => `$${baseParam + i}`
|
|
2923
|
-
);
|
|
2937
|
+
const rowPlaceholders = Array.from({ length: columnsPerRow }, (_, i) => `$${baseParam + i}`);
|
|
2924
2938
|
placeholders.push(`(${rowPlaceholders.join(", ")})`);
|
|
2925
2939
|
const premium = parseFloat(record["Premium"]);
|
|
2926
2940
|
const numContracts = parseInt(record["No. of Contracts"], 10);
|
|
@@ -2988,10 +3002,7 @@ async function insertReportingBatch(conn, blockId, trades, tickers, startIdx, ba
|
|
|
2988
3002
|
for (let rowIdx = 0; rowIdx < batch.length; rowIdx++) {
|
|
2989
3003
|
const trade = batch[rowIdx];
|
|
2990
3004
|
const baseParam = rowIdx * columnsPerRow + 1;
|
|
2991
|
-
const rowPlaceholders = Array.from(
|
|
2992
|
-
{ length: columnsPerRow },
|
|
2993
|
-
(_, i) => `$${baseParam + i}`
|
|
2994
|
-
);
|
|
3005
|
+
const rowPlaceholders = Array.from({ length: columnsPerRow }, (_, i) => `$${baseParam + i}`);
|
|
2995
3006
|
placeholders.push(`(${rowPlaceholders.join(", ")})`);
|
|
2996
3007
|
params.push(
|
|
2997
3008
|
blockId,
|
|
@@ -3025,7 +3036,7 @@ function versionedHash(hash) {
|
|
|
3025
3036
|
return `${hash}:${PARSE_VERSION}`;
|
|
3026
3037
|
}
|
|
3027
3038
|
async function syncBlockInternal(conn, blockId, blockPath) {
|
|
3028
|
-
const blocksDir =
|
|
3039
|
+
const blocksDir = path11.dirname(blockPath);
|
|
3029
3040
|
try {
|
|
3030
3041
|
const existingMetadata = await getSyncMetadata(conn, blockId, blocksDir);
|
|
3031
3042
|
const tradelogFilename = await findTradelogFile(blockPath);
|
|
@@ -3033,14 +3044,8 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3033
3044
|
if (existingMetadata) {
|
|
3034
3045
|
await conn.run("BEGIN TRANSACTION");
|
|
3035
3046
|
try {
|
|
3036
|
-
await conn.run(
|
|
3037
|
-
|
|
3038
|
-
[blockId]
|
|
3039
|
-
);
|
|
3040
|
-
await conn.run(
|
|
3041
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
3042
|
-
[blockId]
|
|
3043
|
-
);
|
|
3047
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
3048
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
3044
3049
|
await deleteSyncMetadata(conn, blockId, blocksDir);
|
|
3045
3050
|
await conn.run("COMMIT");
|
|
3046
3051
|
return { blockId, status: "deleted" };
|
|
@@ -3055,12 +3060,12 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3055
3060
|
error: "No tradelog CSV found in block"
|
|
3056
3061
|
};
|
|
3057
3062
|
}
|
|
3058
|
-
const tradelogPath =
|
|
3063
|
+
const tradelogPath = path11.join(blockPath, tradelogFilename);
|
|
3059
3064
|
const tradelogHash = versionedHash(await hashFileContent(tradelogPath));
|
|
3060
3065
|
if (existingMetadata && existingMetadata.tradelog_hash === tradelogHash) {
|
|
3061
3066
|
const optionalLogs = await findOptionalLogFiles(blockPath);
|
|
3062
3067
|
if (optionalLogs.reportinglog) {
|
|
3063
|
-
const reportinglogPath =
|
|
3068
|
+
const reportinglogPath = path11.join(blockPath, optionalLogs.reportinglog);
|
|
3064
3069
|
const reportinglogHash = versionedHash(await hashFileContent(reportinglogPath));
|
|
3065
3070
|
if (existingMetadata.reportinglog_hash !== reportinglogHash) {
|
|
3066
3071
|
} else {
|
|
@@ -3075,10 +3080,7 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3075
3080
|
}
|
|
3076
3081
|
await conn.run("BEGIN TRANSACTION");
|
|
3077
3082
|
try {
|
|
3078
|
-
await conn.run(
|
|
3079
|
-
"DELETE FROM trades.trade_data WHERE block_id = $1",
|
|
3080
|
-
[blockId]
|
|
3081
|
-
);
|
|
3083
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
3082
3084
|
const csvContent = await fs8.readFile(tradelogPath, "utf-8");
|
|
3083
3085
|
const records = parseCSV2(csvContent);
|
|
3084
3086
|
const batchSize = 500;
|
|
@@ -3090,26 +3092,23 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3090
3092
|
let reportinglogHash = null;
|
|
3091
3093
|
if (optionalLogs.dailylog) {
|
|
3092
3094
|
try {
|
|
3093
|
-
dailylogHash = versionedHash(
|
|
3094
|
-
|
|
3095
|
-
)
|
|
3095
|
+
dailylogHash = versionedHash(
|
|
3096
|
+
await hashFileContent(path11.join(blockPath, optionalLogs.dailylog))
|
|
3097
|
+
);
|
|
3096
3098
|
} catch {
|
|
3097
3099
|
}
|
|
3098
3100
|
}
|
|
3099
3101
|
if (optionalLogs.reportinglog) {
|
|
3100
3102
|
try {
|
|
3101
|
-
reportinglogHash = versionedHash(
|
|
3102
|
-
|
|
3103
|
-
)
|
|
3103
|
+
reportinglogHash = versionedHash(
|
|
3104
|
+
await hashFileContent(path11.join(blockPath, optionalLogs.reportinglog))
|
|
3105
|
+
);
|
|
3104
3106
|
} catch {
|
|
3105
3107
|
}
|
|
3106
3108
|
}
|
|
3107
|
-
await conn.run(
|
|
3108
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
3109
|
-
[blockId]
|
|
3110
|
-
);
|
|
3109
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
3111
3110
|
if (optionalLogs.reportinglog && reportinglogHash) {
|
|
3112
|
-
const reportingPath =
|
|
3111
|
+
const reportingPath = path11.join(blockPath, optionalLogs.reportinglog);
|
|
3113
3112
|
const reportingContent = await fs8.readFile(reportingPath, "utf-8");
|
|
3114
3113
|
const reportingRecords = parseCSV2(reportingContent);
|
|
3115
3114
|
const reportingTrades = [];
|
|
@@ -3122,7 +3121,14 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3122
3121
|
}
|
|
3123
3122
|
}
|
|
3124
3123
|
for (let i = 0; i < reportingTrades.length; i += batchSize) {
|
|
3125
|
-
await insertReportingBatch(
|
|
3124
|
+
await insertReportingBatch(
|
|
3125
|
+
conn,
|
|
3126
|
+
blockId,
|
|
3127
|
+
reportingTrades,
|
|
3128
|
+
reportingTickers,
|
|
3129
|
+
i,
|
|
3130
|
+
batchSize
|
|
3131
|
+
);
|
|
3126
3132
|
}
|
|
3127
3133
|
}
|
|
3128
3134
|
const newMetadata = {
|
|
@@ -3145,14 +3151,8 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3145
3151
|
if (existingMetadata) {
|
|
3146
3152
|
try {
|
|
3147
3153
|
await conn.run("BEGIN TRANSACTION");
|
|
3148
|
-
await conn.run(
|
|
3149
|
-
|
|
3150
|
-
[blockId]
|
|
3151
|
-
);
|
|
3152
|
-
await conn.run(
|
|
3153
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
3154
|
-
[blockId]
|
|
3155
|
-
);
|
|
3154
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
3155
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
3156
3156
|
await deleteSyncMetadata(conn, blockId, blocksDir);
|
|
3157
3157
|
await conn.run("COMMIT");
|
|
3158
3158
|
} catch {
|
|
@@ -3182,10 +3182,11 @@ async function detectBlockChanges(conn, baseDir) {
|
|
|
3182
3182
|
for (const entry of entries) {
|
|
3183
3183
|
if (!entry.isDirectory()) continue;
|
|
3184
3184
|
if (entry.name.startsWith(".") || entry.name.startsWith("_")) continue;
|
|
3185
|
-
if (entry.name.endsWith(".tmp") || entry.name.endsWith(".duckdb") || entry.name.endsWith(".duckdb.tmp"))
|
|
3185
|
+
if (entry.name.endsWith(".tmp") || entry.name.endsWith(".duckdb") || entry.name.endsWith(".duckdb.tmp"))
|
|
3186
|
+
continue;
|
|
3186
3187
|
const blockId = entry.name;
|
|
3187
3188
|
folderNames.add(blockId);
|
|
3188
|
-
const blockPath =
|
|
3189
|
+
const blockPath = path11.join(baseDir, blockId);
|
|
3189
3190
|
if (!syncedBlockIds.has(blockId)) {
|
|
3190
3191
|
const tradelog = await findTradelogFile(blockPath);
|
|
3191
3192
|
if (tradelog) {
|
|
@@ -3199,7 +3200,7 @@ async function detectBlockChanges(conn, baseDir) {
|
|
|
3199
3200
|
continue;
|
|
3200
3201
|
}
|
|
3201
3202
|
try {
|
|
3202
|
-
const tradelogPath =
|
|
3203
|
+
const tradelogPath = path11.join(blockPath, tradelogFilename);
|
|
3203
3204
|
const currentHash = versionedHash(await hashFileContent(tradelogPath));
|
|
3204
3205
|
const metadata = await getSyncMetadata(conn, blockId, baseDir);
|
|
3205
3206
|
if (!metadata || metadata.tradelog_hash !== currentHash) {
|
|
@@ -3207,7 +3208,7 @@ async function detectBlockChanges(conn, baseDir) {
|
|
|
3207
3208
|
} else {
|
|
3208
3209
|
const optionalLogs = await findOptionalLogFiles(blockPath);
|
|
3209
3210
|
if (optionalLogs.reportinglog) {
|
|
3210
|
-
const reportinglogPath =
|
|
3211
|
+
const reportinglogPath = path11.join(blockPath, optionalLogs.reportinglog);
|
|
3211
3212
|
const reportingHash = versionedHash(await hashFileContent(reportinglogPath));
|
|
3212
3213
|
if (metadata.reportinglog_hash !== reportingHash) {
|
|
3213
3214
|
toSync.push(blockId);
|
|
@@ -3231,14 +3232,8 @@ async function cleanupDeletedBlocks(conn, deletedBlockIds, blocksDir) {
|
|
|
3231
3232
|
for (const blockId of deletedBlockIds) {
|
|
3232
3233
|
await conn.run("BEGIN TRANSACTION");
|
|
3233
3234
|
try {
|
|
3234
|
-
await conn.run(
|
|
3235
|
-
|
|
3236
|
-
[blockId]
|
|
3237
|
-
);
|
|
3238
|
-
await conn.run(
|
|
3239
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
3240
|
-
[blockId]
|
|
3241
|
-
);
|
|
3235
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
3236
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
3242
3237
|
await deleteSyncMetadata(conn, blockId, blocksDir);
|
|
3243
3238
|
await conn.run("COMMIT");
|
|
3244
3239
|
} catch (err) {
|
|
@@ -3255,7 +3250,7 @@ function setBlocksDir(dir) {
|
|
|
3255
3250
|
}
|
|
3256
3251
|
function getBlocksDir(baseDir) {
|
|
3257
3252
|
if (_blocksDir) return _blocksDir;
|
|
3258
|
-
const nestedBlocksDir =
|
|
3253
|
+
const nestedBlocksDir = path12.join(baseDir, "blocks");
|
|
3259
3254
|
return existsSync2(nestedBlocksDir) ? nestedBlocksDir : baseDir;
|
|
3260
3255
|
}
|
|
3261
3256
|
async function syncAllBlocks(baseDir) {
|
|
@@ -3274,7 +3269,7 @@ async function syncAllBlocks(baseDir) {
|
|
|
3274
3269
|
}
|
|
3275
3270
|
}
|
|
3276
3271
|
for (const blockId of toSync) {
|
|
3277
|
-
const blockPath =
|
|
3272
|
+
const blockPath = path12.join(blocksDir, blockId);
|
|
3278
3273
|
const result = await syncBlockInternal(conn, blockId, blockPath);
|
|
3279
3274
|
results.push(result);
|
|
3280
3275
|
if (result.status === "error" && result.error) {
|
|
@@ -3293,7 +3288,7 @@ async function syncAllBlocks(baseDir) {
|
|
|
3293
3288
|
async function syncBlock(blockId, baseDir) {
|
|
3294
3289
|
const conn = await getConnection(baseDir);
|
|
3295
3290
|
const blocksDir = getBlocksDir(baseDir);
|
|
3296
|
-
const blockPath =
|
|
3291
|
+
const blockPath = path12.join(blocksDir, blockId);
|
|
3297
3292
|
try {
|
|
3298
3293
|
await fs9.access(blockPath);
|
|
3299
3294
|
} catch {
|
|
@@ -3310,6 +3305,7 @@ async function syncBlock(blockId, baseDir) {
|
|
|
3310
3305
|
export {
|
|
3311
3306
|
ensureMutableMarketTables,
|
|
3312
3307
|
ensureMarketDataTables,
|
|
3308
|
+
resolveDbPath,
|
|
3313
3309
|
setDataRoot,
|
|
3314
3310
|
getDataRoot,
|
|
3315
3311
|
resetDataRoot,
|
|
@@ -3397,4 +3393,4 @@ export {
|
|
|
3397
3393
|
loadReportingLog,
|
|
3398
3394
|
importCsv
|
|
3399
3395
|
};
|
|
3400
|
-
//# sourceMappingURL=chunk-
|
|
3396
|
+
//# sourceMappingURL=chunk-4HZLVUOZ.js.map
|