tradeblocks-mcp 3.0.2 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -80
- package/dist/{chunk-LDKTV7GW.js → chunk-27S67XW3.js} +24 -52
- package/dist/chunk-27S67XW3.js.map +1 -0
- package/dist/{chunk-FGZH632F.js → chunk-2E63THNI.js} +13 -11
- package/dist/chunk-2E63THNI.js.map +1 -0
- package/dist/{chunk-QTTR7AAW.js → chunk-2I3S2ZLT.js} +325 -84
- package/dist/chunk-2I3S2ZLT.js.map +1 -0
- package/dist/{chunk-4BLCXNQ6.js → chunk-4HZLVUOZ.js} +179 -183
- package/dist/chunk-4HZLVUOZ.js.map +1 -0
- package/dist/{chunk-XXYOUIZY.js → chunk-NWNSKALN.js} +63 -80
- package/dist/chunk-NWNSKALN.js.map +1 -0
- package/dist/{chunk-PRAYH3RT.js → chunk-WNTYA7ER.js} +29 -12
- package/dist/chunk-WNTYA7ER.js.map +1 -0
- package/dist/{chunk-BKQ4PM4Y.js → chunk-YZA4RS76.js} +24 -18
- package/dist/chunk-YZA4RS76.js.map +1 -0
- package/dist/{chunk-W2PP3LEH.js → chunk-ZTJXC3LS.js} +17 -8
- package/dist/chunk-ZTJXC3LS.js.map +1 -0
- package/dist/daily-log-processor-MZW2XBZY.js +9 -0
- package/dist/{daily-logs-store-D3QC2XPV.js → daily-logs-store-3C3OTGPK.js} +2 -2
- package/dist/iv-solver-worker.js +9 -1
- package/dist/market-provider-6U33TQUT.js +16 -0
- package/dist/{proto-3HZTCWK4.js → proto-V23GIZG2.js} +2 -2
- package/dist/{sync-PO4IPCYV.js → sync-3UBC37VW.js} +7 -7
- package/dist/test-exports.js +876 -483
- package/dist/test-exports.js.map +1 -1
- package/dist/trade-processor-DKDCWGKD.js +9 -0
- package/dist/{trades-store-LOL3FQSF.js → trades-store-R3NL25T2.js} +2 -2
- package/package.json +2 -6
- package/server/{chunk-CSDVJPBB.js → chunk-536NSFED.js} +29 -12
- package/server/chunk-536NSFED.js.map +1 -0
- package/server/{chunk-WA5AAPCH.js → chunk-A3UKD64A.js} +17 -8
- package/server/chunk-A3UKD64A.js.map +1 -0
- package/server/{chunk-PNKG7RY7.js → chunk-OMUDNJBJ.js} +63 -80
- package/server/chunk-OMUDNJBJ.js.map +1 -0
- package/server/{chunk-72GKJE2U.js → chunk-RJDJHZ5Y.js} +934 -335
- package/server/chunk-RJDJHZ5Y.js.map +1 -0
- package/server/{chunk-NZO6PT64.js → chunk-RM4B2VKS.js} +167 -177
- package/server/chunk-RM4B2VKS.js.map +1 -0
- package/server/{chunk-5EBXHT6C.js → chunk-SY6GDNVI.js} +24 -18
- package/server/chunk-SY6GDNVI.js.map +1 -0
- package/server/{chunk-FBNDMCT5.js → chunk-T66KH2XH.js} +13 -11
- package/server/chunk-T66KH2XH.js.map +1 -0
- package/server/{chunk-NRFXAJF7.js → chunk-W5E7FHC4.js} +24 -52
- package/server/chunk-W5E7FHC4.js.map +1 -0
- package/server/{config-6IZXEFEX.js → config-DK7KOMNL.js} +1 -1
- package/server/config-DK7KOMNL.js.map +1 -0
- package/server/daily-log-processor-B2VPU2FQ.js +10 -0
- package/server/{daily-logs-store-NU7M4L5C.js → daily-logs-store-YFNXTB7J.js} +2 -2
- package/server/http-server.js +2 -5
- package/server/index.js +2497 -2029
- package/server/index.js.map +1 -1
- package/server/iv-solver-worker.js +9 -1
- package/server/market-provider-L56RFHAC.js +17 -0
- package/server/{proto-DWRZJO4E.js → proto-YZ4TLX2N.js} +2 -2
- package/server/{sync-QFI5L7S7.js → sync-CENKDN53.js} +7 -7
- package/server/trade-processor-3PI4LHZI.js +10 -0
- package/server/{trades-store-VY26MWER.js → trades-store-RTTEYMXH.js} +2 -2
- package/src/auth/clients-store.ts +4 -4
- package/src/auth/code-store.ts +1 -1
- package/src/auth/config.ts +11 -12
- package/src/auth/login-page.ts +8 -10
- package/src/auth/provider.ts +35 -30
- package/src/auth/token.ts +17 -15
- package/src/db/backtest-schemas.ts +38 -8
- package/src/db/connection.ts +107 -40
- package/src/db/data-root.ts +22 -0
- package/src/db/index.ts +37 -5
- package/src/db/json-adapters.ts +22 -31
- package/src/db/json-migration.ts +11 -15
- package/src/db/json-store.ts +5 -3
- package/src/db/market-datasets.ts +48 -15
- package/src/db/market-schemas.ts +11 -5
- package/src/db/market-views.ts +119 -32
- package/src/db/parquet-writer.ts +7 -7
- package/src/db/profile-schemas.ts +39 -22
- package/src/db/schemas.ts +2 -2
- package/src/http-server.ts +7 -12
- package/src/index.ts +8 -9
- package/src/market/ingestor/index.ts +1 -4
- package/src/market/ingestor/market-ingestor.ts +140 -67
- package/src/market/ingestor/types.ts +1 -3
- package/src/market/stores/chain-sql.ts +2 -8
- package/src/market/stores/chain-store.ts +1 -4
- package/src/market/stores/duckdb-chain-store.ts +3 -14
- package/src/market/stores/duckdb-enriched-store.ts +3 -13
- package/src/market/stores/duckdb-quote-store.ts +10 -27
- package/src/market/stores/duckdb-spot-store.ts +4 -20
- package/src/market/stores/enriched-sql.ts +3 -9
- package/src/market/stores/enriched-store.ts +2 -2
- package/src/market/stores/parquet-chain-store.ts +5 -22
- package/src/market/stores/parquet-enriched-store.ts +4 -16
- package/src/market/stores/parquet-oi-daily-store.ts +3 -14
- package/src/market/stores/parquet-quote-store.ts +9 -29
- package/src/market/stores/parquet-spot-store.ts +14 -33
- package/src/market/stores/quote-store.ts +2 -10
- package/src/market/stores/spot-sql.ts +3 -15
- package/src/market/stores/spot-store.ts +1 -1
- package/src/market/tickers/defaults.json +6 -6
- package/src/market/tickers/loader.ts +2 -7
- package/src/market/tickers/registry.ts +5 -15
- package/src/market/tickers/schemas.ts +2 -6
- package/src/models/strategy-profile.ts +39 -39
- package/src/sync/block-sync.ts +44 -65
- package/src/sync/index.ts +1 -4
- package/src/sync/metadata.ts +14 -20
- package/src/test-exports.ts +139 -123
- package/src/tools/analysis.ts +180 -159
- package/src/tools/batch-exit-analysis.ts +100 -59
- package/src/tools/blocks/analysis.ts +116 -101
- package/src/tools/blocks/comparison.ts +281 -364
- package/src/tools/blocks/core.ts +208 -253
- package/src/tools/blocks/health.ts +571 -563
- package/src/tools/blocks/index.ts +2 -0
- package/src/tools/blocks/paired-comparison.ts +425 -0
- package/src/tools/blocks/similarity.ts +316 -283
- package/src/tools/edge-decay.ts +124 -197
- package/src/tools/exit-analysis.ts +130 -77
- package/src/tools/greeks-attribution.ts +84 -35
- package/src/tools/guides.ts +4 -6
- package/src/tools/imports.ts +11 -14
- package/src/tools/market-data.ts +1067 -757
- package/src/tools/market-enrichment.ts +3 -3
- package/src/tools/market-fetch.ts +148 -67
- package/src/tools/market-imports.ts +12 -12
- package/src/tools/middleware/sync-middleware.ts +5 -6
- package/src/tools/performance.ts +185 -302
- package/src/tools/profile-analysis.ts +52 -66
- package/src/tools/profiles.ts +106 -69
- package/src/tools/regime-advisor.ts +20 -45
- package/src/tools/replay.ts +81 -77
- package/src/tools/reports/discrepancies.ts +298 -328
- package/src/tools/reports/fields.ts +7 -25
- package/src/tools/reports/helpers.ts +18 -49
- package/src/tools/reports/predictive.ts +27 -70
- package/src/tools/reports/slippage-trends.ts +315 -345
- package/src/tools/reports/slippage.ts +1 -4
- package/src/tools/reports/strategy-matches.ts +399 -441
- package/src/tools/schema.ts +43 -40
- package/src/tools/shared/filters.ts +3 -9
- package/src/tools/snapshot.ts +9 -30
- package/src/tools/sql.ts +15 -14
- package/src/tools/tickers.ts +1 -4
- package/src/utils/batch-exit-analysis.ts +31 -42
- package/src/utils/black-scholes.ts +39 -29
- package/src/utils/block-loader.ts +69 -83
- package/src/utils/calibration-probe.ts +3 -4
- package/src/utils/chain-loader.ts +3 -3
- package/src/utils/csv-discovery.ts +16 -22
- package/src/utils/data-quality.ts +24 -36
- package/src/utils/exit-triggers.ts +91 -96
- package/src/utils/field-timing.ts +94 -79
- package/src/utils/filter-predicates.ts +13 -9
- package/src/utils/flatfile-importer.ts +94 -64
- package/src/utils/greeks-decomposition.ts +152 -100
- package/src/utils/iv-solver-pool.ts +55 -25
- package/src/utils/iv-solver-worker.ts +5 -5
- package/src/utils/market-enricher.ts +528 -497
- package/src/utils/market-importer.ts +31 -12
- package/src/utils/market-provider.ts +21 -23
- package/src/utils/massive-tier.ts +5 -7
- package/src/utils/migrate-option-data-helpers.ts +2 -8
- package/src/utils/option-quote-greeks.ts +25 -31
- package/src/utils/option-time.ts +4 -8
- package/src/utils/output-formatter.ts +1 -4
- package/src/utils/provider-capabilities.ts +1 -4
- package/src/utils/providers/massive.ts +59 -93
- package/src/utils/providers/thetadata/backfill.ts +14 -23
- package/src/utils/providers/thetadata/client.ts +12 -8
- package/src/utils/providers/thetadata/decode.ts +2 -20
- package/src/utils/providers/thetadata/endpoints.ts +32 -40
- package/src/utils/providers/thetadata/join.ts +11 -10
- package/src/utils/providers/thetadata/proto.ts +12 -10
- package/src/utils/providers/thetadata/quote-mid-greeks.ts +5 -5
- package/src/utils/providers/thetadata.ts +11 -10
- package/src/utils/quote-enricher.ts +4 -4
- package/src/utils/quote-parquet-projection.ts +3 -11
- package/src/utils/sample-date-selector.ts +3 -5
- package/src/utils/schema-metadata.ts +102 -70
- package/src/utils/ticker.ts +5 -9
- package/src/utils/trade-replay.ts +77 -68
- package/dist/chunk-4BLCXNQ6.js.map +0 -1
- package/dist/chunk-BKQ4PM4Y.js.map +0 -1
- package/dist/chunk-FGZH632F.js.map +0 -1
- package/dist/chunk-LDKTV7GW.js.map +0 -1
- package/dist/chunk-PRAYH3RT.js.map +0 -1
- package/dist/chunk-QTTR7AAW.js.map +0 -1
- package/dist/chunk-W2PP3LEH.js.map +0 -1
- package/dist/chunk-XXYOUIZY.js.map +0 -1
- package/dist/daily-log-processor-6MWJ23JK.js +0 -9
- package/dist/market-provider-VDRJUEF2.js +0 -16
- package/dist/trade-processor-NHU4VWRX.js +0 -9
- package/manifest.json +0 -141
- package/server/chunk-5EBXHT6C.js.map +0 -1
- package/server/chunk-72GKJE2U.js.map +0 -1
- package/server/chunk-CSDVJPBB.js.map +0 -1
- package/server/chunk-FBNDMCT5.js.map +0 -1
- package/server/chunk-NRFXAJF7.js.map +0 -1
- package/server/chunk-NZO6PT64.js.map +0 -1
- package/server/chunk-PNKG7RY7.js.map +0 -1
- package/server/chunk-WA5AAPCH.js.map +0 -1
- package/server/config-6IZXEFEX.js.map +0 -1
- package/server/daily-log-processor-Y3PVSVBM.js +0 -10
- package/server/market-provider-VOYYVYWT.js +0 -17
- package/server/trade-processor-JWVS37KM.js +0 -10
- /package/dist/{daily-log-processor-6MWJ23JK.js.map → daily-log-processor-MZW2XBZY.js.map} +0 -0
- /package/dist/{daily-logs-store-D3QC2XPV.js.map → daily-logs-store-3C3OTGPK.js.map} +0 -0
- /package/dist/{market-provider-VDRJUEF2.js.map → market-provider-6U33TQUT.js.map} +0 -0
- /package/dist/{proto-3HZTCWK4.js.map → proto-V23GIZG2.js.map} +0 -0
- /package/dist/{sync-PO4IPCYV.js.map → sync-3UBC37VW.js.map} +0 -0
- /package/dist/{trade-processor-NHU4VWRX.js.map → trade-processor-DKDCWGKD.js.map} +0 -0
- /package/dist/{trades-store-LOL3FQSF.js.map → trades-store-R3NL25T2.js.map} +0 -0
- /package/server/{daily-log-processor-Y3PVSVBM.js.map → daily-log-processor-B2VPU2FQ.js.map} +0 -0
- /package/server/{daily-logs-store-NU7M4L5C.js.map → daily-logs-store-YFNXTB7J.js.map} +0 -0
- /package/server/{market-provider-VOYYVYWT.js.map → market-provider-L56RFHAC.js.map} +0 -0
- /package/server/{proto-DWRZJO4E.js.map → proto-YZ4TLX2N.js.map} +0 -0
- /package/server/{sync-QFI5L7S7.js.map → sync-CENKDN53.js.map} +0 -0
- /package/server/{trade-processor-JWVS37KM.js.map → trade-processor-3PI4LHZI.js.map} +0 -0
- /package/server/{trades-store-VY26MWER.js.map → trades-store-RTTEYMXH.js.map} +0 -0
package/src/db/connection.ts
CHANGED
|
@@ -73,7 +73,7 @@ const isWindows = process.platform === "win32";
|
|
|
73
73
|
* a DuckDB-compatible string like "90GB".
|
|
74
74
|
*/
|
|
75
75
|
function defaultMemoryLimit(): string {
|
|
76
|
-
const totalGB = os.totalmem() /
|
|
76
|
+
const totalGB = os.totalmem() / 1024 ** 3;
|
|
77
77
|
const targetGB = Math.max(1, Math.floor(totalGB * 0.75));
|
|
78
78
|
return `${targetGB}GB`;
|
|
79
79
|
}
|
|
@@ -121,7 +121,12 @@ async function getProcessParentPid(pid: number): Promise<number | null> {
|
|
|
121
121
|
try {
|
|
122
122
|
if (isWindows) {
|
|
123
123
|
const { stdout } = await execFileAsync("wmic", [
|
|
124
|
-
"process",
|
|
124
|
+
"process",
|
|
125
|
+
"where",
|
|
126
|
+
`ProcessId=${pid}`,
|
|
127
|
+
"get",
|
|
128
|
+
"ParentProcessId",
|
|
129
|
+
"/value",
|
|
125
130
|
]);
|
|
126
131
|
const match = stdout.match(/ParentProcessId=(\d+)/);
|
|
127
132
|
if (!match) return null;
|
|
@@ -140,7 +145,12 @@ async function getProcessCommand(pid: number): Promise<string | null> {
|
|
|
140
145
|
try {
|
|
141
146
|
if (isWindows) {
|
|
142
147
|
const { stdout } = await execFileAsync("wmic", [
|
|
143
|
-
"process",
|
|
148
|
+
"process",
|
|
149
|
+
"where",
|
|
150
|
+
`ProcessId=${pid}`,
|
|
151
|
+
"get",
|
|
152
|
+
"CommandLine",
|
|
153
|
+
"/value",
|
|
144
154
|
]);
|
|
145
155
|
const match = stdout.match(/CommandLine=(.+)/);
|
|
146
156
|
if (!match) return null;
|
|
@@ -169,7 +179,7 @@ async function waitForProcessExit(pid: number, timeoutMs: number): Promise<boole
|
|
|
169
179
|
async function tryRecoverLockByTerminatingStaleProcess(
|
|
170
180
|
errorMessage: string,
|
|
171
181
|
dbPath: string,
|
|
172
|
-
forceRecovery: boolean
|
|
182
|
+
forceRecovery: boolean,
|
|
173
183
|
): Promise<boolean> {
|
|
174
184
|
const lockHolderPid = parseLockHolderPid(errorMessage);
|
|
175
185
|
if (!lockHolderPid || lockHolderPid === process.pid) {
|
|
@@ -192,7 +202,8 @@ async function tryRecoverLockByTerminatingStaleProcess(
|
|
|
192
202
|
command.includes("packages\\mcp-server\\server\\index.js");
|
|
193
203
|
// Normalize command paths for consistent comparison (Windows backslashes → forward slashes)
|
|
194
204
|
const normalizedCommand = command.replace(/\\/g, "/");
|
|
195
|
-
const targetsSameDb =
|
|
205
|
+
const targetsSameDb =
|
|
206
|
+
normalizedCommand.includes(normalizedDbPath) || normalizedCommand.includes(normalizedDbDir);
|
|
196
207
|
|
|
197
208
|
if (!isTradeblocksProcess || !targetsSameDb) {
|
|
198
209
|
return false;
|
|
@@ -203,9 +214,7 @@ async function tryRecoverLockByTerminatingStaleProcess(
|
|
|
203
214
|
// Windows: child keeps original PPID even after parent dies — check if parent is still alive.
|
|
204
215
|
// Only kill non-orphaned processes if forceRecovery is explicitly enabled.
|
|
205
216
|
const ppid = await getProcessParentPid(lockHolderPid);
|
|
206
|
-
const orphaned = isWindows
|
|
207
|
-
? (ppid !== null && !isProcessAlive(ppid))
|
|
208
|
-
: ppid === 1;
|
|
217
|
+
const orphaned = isWindows ? ppid !== null && !isProcessAlive(ppid) : ppid === 1;
|
|
209
218
|
if (!orphaned && !forceRecovery) {
|
|
210
219
|
return false;
|
|
211
220
|
}
|
|
@@ -221,12 +230,12 @@ async function tryRecoverLockByTerminatingStaleProcess(
|
|
|
221
230
|
|
|
222
231
|
const exited = await waitForProcessExit(
|
|
223
232
|
lockHolderPid,
|
|
224
|
-
Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 1500
|
|
233
|
+
Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 1500,
|
|
225
234
|
);
|
|
226
235
|
|
|
227
236
|
if (exited) {
|
|
228
237
|
console.error(
|
|
229
|
-
`Recovered DuckDB lock at ${dbPath} by stopping ${reason} tradeblocks-mcp process PID ${lockHolderPid}
|
|
238
|
+
`Recovered DuckDB lock at ${dbPath} by stopping ${reason} tradeblocks-mcp process PID ${lockHolderPid}.`,
|
|
230
239
|
);
|
|
231
240
|
}
|
|
232
241
|
|
|
@@ -267,7 +276,7 @@ function resolveMarketDbPath(dataDir: string): string {
|
|
|
267
276
|
async function attachMarketDb(
|
|
268
277
|
conn: DuckDBConnection,
|
|
269
278
|
marketDbPath: string,
|
|
270
|
-
mode: "read_write" | "read_only"
|
|
279
|
+
mode: "read_write" | "read_only",
|
|
271
280
|
): Promise<void> {
|
|
272
281
|
await fs.mkdir(path.dirname(marketDbPath), { recursive: true });
|
|
273
282
|
const readOnlyClause = mode === "read_only" ? " (READ_ONLY)" : "";
|
|
@@ -278,9 +287,17 @@ async function attachMarketDb(
|
|
|
278
287
|
const msg = error instanceof Error ? error.message : String(error);
|
|
279
288
|
if (msg.includes("corrupt") || msg.includes("Invalid") || msg.includes("cannot open")) {
|
|
280
289
|
console.error(`market.duckdb appears corrupted at ${marketDbPath}. Recreating.`);
|
|
281
|
-
try {
|
|
290
|
+
try {
|
|
291
|
+
await fs.unlink(marketDbPath);
|
|
292
|
+
} catch {
|
|
293
|
+
/* file may not exist */
|
|
294
|
+
}
|
|
282
295
|
// Also try removing WAL file
|
|
283
|
-
try {
|
|
296
|
+
try {
|
|
297
|
+
await fs.unlink(marketDbPath + ".wal");
|
|
298
|
+
} catch {
|
|
299
|
+
/* ignore */
|
|
300
|
+
}
|
|
284
301
|
await conn.run(`ATTACH '${escapedPath}' AS market${readOnlyClause}`);
|
|
285
302
|
} else {
|
|
286
303
|
throw new Error(`Failed to attach market.duckdb at ${marketDbPath}: ${msg}`);
|
|
@@ -303,7 +320,7 @@ async function detachMarketDb(conn: DuckDBConnection): Promise<void> {
|
|
|
303
320
|
async function openReadWriteConnection(
|
|
304
321
|
dbPath: string,
|
|
305
322
|
threads: string,
|
|
306
|
-
memoryLimit: string
|
|
323
|
+
memoryLimit: string,
|
|
307
324
|
): Promise<DuckDBConnection> {
|
|
308
325
|
// enable_external_access must be "true" at instance creation to allow ATTACH of local files.
|
|
309
326
|
// DuckDB 1.4+ blocks all filesystem operations (including local ATTACH) when set to "false"
|
|
@@ -373,7 +390,10 @@ async function openReadWriteConnection(
|
|
|
373
390
|
const blocksDir = (await import("../sync/index.ts")).getBlocksDir(dataRoot);
|
|
374
391
|
await migrateMetadataToJson(connection, dataRoot, blocksDir);
|
|
375
392
|
} catch (err) {
|
|
376
|
-
console.warn(
|
|
393
|
+
console.warn(
|
|
394
|
+
"[json-migration] Migration failed (non-fatal):",
|
|
395
|
+
err instanceof Error ? err.message : err,
|
|
396
|
+
);
|
|
377
397
|
}
|
|
378
398
|
|
|
379
399
|
connectionMode = "read_write";
|
|
@@ -384,7 +404,7 @@ async function openReadWriteConnection(
|
|
|
384
404
|
async function openReadOnlyConnection(
|
|
385
405
|
dbPath: string,
|
|
386
406
|
threads: string,
|
|
387
|
-
memoryLimit: string
|
|
407
|
+
memoryLimit: string,
|
|
388
408
|
): Promise<DuckDBConnection> {
|
|
389
409
|
// enable_external_access must be "true" at instance creation to allow ATTACH.
|
|
390
410
|
// We do NOT call SET enable_external_access = false because it also blocks local
|
|
@@ -405,11 +425,19 @@ async function openReadOnlyConnection(
|
|
|
405
425
|
|
|
406
426
|
function resetConnectionState(): void {
|
|
407
427
|
if (connection) {
|
|
408
|
-
try {
|
|
428
|
+
try {
|
|
429
|
+
connection.closeSync();
|
|
430
|
+
} catch {
|
|
431
|
+
/* non-fatal */
|
|
432
|
+
}
|
|
409
433
|
}
|
|
410
434
|
connection = null;
|
|
411
435
|
if (instance) {
|
|
412
|
-
try {
|
|
436
|
+
try {
|
|
437
|
+
instance.closeSync();
|
|
438
|
+
} catch {
|
|
439
|
+
/* non-fatal */
|
|
440
|
+
}
|
|
413
441
|
}
|
|
414
442
|
instance = null;
|
|
415
443
|
connectionMode = null;
|
|
@@ -469,7 +497,11 @@ export async function getConnection(dataDir: string): Promise<DuckDBConnection>
|
|
|
469
497
|
// Lock recovery: auto-kill orphaned tradeblocks-mcp processes (PPID=1) that hold the lock.
|
|
470
498
|
// With DUCKDB_LOCK_RECOVERY=true, also kills non-orphaned holders (force mode).
|
|
471
499
|
if (isLockError(errorMessage)) {
|
|
472
|
-
const recovered = await tryRecoverLockByTerminatingStaleProcess(
|
|
500
|
+
const recovered = await tryRecoverLockByTerminatingStaleProcess(
|
|
501
|
+
errorMessage,
|
|
502
|
+
dbPath,
|
|
503
|
+
forceRecovery,
|
|
504
|
+
);
|
|
473
505
|
if (recovered) {
|
|
474
506
|
// DuckDB file locks may linger briefly after process death — retry with backoff
|
|
475
507
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
@@ -484,7 +516,7 @@ export async function getConnection(dataDir: string): Promise<DuckDBConnection>
|
|
|
484
516
|
if (attempt < 2 && isLockError(retryMsg)) continue;
|
|
485
517
|
resetConnectionState();
|
|
486
518
|
throw new Error(
|
|
487
|
-
`Failed to initialize DuckDB at ${dbPath} after lock recovery: ${retryMsg}
|
|
519
|
+
`Failed to initialize DuckDB at ${dbPath} after lock recovery: ${retryMsg}`,
|
|
488
520
|
);
|
|
489
521
|
}
|
|
490
522
|
}
|
|
@@ -503,7 +535,7 @@ export async function getConnection(dataDir: string): Promise<DuckDBConnection>
|
|
|
503
535
|
throw new Error(
|
|
504
536
|
`DuckDB database appears corrupted at ${dbPath}. ` +
|
|
505
537
|
`Please delete the file manually and restart. ` +
|
|
506
|
-
`Original error: ${errorMessage}
|
|
538
|
+
`Original error: ${errorMessage}`,
|
|
507
539
|
);
|
|
508
540
|
}
|
|
509
541
|
|
|
@@ -520,8 +552,16 @@ export async function getConnection(dataDir: string): Promise<DuckDBConnection>
|
|
|
520
552
|
*/
|
|
521
553
|
export async function closeConnection(): Promise<void> {
|
|
522
554
|
if (connection) {
|
|
523
|
-
try {
|
|
524
|
-
|
|
555
|
+
try {
|
|
556
|
+
await connection.run("CHECKPOINT");
|
|
557
|
+
} catch {
|
|
558
|
+
/* non-fatal */
|
|
559
|
+
}
|
|
560
|
+
try {
|
|
561
|
+
await detachMarketDb(connection);
|
|
562
|
+
} catch {
|
|
563
|
+
/* non-fatal, log debug */
|
|
564
|
+
}
|
|
525
565
|
try {
|
|
526
566
|
// closeSync is the synchronous close method for DuckDB connections
|
|
527
567
|
connection.closeSync();
|
|
@@ -536,7 +576,11 @@ export async function closeConnection(): Promise<void> {
|
|
|
536
576
|
// Close the DuckDB instance to release the file lock.
|
|
537
577
|
// Without this, the native handle leaks until GC and blocks subsequent RW opens.
|
|
538
578
|
if (instance) {
|
|
539
|
-
try {
|
|
579
|
+
try {
|
|
580
|
+
instance.closeSync();
|
|
581
|
+
} catch {
|
|
582
|
+
/* non-fatal */
|
|
583
|
+
}
|
|
540
584
|
}
|
|
541
585
|
instance = null;
|
|
542
586
|
connectionMode = null;
|
|
@@ -554,7 +598,7 @@ export async function closeConnection(): Promise<void> {
|
|
|
554
598
|
*/
|
|
555
599
|
export async function upgradeToReadWrite(
|
|
556
600
|
dataDir: string,
|
|
557
|
-
options?: { fallbackToReadOnly?: boolean }
|
|
601
|
+
options?: { fallbackToReadOnly?: boolean },
|
|
558
602
|
): Promise<DuckDBConnection> {
|
|
559
603
|
if (connectionMode === "read_write" && connection) return connection;
|
|
560
604
|
await closeConnection();
|
|
@@ -599,9 +643,12 @@ export async function upgradeToReadWrite(
|
|
|
599
643
|
}
|
|
600
644
|
}
|
|
601
645
|
|
|
602
|
-
throw
|
|
603
|
-
|
|
604
|
-
|
|
646
|
+
throw (
|
|
647
|
+
lastError ||
|
|
648
|
+
new Error(
|
|
649
|
+
"Cannot acquire DuckDB write lock. Another process holds it. " +
|
|
650
|
+
"Kill other tradeblocks-mcp processes or restart Claude Code.",
|
|
651
|
+
)
|
|
605
652
|
);
|
|
606
653
|
}
|
|
607
654
|
|
|
@@ -666,9 +713,7 @@ export interface MarketOnlyConnection {
|
|
|
666
713
|
close(): Promise<void>;
|
|
667
714
|
}
|
|
668
715
|
|
|
669
|
-
export async function openMarketOnlyConnection(
|
|
670
|
-
baseDir: string,
|
|
671
|
-
): Promise<MarketOnlyConnection> {
|
|
716
|
+
export async function openMarketOnlyConnection(baseDir: string): Promise<MarketOnlyConnection> {
|
|
672
717
|
const marketDbPath = resolveMarketDbPath(baseDir);
|
|
673
718
|
|
|
674
719
|
// `:memory:` host means the connection does not open any on-disk database
|
|
@@ -704,10 +749,26 @@ export async function openMarketOnlyConnection(
|
|
|
704
749
|
// for the next reader. DETACH is best-effort — if it fails (e.g. an
|
|
705
750
|
// in-flight statement still references the catalog), we still want to
|
|
706
751
|
// close the handle so subsequent processes can acquire the RW lock.
|
|
707
|
-
try {
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
752
|
+
try {
|
|
753
|
+
await conn.run("CHECKPOINT market");
|
|
754
|
+
} catch {
|
|
755
|
+
/* non-fatal */
|
|
756
|
+
}
|
|
757
|
+
try {
|
|
758
|
+
await detachMarketDb(conn);
|
|
759
|
+
} catch {
|
|
760
|
+
/* non-fatal */
|
|
761
|
+
}
|
|
762
|
+
try {
|
|
763
|
+
conn.closeSync();
|
|
764
|
+
} catch {
|
|
765
|
+
/* non-fatal */
|
|
766
|
+
}
|
|
767
|
+
try {
|
|
768
|
+
memoryInstance.closeSync();
|
|
769
|
+
} catch {
|
|
770
|
+
/* non-fatal */
|
|
771
|
+
}
|
|
711
772
|
};
|
|
712
773
|
|
|
713
774
|
return { conn, marketDbPath, close };
|
|
@@ -800,8 +861,16 @@ export async function openMarketParquetConnection(
|
|
|
800
861
|
const close = async (): Promise<void> => {
|
|
801
862
|
if (closed) return;
|
|
802
863
|
closed = true;
|
|
803
|
-
try {
|
|
804
|
-
|
|
864
|
+
try {
|
|
865
|
+
conn.closeSync();
|
|
866
|
+
} catch {
|
|
867
|
+
/* non-fatal */
|
|
868
|
+
}
|
|
869
|
+
try {
|
|
870
|
+
memoryInstance.closeSync();
|
|
871
|
+
} catch {
|
|
872
|
+
/* non-fatal */
|
|
873
|
+
}
|
|
805
874
|
};
|
|
806
875
|
|
|
807
876
|
return { conn, dataRoot, close };
|
|
@@ -815,9 +884,7 @@ export async function openMarketParquetConnection(
|
|
|
815
884
|
*/
|
|
816
885
|
export type MarketReadOnlyConnection = MarketParquetConnection;
|
|
817
886
|
|
|
818
|
-
export function openMarketReadOnlyConnection(
|
|
819
|
-
baseDir: string,
|
|
820
|
-
): Promise<MarketReadOnlyConnection> {
|
|
887
|
+
export function openMarketReadOnlyConnection(baseDir: string): Promise<MarketReadOnlyConnection> {
|
|
821
888
|
return openMarketParquetConnection(baseDir);
|
|
822
889
|
}
|
|
823
890
|
|
package/src/db/data-root.ts
CHANGED
|
@@ -17,9 +17,31 @@
|
|
|
17
17
|
* contexts, or sandboxed iframes will see the same divergence across realm
|
|
18
18
|
* boundaries and must coordinate state another way.
|
|
19
19
|
*/
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
|
|
20
22
|
const KEY = "__tradeblocks_data_root__";
|
|
21
23
|
type GlobalSlot = { [KEY]?: string | null };
|
|
22
24
|
|
|
25
|
+
/**
|
|
26
|
+
* The DuckDB files that live under a data root's `database/` subdirectory.
|
|
27
|
+
* - `analytics` — the base DB hosting the `trades` schema (CSV-imported blocks)
|
|
28
|
+
* - `market` — attached market data (`market.duckdb`)
|
|
29
|
+
* - `backtests` — the backtest run-persistence index (`backtests.duckdb`)
|
|
30
|
+
*/
|
|
31
|
+
export type DbKind = "analytics" | "market" | "backtests";
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Single source of truth for where a DuckDB file lives under a data root:
|
|
35
|
+
* `<dataRoot>/database/<kind>.duckdb`. Every writer and reader across the fleet
|
|
36
|
+
* (the engine's backtests-db resolver, the console reader, the calibration
|
|
37
|
+
* probe) MUST resolve through this so the `database/` segment can never drift —
|
|
38
|
+
* a missing segment silently splits reads from writes (runs succeed, the console
|
|
39
|
+
* shows an empty history forever). See tradeblocks-org/enterprise#983.
|
|
40
|
+
*/
|
|
41
|
+
export function resolveDbPath(dataRoot: string, kind: DbKind): string {
|
|
42
|
+
return path.join(dataRoot, "database", `${kind}.duckdb`);
|
|
43
|
+
}
|
|
44
|
+
|
|
23
45
|
export function setDataRoot(dir: string): void {
|
|
24
46
|
(globalThis as GlobalSlot)[KEY] = dir;
|
|
25
47
|
}
|
package/src/db/index.ts
CHANGED
|
@@ -5,8 +5,23 @@
|
|
|
5
5
|
* (analytics.duckdb) and the market database (market.duckdb).
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
export {
|
|
9
|
-
|
|
8
|
+
export {
|
|
9
|
+
getConnection,
|
|
10
|
+
closeConnection,
|
|
11
|
+
isConnected,
|
|
12
|
+
upgradeToReadWrite,
|
|
13
|
+
downgradeToReadOnly,
|
|
14
|
+
getConnectionMode,
|
|
15
|
+
getCurrentConnection,
|
|
16
|
+
openMarketOnlyConnection,
|
|
17
|
+
openMarketParquetConnection,
|
|
18
|
+
openMarketReadOnlyConnection,
|
|
19
|
+
} from "./connection.ts";
|
|
20
|
+
export type {
|
|
21
|
+
MarketOnlyConnection,
|
|
22
|
+
MarketParquetConnection,
|
|
23
|
+
MarketReadOnlyConnection,
|
|
24
|
+
} from "./connection.ts";
|
|
10
25
|
export {
|
|
11
26
|
ensureSyncTables,
|
|
12
27
|
ensureTradeDataTable,
|
|
@@ -14,8 +29,19 @@ export {
|
|
|
14
29
|
tableExists,
|
|
15
30
|
} from "./schemas.ts";
|
|
16
31
|
export { ensureMutableMarketTables, ensureMarketDataTables } from "./market-schemas.ts";
|
|
17
|
-
export {
|
|
18
|
-
|
|
32
|
+
export {
|
|
33
|
+
ensureProfilesSchema,
|
|
34
|
+
upsertProfile,
|
|
35
|
+
getProfile,
|
|
36
|
+
listProfiles,
|
|
37
|
+
deleteProfile,
|
|
38
|
+
} from "./profile-schemas.ts";
|
|
39
|
+
export {
|
|
40
|
+
isParquetMode,
|
|
41
|
+
writeParquetAtomic,
|
|
42
|
+
writeParquetPartition,
|
|
43
|
+
resolveMarketDir,
|
|
44
|
+
} from "./parquet-writer.ts";
|
|
19
45
|
export {
|
|
20
46
|
resolveCanonicalMarketFile,
|
|
21
47
|
resolveCanonicalMarketPartitionDir,
|
|
@@ -23,4 +49,10 @@ export {
|
|
|
23
49
|
resolveCanonicalMarketPartitionFile,
|
|
24
50
|
canonicalMarketTableName,
|
|
25
51
|
} from "./market-datasets.ts";
|
|
26
|
-
export {
|
|
52
|
+
export {
|
|
53
|
+
readJsonFile,
|
|
54
|
+
writeJsonFile,
|
|
55
|
+
deleteJsonFile,
|
|
56
|
+
listJsonFiles,
|
|
57
|
+
toFileSlug,
|
|
58
|
+
} from "./json-store.ts";
|
package/src/db/json-adapters.ts
CHANGED
|
@@ -70,7 +70,7 @@ function profilePath(blockId: string, strategyName: string, blocksDir: string):
|
|
|
70
70
|
*/
|
|
71
71
|
export async function upsertProfileJson(
|
|
72
72
|
profile: Omit<StrategyProfile, "createdAt" | "updatedAt">,
|
|
73
|
-
blocksDir: string
|
|
73
|
+
blocksDir: string,
|
|
74
74
|
): Promise<StrategyProfile> {
|
|
75
75
|
const filePath = profilePath(profile.blockId, profile.strategyName, blocksDir);
|
|
76
76
|
const now = new Date();
|
|
@@ -97,7 +97,7 @@ export async function upsertProfileJson(
|
|
|
97
97
|
export async function getProfileJson(
|
|
98
98
|
blockId: string,
|
|
99
99
|
strategyName: string,
|
|
100
|
-
blocksDir: string
|
|
100
|
+
blocksDir: string,
|
|
101
101
|
): Promise<StrategyProfile | null> {
|
|
102
102
|
const filePath = profilePath(blockId, strategyName, blocksDir);
|
|
103
103
|
const json = await readJsonFile<ProfileJson>(filePath);
|
|
@@ -113,7 +113,7 @@ export async function getProfileJson(
|
|
|
113
113
|
*/
|
|
114
114
|
export async function listProfilesJson(
|
|
115
115
|
blocksDir: string,
|
|
116
|
-
blockId?: string
|
|
116
|
+
blockId?: string,
|
|
117
117
|
): Promise<StrategyProfile[]> {
|
|
118
118
|
if (blockId) {
|
|
119
119
|
const profileDir = path.join(blocksDir, blockId, "profiles");
|
|
@@ -151,7 +151,7 @@ export async function listProfilesJson(
|
|
|
151
151
|
export async function deleteProfileJson(
|
|
152
152
|
blockId: string,
|
|
153
153
|
strategyName: string,
|
|
154
|
-
blocksDir: string
|
|
154
|
+
blocksDir: string,
|
|
155
155
|
): Promise<boolean> {
|
|
156
156
|
const filePath = profilePath(blockId, strategyName, blocksDir);
|
|
157
157
|
return deleteJsonFile(filePath);
|
|
@@ -195,7 +195,7 @@ function syncMetaPath(blockId: string, blocksDir: string): string {
|
|
|
195
195
|
*/
|
|
196
196
|
export async function getSyncMetadataJson(
|
|
197
197
|
blockId: string,
|
|
198
|
-
blocksDir: string
|
|
198
|
+
blocksDir: string,
|
|
199
199
|
): Promise<BlockSyncMetadata | null> {
|
|
200
200
|
const filePath = syncMetaPath(blockId, blocksDir);
|
|
201
201
|
const json = await readJsonFile<SyncMetadataJson>(filePath);
|
|
@@ -207,7 +207,7 @@ export async function getSyncMetadataJson(
|
|
|
207
207
|
*/
|
|
208
208
|
export async function upsertSyncMetadataJson(
|
|
209
209
|
metadata: BlockSyncMetadata,
|
|
210
|
-
blocksDir: string
|
|
210
|
+
blocksDir: string,
|
|
211
211
|
): Promise<void> {
|
|
212
212
|
const filePath = syncMetaPath(metadata.block_id, blocksDir);
|
|
213
213
|
await writeJsonFile(filePath, syncMetaToJson(metadata));
|
|
@@ -218,10 +218,7 @@ export async function upsertSyncMetadataJson(
|
|
|
218
218
|
*
|
|
219
219
|
* @returns true if deleted, false if not found
|
|
220
220
|
*/
|
|
221
|
-
export async function deleteSyncMetadataJson(
|
|
222
|
-
blockId: string,
|
|
223
|
-
blocksDir: string
|
|
224
|
-
): Promise<boolean> {
|
|
221
|
+
export async function deleteSyncMetadataJson(blockId: string, blocksDir: string): Promise<boolean> {
|
|
225
222
|
const filePath = syncMetaPath(blockId, blocksDir);
|
|
226
223
|
return deleteJsonFile(filePath);
|
|
227
224
|
}
|
|
@@ -231,9 +228,7 @@ export async function deleteSyncMetadataJson(
|
|
|
231
228
|
*
|
|
232
229
|
* @returns Array of block IDs that have sync metadata
|
|
233
230
|
*/
|
|
234
|
-
export async function getAllSyncedBlockIdsJson(
|
|
235
|
-
blocksDir: string
|
|
236
|
-
): Promise<string[]> {
|
|
231
|
+
export async function getAllSyncedBlockIdsJson(blocksDir: string): Promise<string[]> {
|
|
237
232
|
let entries: import("fs").Dirent[];
|
|
238
233
|
try {
|
|
239
234
|
entries = await fs.readdir(blocksDir, { withFileTypes: true });
|
|
@@ -287,7 +282,7 @@ export async function getMarketImportMetadataJson(
|
|
|
287
282
|
source: string,
|
|
288
283
|
ticker: string,
|
|
289
284
|
targetTable: string,
|
|
290
|
-
dataDir: string
|
|
285
|
+
dataDir: string,
|
|
291
286
|
): Promise<MarketImportMetadata | null> {
|
|
292
287
|
const filePath = marketMetaFilePath(dataDir);
|
|
293
288
|
const file = await readJsonFile<MarketSyncMetadataFile>(filePath);
|
|
@@ -309,10 +304,10 @@ export async function getMarketImportMetadataJson(
|
|
|
309
304
|
*/
|
|
310
305
|
export async function upsertMarketImportMetadataJson(
|
|
311
306
|
metadata: MarketImportMetadata,
|
|
312
|
-
dataDir: string
|
|
307
|
+
dataDir: string,
|
|
313
308
|
): Promise<void> {
|
|
314
309
|
const filePath = marketMetaFilePath(dataDir);
|
|
315
|
-
const file = await readJsonFile<MarketSyncMetadataFile>(filePath) ?? { entries: {} };
|
|
310
|
+
const file = (await readJsonFile<MarketSyncMetadataFile>(filePath)) ?? { entries: {} };
|
|
316
311
|
|
|
317
312
|
const key = marketMetaKey(metadata.source, metadata.ticker, metadata.target_table);
|
|
318
313
|
file.entries[key] = {
|
|
@@ -362,7 +357,7 @@ export async function getFlatImportLogJson(
|
|
|
362
357
|
underlying: string,
|
|
363
358
|
from: string,
|
|
364
359
|
to: string,
|
|
365
|
-
dataDir: string
|
|
360
|
+
dataDir: string,
|
|
366
361
|
): Promise<Set<string>> {
|
|
367
362
|
const filePath = flatLogFilePath(dataDir);
|
|
368
363
|
const file = await readJsonFile<FlatImportLogFile>(filePath);
|
|
@@ -388,13 +383,13 @@ export async function getFlatImportLogJson(
|
|
|
388
383
|
*/
|
|
389
384
|
export async function upsertFlatImportLogJson(
|
|
390
385
|
entry: FlatImportLogEntry,
|
|
391
|
-
dataDir: string
|
|
386
|
+
dataDir: string,
|
|
392
387
|
): Promise<void> {
|
|
393
388
|
const filePath = flatLogFilePath(dataDir);
|
|
394
|
-
const file = await readJsonFile<FlatImportLogFile>(filePath) ?? { entries: [] };
|
|
389
|
+
const file = (await readJsonFile<FlatImportLogFile>(filePath)) ?? { entries: [] };
|
|
395
390
|
|
|
396
391
|
const key = flatLogKey(entry);
|
|
397
|
-
const idx = file.entries.findIndex(e => flatLogKey(e) === key);
|
|
392
|
+
const idx = file.entries.findIndex((e) => flatLogKey(e) === key);
|
|
398
393
|
if (idx >= 0) {
|
|
399
394
|
file.entries[idx] = entry;
|
|
400
395
|
} else {
|
|
@@ -430,7 +425,10 @@ const TickerEntrySchema = z
|
|
|
430
425
|
.object({
|
|
431
426
|
// ISO calendar date `YYYY-MM-DD` or null when the ticker has been
|
|
432
427
|
// registered but not yet enriched through any date.
|
|
433
|
-
enriched_through: z
|
|
428
|
+
enriched_through: z
|
|
429
|
+
.string()
|
|
430
|
+
.regex(/^\d{4}-\d{2}-\d{2}$/)
|
|
431
|
+
.nullable(),
|
|
434
432
|
})
|
|
435
433
|
// Passthrough keeps forward compatibility — fields like `wilder_state` added
|
|
436
434
|
// by future phases survive load/upsert round-trips without schema changes.
|
|
@@ -458,16 +456,12 @@ function watermarksFilePath(dataDir: string): string {
|
|
|
458
456
|
* clear error; we never silently reset to empty, which would lose data
|
|
459
457
|
* invisibly.
|
|
460
458
|
*/
|
|
461
|
-
export async function loadEnrichmentWatermarks(
|
|
462
|
-
dataDir: string,
|
|
463
|
-
): Promise<EnrichmentWatermarks> {
|
|
459
|
+
export async function loadEnrichmentWatermarks(dataDir: string): Promise<EnrichmentWatermarks> {
|
|
464
460
|
const raw = await readJsonFile<unknown>(watermarksFilePath(dataDir));
|
|
465
461
|
if (raw === null) return { version: 1, watermarks: {} };
|
|
466
462
|
const parsed = EnrichmentWatermarksSchema.safeParse(raw);
|
|
467
463
|
if (!parsed.success) {
|
|
468
|
-
throw new Error(
|
|
469
|
-
`enrichment-watermarks.json is malformed: ${parsed.error.message}`,
|
|
470
|
-
);
|
|
464
|
+
throw new Error(`enrichment-watermarks.json is malformed: ${parsed.error.message}`);
|
|
471
465
|
}
|
|
472
466
|
return parsed.data;
|
|
473
467
|
}
|
|
@@ -479,10 +473,7 @@ export async function loadEnrichmentWatermarks(
|
|
|
479
473
|
* same (most common case) can use the value directly; callers that need to
|
|
480
474
|
* distinguish can read the raw file via `loadEnrichmentWatermarks`.
|
|
481
475
|
*/
|
|
482
|
-
export async function getEnrichedThrough(
|
|
483
|
-
ticker: string,
|
|
484
|
-
dataDir: string,
|
|
485
|
-
): Promise<string | null> {
|
|
476
|
+
export async function getEnrichedThrough(ticker: string, dataDir: string): Promise<string | null> {
|
|
486
477
|
const { watermarks } = await loadEnrichmentWatermarks(dataDir);
|
|
487
478
|
return watermarks[ticker]?.enriched_through ?? null;
|
|
488
479
|
}
|
package/src/db/json-migration.ts
CHANGED
|
@@ -15,11 +15,7 @@
|
|
|
15
15
|
import type { DuckDBConnection } from "@duckdb/node-api";
|
|
16
16
|
import * as path from "path";
|
|
17
17
|
import { isParquetMode } from "./parquet-writer.ts";
|
|
18
|
-
import {
|
|
19
|
-
readJsonFile,
|
|
20
|
-
writeJsonFile,
|
|
21
|
-
toFileSlug,
|
|
22
|
-
} from "./json-store.ts";
|
|
18
|
+
import { readJsonFile, writeJsonFile, toFileSlug } from "./json-store.ts";
|
|
23
19
|
import {
|
|
24
20
|
listProfilesJson,
|
|
25
21
|
upsertSyncMetadataJson,
|
|
@@ -149,12 +145,7 @@ async function migrateProfiles(conn: DuckDBConnection, blocksDir: string): Promi
|
|
|
149
145
|
};
|
|
150
146
|
|
|
151
147
|
// Write directly using json-store to preserve original timestamps
|
|
152
|
-
const filePath = path.join(
|
|
153
|
-
blocksDir,
|
|
154
|
-
blockId,
|
|
155
|
-
"profiles",
|
|
156
|
-
toFileSlug(strategyName) + ".json"
|
|
157
|
-
);
|
|
148
|
+
const filePath = path.join(blocksDir, blockId, "profiles", toFileSlug(strategyName) + ".json");
|
|
158
149
|
await writeJsonFile(filePath, profileJson);
|
|
159
150
|
}
|
|
160
151
|
|
|
@@ -202,7 +193,10 @@ async function migrateSyncMetadata(conn: DuckDBConnection, blocksDir: string): P
|
|
|
202
193
|
* Migrate market import metadata from DuckDB to JSON.
|
|
203
194
|
* @returns Count of migrated entries
|
|
204
195
|
*/
|
|
205
|
-
async function migrateMarketImportMetadata(
|
|
196
|
+
async function migrateMarketImportMetadata(
|
|
197
|
+
conn: DuckDBConnection,
|
|
198
|
+
dataDir: string,
|
|
199
|
+
): Promise<number> {
|
|
206
200
|
// Idempotency: skip if aggregate JSON file already exists
|
|
207
201
|
const existingFile = await readJsonFile(path.join(dataDir, "market-meta", "sync-metadata.json"));
|
|
208
202
|
if (existingFile !== null) return 0;
|
|
@@ -240,7 +234,9 @@ async function migrateMarketImportMetadata(conn: DuckDBConnection, dataDir: stri
|
|
|
240
234
|
*/
|
|
241
235
|
async function migrateFlatImportLog(conn: DuckDBConnection, dataDir: string): Promise<number> {
|
|
242
236
|
// Idempotency: skip if aggregate JSON file already exists
|
|
243
|
-
const existingFile = await readJsonFile(
|
|
237
|
+
const existingFile = await readJsonFile(
|
|
238
|
+
path.join(dataDir, "market-meta", "flat-import-log.json"),
|
|
239
|
+
);
|
|
244
240
|
if (existingFile !== null) return 0;
|
|
245
241
|
|
|
246
242
|
let rows: unknown[][];
|
|
@@ -293,7 +289,7 @@ async function migrateFlatImportLog(conn: DuckDBConnection, dataDir: string): Pr
|
|
|
293
289
|
export async function migrateMetadataToJson(
|
|
294
290
|
conn: DuckDBConnection,
|
|
295
291
|
dataDir: string,
|
|
296
|
-
blocksDir: string
|
|
292
|
+
blocksDir: string,
|
|
297
293
|
): Promise<MigrationResult> {
|
|
298
294
|
// Guard: only run in Parquet mode
|
|
299
295
|
if (!isParquetMode()) {
|
|
@@ -310,7 +306,7 @@ export async function migrateMetadataToJson(
|
|
|
310
306
|
if (migrated) {
|
|
311
307
|
console.log(
|
|
312
308
|
`[json-migration] Migrated: ${profiles} profiles, ${syncMeta} sync metadata, ` +
|
|
313
|
-
|
|
309
|
+
`${marketMeta} market import records, ${flatLog} flat import records`,
|
|
314
310
|
);
|
|
315
311
|
}
|
|
316
312
|
|
package/src/db/json-store.ts
CHANGED
|
@@ -23,7 +23,9 @@ import * as path from "path";
|
|
|
23
23
|
|
|
24
24
|
/** Check for Node.js ENOENT errors */
|
|
25
25
|
function isEnoent(err: unknown): boolean {
|
|
26
|
-
return
|
|
26
|
+
return (
|
|
27
|
+
typeof err === "object" && err !== null && (err as Record<string, unknown>).code === "ENOENT"
|
|
28
|
+
);
|
|
27
29
|
}
|
|
28
30
|
|
|
29
31
|
/**
|
|
@@ -94,8 +96,8 @@ export async function listJsonFiles(dirPath: string, suffix = ".json"): Promise<
|
|
|
94
96
|
try {
|
|
95
97
|
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
|
96
98
|
return entries
|
|
97
|
-
.filter(e => e.isFile() && e.name.endsWith(suffix))
|
|
98
|
-
.map(e => path.join(dirPath, e.name))
|
|
99
|
+
.filter((e) => e.isFile() && e.name.endsWith(suffix))
|
|
100
|
+
.map((e) => path.join(dirPath, e.name))
|
|
99
101
|
.sort();
|
|
100
102
|
} catch (err: unknown) {
|
|
101
103
|
if (isEnoent(err)) {
|