tradeblocks-mcp 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-6J7OBXZX.js → chunk-62GCLMQA.js} +33 -13
- package/dist/chunk-62GCLMQA.js.map +1 -0
- package/dist/market/provenance/index.js +29 -9
- package/dist/market/provenance/index.js.map +1 -1
- package/dist/{sync-EH7J3HBY.js → sync-JL7OPLCT.js} +2 -2
- package/dist/test-exports.js +90 -37
- package/dist/test-exports.js.map +1 -1
- package/package.json +1 -1
- package/server/{chunk-N5WZQZB4.js → chunk-J4GWE6JR.js} +33 -13
- package/server/chunk-J4GWE6JR.js.map +1 -0
- package/server/index.js +93 -40
- package/server/index.js.map +1 -1
- package/server/{sync-SBFJFPOU.js → sync-ZV3QEGMO.js} +2 -2
- package/src/db/parquet-writer.ts +31 -1
- package/src/market/ingestor/market-ingestor.ts +81 -34
- package/src/market/stores/parquet-quote-store.ts +19 -2
- package/dist/chunk-6J7OBXZX.js.map +0 -1
- package/server/chunk-N5WZQZB4.js.map +0 -1
- /package/dist/{sync-EH7J3HBY.js.map → sync-JL7OPLCT.js.map} +0 -0
- /package/server/{sync-SBFJFPOU.js.map → sync-ZV3QEGMO.js.map} +0 -0
|
@@ -33,7 +33,7 @@ var require_extends = __commonJS({
|
|
|
33
33
|
});
|
|
34
34
|
|
|
35
35
|
// src/sync/index.ts
|
|
36
|
-
import { existsSync as
|
|
36
|
+
import { existsSync as existsSync3 } from "fs";
|
|
37
37
|
import * as fs12 from "fs/promises";
|
|
38
38
|
import * as path16 from "path";
|
|
39
39
|
|
|
@@ -252,7 +252,7 @@ async function ensureMarketDataTables(conn) {
|
|
|
252
252
|
|
|
253
253
|
// src/db/parquet-writer.ts
|
|
254
254
|
import { createHash as createHash3 } from "crypto";
|
|
255
|
-
import { createReadStream } from "fs";
|
|
255
|
+
import { createReadStream, existsSync } from "fs";
|
|
256
256
|
import * as fs4 from "fs/promises";
|
|
257
257
|
import * as path7 from "path";
|
|
258
258
|
|
|
@@ -2301,6 +2301,25 @@ async function exactFileFingerprint(filePath, rows) {
|
|
|
2301
2301
|
function escapeSqlLiteral2(value) {
|
|
2302
2302
|
return value.replaceAll("'", "''");
|
|
2303
2303
|
}
|
|
2304
|
+
var MIN_QUOTE_PARTITION_ROW_FRACTION = 0.5;
|
|
2305
|
+
var QuotePartitionShrinkError = class extends Error {
|
|
2306
|
+
constructor(partition, existing, candidate) {
|
|
2307
|
+
super(
|
|
2308
|
+
`Quote partition shrink refused: underlying=${partition.underlying}/date=${partition.date}; existing=${existing}, candidate=${candidate}, threshold=${MIN_QUOTE_PARTITION_ROW_FRACTION}`
|
|
2309
|
+
);
|
|
2310
|
+
this.name = "QuotePartitionShrinkError";
|
|
2311
|
+
}
|
|
2312
|
+
};
|
|
2313
|
+
async function assertQuotePartitionRowCount(conn, targetPath, provenance, candidate) {
|
|
2314
|
+
if (provenance?.dataset !== "option_quote_minutes" || !existsSync(targetPath)) return;
|
|
2315
|
+
const reader = await conn.runAndReadAll(
|
|
2316
|
+
`SELECT COUNT(*) FROM read_parquet('${escapeSqlLiteral2(targetPath)}', hive_partitioning=false)`
|
|
2317
|
+
);
|
|
2318
|
+
const existing = Number(reader.getRows()[0][0]);
|
|
2319
|
+
if (candidate < existing * MIN_QUOTE_PARTITION_ROW_FRACTION) {
|
|
2320
|
+
throw new QuotePartitionShrinkError(provenance.partition, existing, candidate);
|
|
2321
|
+
}
|
|
2322
|
+
}
|
|
2304
2323
|
async function inspectPreparedParquet(conn, preparedPath, coverage) {
|
|
2305
2324
|
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(coverage.column)) {
|
|
2306
2325
|
throw new TypeError(`Unsafe provenance coverage column: ${JSON.stringify(coverage.column)}`);
|
|
@@ -2394,6 +2413,7 @@ async function writeParquetAtomic(conn, opts) {
|
|
|
2394
2413
|
const rowCount = inspected ? inspected.rowCount : Number(
|
|
2395
2414
|
(await conn.runAndReadAll(`SELECT COUNT(*)::INTEGER AS cnt FROM "${stagingName}"`)).getRows()[0][0]
|
|
2396
2415
|
);
|
|
2416
|
+
await assertQuotePartitionRowCount(conn, targetPath, provenance, rowCount);
|
|
2397
2417
|
const logicalCoverage = inspected?.coverage;
|
|
2398
2418
|
const quality = provenanceAttempt ? resolveQuality(rowCount, provenance.quality) : void 0;
|
|
2399
2419
|
const fingerprint = provenanceAttempt ? await exactFileFingerprint(tempPath, rowCount) : void 0;
|
|
@@ -3005,7 +3025,7 @@ async function deleteProfile(conn, blockId, strategyName, baseDir) {
|
|
|
3005
3025
|
}
|
|
3006
3026
|
|
|
3007
3027
|
// src/db/market-views.ts
|
|
3008
|
-
import { existsSync, readdirSync } from "fs";
|
|
3028
|
+
import { existsSync as existsSync2, readdirSync } from "fs";
|
|
3009
3029
|
import * as path10 from "path";
|
|
3010
3030
|
|
|
3011
3031
|
// src/utils/quote-parquet-projection.ts
|
|
@@ -3136,7 +3156,7 @@ function quoteParquetCanonicalWriteProjection(columns, alias = "q") {
|
|
|
3136
3156
|
|
|
3137
3157
|
// src/db/market-views.ts
|
|
3138
3158
|
function hasParquetPartitions(dir, partitionKey = "date") {
|
|
3139
|
-
if (!
|
|
3159
|
+
if (!existsSync2(dir)) return false;
|
|
3140
3160
|
try {
|
|
3141
3161
|
const prefix = `${partitionKey}=`;
|
|
3142
3162
|
return readdirSync(dir).some((entry) => {
|
|
@@ -3162,14 +3182,14 @@ function hasParquetPartitions(dir, partitionKey = "date") {
|
|
|
3162
3182
|
}
|
|
3163
3183
|
}
|
|
3164
3184
|
function hasEnrichedTickerFiles(dir) {
|
|
3165
|
-
if (!
|
|
3185
|
+
if (!existsSync2(dir)) return false;
|
|
3166
3186
|
try {
|
|
3167
3187
|
return readdirSync(dir).some((entry) => {
|
|
3168
3188
|
if (!entry.startsWith("ticker=")) return false;
|
|
3169
3189
|
const tickerDir = path10.join(dir, entry);
|
|
3170
3190
|
try {
|
|
3171
3191
|
return readdirSync(tickerDir).some(
|
|
3172
|
-
(dateEntry) => dateEntry.startsWith("date=") &&
|
|
3192
|
+
(dateEntry) => dateEntry.startsWith("date=") && existsSync2(path10.join(tickerDir, dateEntry, "data.parquet"))
|
|
3173
3193
|
);
|
|
3174
3194
|
} catch {
|
|
3175
3195
|
return false;
|
|
@@ -3181,10 +3201,10 @@ function hasEnrichedTickerFiles(dir) {
|
|
|
3181
3201
|
}
|
|
3182
3202
|
function hasEnrichedContextFile(dir) {
|
|
3183
3203
|
const contextDir = path10.join(dir, "context");
|
|
3184
|
-
if (!
|
|
3204
|
+
if (!existsSync2(contextDir)) return false;
|
|
3185
3205
|
try {
|
|
3186
3206
|
return readdirSync(contextDir).some(
|
|
3187
|
-
(entry) => entry.startsWith("date=") &&
|
|
3207
|
+
(entry) => entry.startsWith("date=") && existsSync2(path10.join(contextDir, entry, "data.parquet"))
|
|
3188
3208
|
);
|
|
3189
3209
|
} catch {
|
|
3190
3210
|
return false;
|
|
@@ -3202,8 +3222,8 @@ async function createMarketParquetViews(conn, dataDir) {
|
|
|
3202
3222
|
];
|
|
3203
3223
|
for (const { name: name79, subdir, partitionKey } of hiveViews) {
|
|
3204
3224
|
const dirPath = subdir;
|
|
3205
|
-
const hasNewLayout =
|
|
3206
|
-
const hasLegacyLayout = partitionKey !== "date" &&
|
|
3225
|
+
const hasNewLayout = existsSync2(dirPath) && hasParquetPartitions(dirPath, partitionKey);
|
|
3226
|
+
const hasLegacyLayout = partitionKey !== "date" && existsSync2(dirPath) && hasParquetPartitions(dirPath, "date");
|
|
3207
3227
|
if (hasNewLayout || hasLegacyLayout) {
|
|
3208
3228
|
try {
|
|
3209
3229
|
await conn.run(`DROP VIEW IF EXISTS market.${name79}`);
|
|
@@ -3770,7 +3790,7 @@ async function openReadWriteConnection(dbPath, threads, memoryLimit) {
|
|
|
3770
3790
|
await createMarketParquetViews(connection, dataRoot);
|
|
3771
3791
|
await ensureMarketDataTables(connection);
|
|
3772
3792
|
try {
|
|
3773
|
-
const blocksDir = (await import("./sync-
|
|
3793
|
+
const blocksDir = (await import("./sync-JL7OPLCT.js")).getBlocksDir(dataRoot);
|
|
3774
3794
|
await migrateMetadataToJson(connection, dataRoot, blocksDir);
|
|
3775
3795
|
} catch (err) {
|
|
3776
3796
|
console.warn(
|
|
@@ -23050,7 +23070,7 @@ function setBlocksDir(dir) {
|
|
|
23050
23070
|
function getBlocksDir(baseDir) {
|
|
23051
23071
|
if (_blocksDir) return _blocksDir;
|
|
23052
23072
|
const nestedBlocksDir = path16.join(baseDir, "blocks");
|
|
23053
|
-
return
|
|
23073
|
+
return existsSync3(nestedBlocksDir) ? nestedBlocksDir : baseDir;
|
|
23054
23074
|
}
|
|
23055
23075
|
async function syncAllBlocks(baseDir) {
|
|
23056
23076
|
const conn = await getConnection(baseDir);
|
|
@@ -23253,4 +23273,4 @@ decimal.js/decimal.mjs:
|
|
|
23253
23273
|
* MIT Licence
|
|
23254
23274
|
*)
|
|
23255
23275
|
*/
|
|
23256
|
-
//# sourceMappingURL=chunk-
|
|
23276
|
+
//# sourceMappingURL=chunk-62GCLMQA.js.map
|