tradeblocks-mcp 3.0.2 → 3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -68
- package/dist/{chunk-LDKTV7GW.js → chunk-27S67XW3.js} +24 -52
- package/dist/chunk-27S67XW3.js.map +1 -0
- package/dist/{chunk-FGZH632F.js → chunk-2E63THNI.js} +13 -11
- package/dist/chunk-2E63THNI.js.map +1 -0
- package/dist/{chunk-BKQ4PM4Y.js → chunk-6S37CXUA.js} +11 -13
- package/dist/chunk-6S37CXUA.js.map +1 -0
- package/dist/{chunk-PRAYH3RT.js → chunk-AP7IHUUR.js} +29 -12
- package/dist/chunk-AP7IHUUR.js.map +1 -0
- package/dist/{chunk-QTTR7AAW.js → chunk-C6LL746C.js} +67 -84
- package/dist/{chunk-QTTR7AAW.js.map → chunk-C6LL746C.js.map} +1 -1
- package/dist/{chunk-W2PP3LEH.js → chunk-SEUZYQGQ.js} +17 -8
- package/dist/chunk-SEUZYQGQ.js.map +1 -0
- package/dist/{chunk-4BLCXNQ6.js → chunk-UBUC5A66.js} +105 -114
- package/dist/chunk-UBUC5A66.js.map +1 -0
- package/dist/{chunk-XXYOUIZY.js → chunk-VDU25Z6X.js} +63 -80
- package/dist/chunk-VDU25Z6X.js.map +1 -0
- package/dist/daily-log-processor-BY3ISY6K.js +9 -0
- package/dist/{daily-logs-store-D3QC2XPV.js → daily-logs-store-3C3OTGPK.js} +2 -2
- package/dist/iv-solver-worker.js +9 -1
- package/dist/market-provider-H3ARIVZ4.js +16 -0
- package/dist/{proto-3HZTCWK4.js → proto-V23GIZG2.js} +2 -2
- package/dist/{sync-PO4IPCYV.js → sync-EVLKB3ZJ.js} +7 -7
- package/dist/test-exports.js +358 -254
- package/dist/test-exports.js.map +1 -1
- package/dist/trade-processor-EYA3I3XB.js +9 -0
- package/dist/{trades-store-LOL3FQSF.js → trades-store-R3NL25T2.js} +2 -2
- package/manifest.json +3 -11
- package/package.json +1 -1
- package/server/{chunk-NZO6PT64.js → chunk-4P7D7YZP.js} +102 -113
- package/server/chunk-4P7D7YZP.js.map +1 -0
- package/server/{chunk-72GKJE2U.js → chunk-BOPHW5M6.js} +521 -235
- package/server/chunk-BOPHW5M6.js.map +1 -0
- package/server/{chunk-WA5AAPCH.js → chunk-GH2552SE.js} +17 -8
- package/server/chunk-GH2552SE.js.map +1 -0
- package/server/{chunk-5EBXHT6C.js → chunk-OBYKFW2B.js} +11 -13
- package/server/chunk-OBYKFW2B.js.map +1 -0
- package/server/{chunk-FBNDMCT5.js → chunk-T66KH2XH.js} +13 -11
- package/server/chunk-T66KH2XH.js.map +1 -0
- package/server/{chunk-NRFXAJF7.js → chunk-W5E7FHC4.js} +24 -52
- package/server/chunk-W5E7FHC4.js.map +1 -0
- package/server/{chunk-CSDVJPBB.js → chunk-YUCOAJ4Z.js} +29 -12
- package/server/chunk-YUCOAJ4Z.js.map +1 -0
- package/server/{chunk-PNKG7RY7.js → chunk-ZBJCF4ZG.js} +63 -80
- package/server/chunk-ZBJCF4ZG.js.map +1 -0
- package/server/{config-6IZXEFEX.js → config-DK7KOMNL.js} +1 -1
- package/server/config-DK7KOMNL.js.map +1 -0
- package/server/daily-log-processor-ENEUT22A.js +10 -0
- package/server/{daily-logs-store-NU7M4L5C.js → daily-logs-store-YFNXTB7J.js} +2 -2
- package/server/http-server.js +2 -5
- package/server/index.js +1771 -1538
- package/server/index.js.map +1 -1
- package/server/iv-solver-worker.js +9 -1
- package/server/market-provider-B437HKLW.js +17 -0
- package/server/{proto-DWRZJO4E.js → proto-YZ4TLX2N.js} +2 -2
- package/server/{sync-QFI5L7S7.js → sync-V25UQJA3.js} +7 -7
- package/server/trade-processor-L3PIQ5TG.js +10 -0
- package/server/{trades-store-VY26MWER.js → trades-store-RTTEYMXH.js} +2 -2
- package/src/auth/clients-store.ts +4 -4
- package/src/auth/code-store.ts +1 -1
- package/src/auth/config.ts +11 -12
- package/src/auth/login-page.ts +8 -10
- package/src/auth/provider.ts +35 -30
- package/src/auth/token.ts +17 -15
- package/src/db/backtest-schemas.ts +12 -4
- package/src/db/connection.ts +107 -40
- package/src/db/index.ts +37 -5
- package/src/db/json-adapters.ts +22 -31
- package/src/db/json-migration.ts +11 -15
- package/src/db/json-store.ts +5 -3
- package/src/db/market-datasets.ts +48 -15
- package/src/db/market-schemas.ts +11 -5
- package/src/db/market-views.ts +119 -32
- package/src/db/parquet-writer.ts +7 -7
- package/src/db/profile-schemas.ts +39 -22
- package/src/db/schemas.ts +2 -2
- package/src/http-server.ts +7 -12
- package/src/index.ts +8 -9
- package/src/market/ingestor/index.ts +1 -4
- package/src/market/ingestor/market-ingestor.ts +140 -67
- package/src/market/ingestor/types.ts +1 -3
- package/src/market/stores/chain-sql.ts +2 -8
- package/src/market/stores/chain-store.ts +1 -4
- package/src/market/stores/duckdb-chain-store.ts +3 -14
- package/src/market/stores/duckdb-enriched-store.ts +3 -13
- package/src/market/stores/duckdb-quote-store.ts +10 -27
- package/src/market/stores/duckdb-spot-store.ts +4 -20
- package/src/market/stores/enriched-sql.ts +3 -9
- package/src/market/stores/enriched-store.ts +2 -2
- package/src/market/stores/parquet-chain-store.ts +5 -22
- package/src/market/stores/parquet-enriched-store.ts +4 -16
- package/src/market/stores/parquet-oi-daily-store.ts +3 -14
- package/src/market/stores/parquet-quote-store.ts +9 -29
- package/src/market/stores/parquet-spot-store.ts +14 -33
- package/src/market/stores/quote-store.ts +2 -10
- package/src/market/stores/spot-sql.ts +3 -15
- package/src/market/stores/spot-store.ts +1 -1
- package/src/market/tickers/defaults.json +6 -6
- package/src/market/tickers/loader.ts +2 -7
- package/src/market/tickers/registry.ts +5 -15
- package/src/market/tickers/schemas.ts +2 -6
- package/src/models/strategy-profile.ts +39 -39
- package/src/sync/block-sync.ts +44 -65
- package/src/sync/index.ts +1 -4
- package/src/sync/metadata.ts +14 -20
- package/src/test-exports.ts +127 -123
- package/src/tools/analysis.ts +180 -159
- package/src/tools/batch-exit-analysis.ts +100 -59
- package/src/tools/blocks/analysis.ts +116 -101
- package/src/tools/blocks/comparison.ts +281 -364
- package/src/tools/blocks/core.ts +208 -253
- package/src/tools/blocks/health.ts +571 -563
- package/src/tools/blocks/similarity.ts +316 -283
- package/src/tools/edge-decay.ts +124 -197
- package/src/tools/exit-analysis.ts +130 -77
- package/src/tools/greeks-attribution.ts +84 -35
- package/src/tools/guides.ts +4 -6
- package/src/tools/imports.ts +9 -12
- package/src/tools/market-data.ts +1067 -757
- package/src/tools/market-enrichment.ts +3 -3
- package/src/tools/market-fetch.ts +148 -67
- package/src/tools/market-imports.ts +12 -12
- package/src/tools/middleware/sync-middleware.ts +5 -6
- package/src/tools/performance.ts +185 -302
- package/src/tools/profile-analysis.ts +52 -66
- package/src/tools/profiles.ts +106 -69
- package/src/tools/regime-advisor.ts +20 -45
- package/src/tools/replay.ts +81 -77
- package/src/tools/reports/discrepancies.ts +298 -328
- package/src/tools/reports/fields.ts +7 -25
- package/src/tools/reports/helpers.ts +18 -49
- package/src/tools/reports/predictive.ts +27 -70
- package/src/tools/reports/slippage-trends.ts +315 -345
- package/src/tools/reports/slippage.ts +1 -4
- package/src/tools/reports/strategy-matches.ts +399 -441
- package/src/tools/schema.ts +43 -40
- package/src/tools/shared/filters.ts +3 -9
- package/src/tools/snapshot.ts +9 -30
- package/src/tools/sql.ts +15 -14
- package/src/tools/tickers.ts +1 -4
- package/src/utils/batch-exit-analysis.ts +31 -42
- package/src/utils/black-scholes.ts +39 -29
- package/src/utils/block-loader.ts +68 -82
- package/src/utils/calibration-probe.ts +1 -2
- package/src/utils/chain-loader.ts +3 -3
- package/src/utils/csv-discovery.ts +16 -22
- package/src/utils/data-quality.ts +24 -36
- package/src/utils/exit-triggers.ts +91 -96
- package/src/utils/field-timing.ts +94 -79
- package/src/utils/filter-predicates.ts +13 -9
- package/src/utils/flatfile-importer.ts +94 -64
- package/src/utils/greeks-decomposition.ts +152 -100
- package/src/utils/iv-solver-pool.ts +55 -25
- package/src/utils/iv-solver-worker.ts +5 -5
- package/src/utils/market-enricher.ts +528 -497
- package/src/utils/market-importer.ts +31 -12
- package/src/utils/market-provider.ts +21 -23
- package/src/utils/massive-tier.ts +5 -7
- package/src/utils/migrate-option-data-helpers.ts +2 -8
- package/src/utils/option-quote-greeks.ts +25 -31
- package/src/utils/option-time.ts +4 -8
- package/src/utils/output-formatter.ts +1 -4
- package/src/utils/provider-capabilities.ts +1 -4
- package/src/utils/providers/massive.ts +59 -93
- package/src/utils/providers/thetadata/backfill.ts +14 -23
- package/src/utils/providers/thetadata/client.ts +12 -8
- package/src/utils/providers/thetadata/decode.ts +2 -20
- package/src/utils/providers/thetadata/endpoints.ts +32 -40
- package/src/utils/providers/thetadata/join.ts +11 -10
- package/src/utils/providers/thetadata/proto.ts +12 -10
- package/src/utils/providers/thetadata/quote-mid-greeks.ts +5 -5
- package/src/utils/providers/thetadata.ts +11 -10
- package/src/utils/quote-enricher.ts +4 -4
- package/src/utils/quote-parquet-projection.ts +3 -11
- package/src/utils/sample-date-selector.ts +3 -5
- package/src/utils/schema-metadata.ts +102 -70
- package/src/utils/ticker.ts +5 -9
- package/src/utils/trade-replay.ts +77 -68
- package/dist/chunk-4BLCXNQ6.js.map +0 -1
- package/dist/chunk-BKQ4PM4Y.js.map +0 -1
- package/dist/chunk-FGZH632F.js.map +0 -1
- package/dist/chunk-LDKTV7GW.js.map +0 -1
- package/dist/chunk-PRAYH3RT.js.map +0 -1
- package/dist/chunk-W2PP3LEH.js.map +0 -1
- package/dist/chunk-XXYOUIZY.js.map +0 -1
- package/dist/daily-log-processor-6MWJ23JK.js +0 -9
- package/dist/market-provider-VDRJUEF2.js +0 -16
- package/dist/trade-processor-NHU4VWRX.js +0 -9
- package/server/chunk-5EBXHT6C.js.map +0 -1
- package/server/chunk-72GKJE2U.js.map +0 -1
- package/server/chunk-CSDVJPBB.js.map +0 -1
- package/server/chunk-FBNDMCT5.js.map +0 -1
- package/server/chunk-NRFXAJF7.js.map +0 -1
- package/server/chunk-NZO6PT64.js.map +0 -1
- package/server/chunk-PNKG7RY7.js.map +0 -1
- package/server/chunk-WA5AAPCH.js.map +0 -1
- package/server/config-6IZXEFEX.js.map +0 -1
- package/server/daily-log-processor-Y3PVSVBM.js +0 -10
- package/server/market-provider-VOYYVYWT.js +0 -17
- package/server/trade-processor-JWVS37KM.js +0 -10
- /package/dist/{daily-log-processor-6MWJ23JK.js.map → daily-log-processor-BY3ISY6K.js.map} +0 -0
- /package/dist/{daily-logs-store-D3QC2XPV.js.map → daily-logs-store-3C3OTGPK.js.map} +0 -0
- /package/dist/{market-provider-VDRJUEF2.js.map → market-provider-H3ARIVZ4.js.map} +0 -0
- /package/dist/{proto-3HZTCWK4.js.map → proto-V23GIZG2.js.map} +0 -0
- /package/dist/{sync-PO4IPCYV.js.map → sync-EVLKB3ZJ.js.map} +0 -0
- /package/dist/{trade-processor-NHU4VWRX.js.map → trade-processor-EYA3I3XB.js.map} +0 -0
- /package/dist/{trades-store-LOL3FQSF.js.map → trades-store-R3NL25T2.js.map} +0 -0
- /package/server/{daily-log-processor-Y3PVSVBM.js.map → daily-log-processor-ENEUT22A.js.map} +0 -0
- /package/server/{daily-logs-store-NU7M4L5C.js.map → daily-logs-store-YFNXTB7J.js.map} +0 -0
- /package/server/{market-provider-VOYYVYWT.js.map → market-provider-B437HKLW.js.map} +0 -0
- /package/server/{proto-DWRZJO4E.js.map → proto-YZ4TLX2N.js.map} +0 -0
- /package/server/{sync-QFI5L7S7.js.map → sync-V25UQJA3.js.map} +0 -0
- /package/server/{trade-processor-JWVS37KM.js.map → trade-processor-L3PIQ5TG.js.map} +0 -0
- /package/server/{trades-store-VY26MWER.js.map → trades-store-RTTEYMXH.js.map} +0 -0
|
@@ -3,14 +3,14 @@ import {
|
|
|
3
3
|
dailyLogEntrySchema,
|
|
4
4
|
findMissingHeaders,
|
|
5
5
|
rawDailyLogDataSchema
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-6S37CXUA.js";
|
|
7
7
|
|
|
8
8
|
// ../lib/models/daily-log.ts
|
|
9
9
|
var DAILY_LOG_COLUMN_MAPPING = {
|
|
10
|
-
|
|
10
|
+
Date: "date",
|
|
11
11
|
"Net Liquidity": "netLiquidity",
|
|
12
12
|
"Current Funds": "currentFunds",
|
|
13
|
-
|
|
13
|
+
Withdrawn: "withdrawn",
|
|
14
14
|
"Trading Funds": "tradingFunds",
|
|
15
15
|
"P/L": "dailyPl",
|
|
16
16
|
"P/L %": "dailyPlPct",
|
|
@@ -210,7 +210,10 @@ var DailyLogProcessor = class {
|
|
|
210
210
|
rawDailyLogDataSchema.parse(normalizedRow);
|
|
211
211
|
return normalizedRow;
|
|
212
212
|
} catch (error) {
|
|
213
|
-
console.warn(
|
|
213
|
+
console.warn(
|
|
214
|
+
`[DailyLogProcessor] Row ${rowIndex + 2} validation failed:`,
|
|
215
|
+
error instanceof Error ? error.message : error
|
|
216
|
+
);
|
|
214
217
|
return null;
|
|
215
218
|
}
|
|
216
219
|
}
|
|
@@ -276,7 +279,9 @@ var DailyLogProcessor = class {
|
|
|
276
279
|
const validatedEntry = dailyLogEntrySchema.parse(entry);
|
|
277
280
|
return validatedEntry;
|
|
278
281
|
} catch (error) {
|
|
279
|
-
throw new Error(
|
|
282
|
+
throw new Error(
|
|
283
|
+
`Daily log entry conversion failed: ${error instanceof Error ? error.message : String(error)}`
|
|
284
|
+
);
|
|
280
285
|
}
|
|
281
286
|
}
|
|
282
287
|
/**
|
|
@@ -293,13 +298,17 @@ var DailyLogProcessor = class {
|
|
|
293
298
|
static validateDataConsistency(entries) {
|
|
294
299
|
const warnings = [];
|
|
295
300
|
if (entries.length === 0) return warnings;
|
|
296
|
-
const sortedEntries = [...entries].sort(
|
|
301
|
+
const sortedEntries = [...entries].sort(
|
|
302
|
+
(a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()
|
|
303
|
+
);
|
|
297
304
|
for (let i = 1; i < sortedEntries.length; i++) {
|
|
298
305
|
const prevDate = new Date(sortedEntries[i - 1].date);
|
|
299
306
|
const currentDate = new Date(sortedEntries[i].date);
|
|
300
307
|
const daysDiff = (currentDate.getTime() - prevDate.getTime()) / (1e3 * 60 * 60 * 24);
|
|
301
308
|
if (daysDiff > 7) {
|
|
302
|
-
warnings.push(
|
|
309
|
+
warnings.push(
|
|
310
|
+
`Large date gap detected: ${daysDiff.toFixed(0)} days between ${prevDate.toISOString().split("T")[0]} and ${currentDate.toISOString().split("T")[0]}`
|
|
311
|
+
);
|
|
303
312
|
}
|
|
304
313
|
}
|
|
305
314
|
const negativeEntries = sortedEntries.filter((entry) => entry.netLiquidity < 0);
|
|
@@ -317,4 +326,4 @@ var DailyLogProcessor = class {
|
|
|
317
326
|
export {
|
|
318
327
|
DailyLogProcessor
|
|
319
328
|
};
|
|
320
|
-
//# sourceMappingURL=chunk-
|
|
329
|
+
//# sourceMappingURL=chunk-SEUZYQGQ.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../lib/models/daily-log.ts","../../lib/processing/daily-log-processor.ts"],"sourcesContent":["/**\n * Daily log model based on legacy Python DailyLogEntry class\n * Represents daily portfolio performance data from OptionOmega\n */\nexport interface DailyLogEntry {\n date: Date;\n netLiquidity: number;\n currentFunds: number;\n withdrawn: number;\n tradingFunds: number;\n dailyPl: number; // P/L for the day\n dailyPlPct: number; // P/L percentage\n drawdownPct: number; // Drawdown percentage\n blockId?: string; // Optional block ID for linking to trades\n\n /**\n * Custom fields from extra columns in the daily log CSV\n * Keys are the original column names, values are auto-detected as number or string\n * These fields can be joined to trades by date for analysis (e.g., dayOpenVix, spyOpen)\n */\n customFields?: Record<string, number | string>;\n}\n\n/**\n * Raw daily log data as it comes from CSV before processing\n */\nexport interface RawDailyLogData {\n Date: string;\n \"Net Liquidity\": string;\n \"Current Funds\": string;\n Withdrawn: string;\n \"Trading Funds\": string;\n \"P/L\": string;\n \"P/L %\": string;\n \"Drawdown %\": string;\n}\n\n/**\n * Processed daily log collection with metadata\n */\nexport interface DailyLog {\n entries: DailyLogEntry[];\n uploadTimestamp: Date;\n filename: string;\n totalEntries: number;\n dateRangeStart: Date;\n dateRangeEnd: Date;\n finalPortfolioValue: number;\n maxDrawdown: number;\n}\n\n/**\n * Column mapping from CSV headers to DailyLogEntry interface properties\n */\nexport const DAILY_LOG_COLUMN_MAPPING = {\n Date: \"date\",\n \"Net Liquidity\": \"netLiquidity\",\n \"Current Funds\": \"currentFunds\",\n Withdrawn: \"withdrawn\",\n \"Trading Funds\": \"tradingFunds\",\n \"P/L\": \"dailyPl\",\n \"P/L %\": \"dailyPlPct\",\n \"Drawdown %\": \"drawdownPct\",\n} as const;\n\n/**\n * Required columns for daily log processing\n */\nexport const REQUIRED_DAILY_LOG_COLUMNS = [\n \"Date\",\n \"Net Liquidity\",\n \"Current Funds\",\n \"Trading Funds\",\n \"P/L\",\n \"P/L %\",\n \"Drawdown %\",\n] as const;\n","/**\n * Daily Log Processor\n *\n * Handles parsing and processing of daily log CSV files from OptionOmega.\n * Converts raw CSV data to validated DailyLogEntry objects.\n */\n\nimport {\n type DailyLogEntry,\n REQUIRED_DAILY_LOG_COLUMNS,\n DAILY_LOG_COLUMN_MAPPING,\n} from \"../models/daily-log.ts\";\n\n/**\n * Set of known daily log column names (canonical names from DAILY_LOG_COLUMN_MAPPING)\n * Used to identify custom columns that should be preserved\n */\nconst KNOWN_DAILY_LOG_COLUMNS = new Set([\n ...Object.keys(DAILY_LOG_COLUMN_MAPPING),\n \"Withdrawn\", // Optional column that may not be in REQUIRED but is known\n]);\nimport type { ValidationError, ProcessingError } from \"../models/index.ts\";\nimport { rawDailyLogDataSchema, dailyLogEntrySchema } from \"../models/validators.ts\";\nimport { CSVParser, type ParseProgress } from \"./csv-parser.ts\";\nimport { findMissingHeaders } from \"../utils/csv-headers.ts\";\n// import { CSVParseResult } from './csv-parser.ts'\n\n/**\n * Daily log processing configuration\n */\nexport interface DailyLogProcessingConfig {\n maxEntries?: number;\n strictValidation?: boolean;\n progressCallback?: (progress: DailyLogProcessingProgress) => void;\n}\n\n/**\n * Daily log processing progress\n */\nexport interface DailyLogProcessingProgress extends ParseProgress {\n stage: \"reading\" | \"parsing\" | \"validating\" | \"converting\" | \"completed\";\n validEntries: number;\n invalidEntries: number;\n}\n\n/**\n * Daily log processing result\n */\nexport interface DailyLogProcessingResult {\n entries: DailyLogEntry[];\n totalRows: number;\n validEntries: number;\n invalidEntries: number;\n errors: ProcessingError[];\n warnings: string[];\n stats: {\n processingTimeMs: number;\n dateRange: { start: Date | null; end: Date | null };\n finalPortfolioValue: number;\n maxDrawdown: number;\n totalPL: number;\n };\n}\n\n/**\n * Daily log processor class\n */\nexport class DailyLogProcessor {\n private config: Required<DailyLogProcessingConfig>;\n\n constructor(config: DailyLogProcessingConfig = {}) {\n this.config = {\n maxEntries: 10000, // Reasonable limit for daily entries\n strictValidation: false,\n progressCallback: () => {},\n ...config,\n };\n }\n\n /**\n * Process daily log file\n */\n async processFile(file: File, blockId?: string): Promise<DailyLogProcessingResult> {\n const startTime = Date.now();\n const errors: ProcessingError[] = [];\n const warnings: string[] = [];\n\n try {\n // Validate file\n const fileValidation = CSVParser.validateCSVFile(file);\n if (!fileValidation.valid) {\n throw new Error(fileValidation.error);\n }\n\n // Configure CSV parser\n const csvParser = new CSVParser({\n maxRows: this.config.maxEntries,\n progressCallback: (progress, rowsProcessed) => {\n this.config.progressCallback({\n stage: \"parsing\",\n progress,\n rowsProcessed,\n totalRows: 0,\n errors: 0,\n validEntries: 0,\n invalidEntries: 0,\n });\n },\n });\n\n // Parse CSV with validation\n const parseResult = await csvParser.parseFileObject(\n file,\n (row, rowIndex) => this.validateRawDailyLogData(row, rowIndex),\n (progress) => {\n this.config.progressCallback({\n ...progress,\n validEntries: 0,\n invalidEntries: 0,\n });\n },\n );\n\n // Collect parsing errors\n errors.push(...parseResult.errors);\n warnings.push(...parseResult.warnings);\n\n // Check for required columns\n const missingColumns = findMissingHeaders(parseResult.headers, REQUIRED_DAILY_LOG_COLUMNS);\n if (missingColumns.length > 0) {\n throw new Error(`Missing required columns: ${missingColumns.join(\", \")}`);\n }\n\n // Update progress for conversion stage\n this.config.progressCallback({\n stage: \"converting\",\n progress: 0,\n rowsProcessed: 0,\n totalRows: parseResult.data.length,\n errors: errors.length,\n validEntries: 0,\n invalidEntries: 0,\n });\n\n // Convert validated data to DailyLogEntry objects\n const entries: DailyLogEntry[] = [];\n let validEntries = 0;\n let invalidEntries = 0;\n\n for (let i = 0; i < parseResult.data.length; i++) {\n try {\n const entry = this.convertToDailyLogEntry(parseResult.data[i], blockId);\n entries.push(entry);\n validEntries++;\n } catch (error) {\n invalidEntries++;\n const errorMessage = `Daily log entry conversion failed at row ${i + 2}: ${error instanceof Error ? error.message : String(error)}`;\n\n // Log conversion errors to console for debugging\n console.warn(`[DailyLogProcessor] ${errorMessage}`);\n\n const validationError: ValidationError = {\n type: \"validation\",\n message: errorMessage,\n details: { row: parseResult.data[i], rowIndex: i + 2 },\n field: \"unknown\",\n value: parseResult.data[i],\n expected: \"Valid daily log entry data\",\n };\n errors.push(validationError);\n\n if (!this.config.strictValidation) {\n continue; // Skip invalid row in non-strict mode\n } else {\n throw error; // Fail fast in strict mode\n }\n }\n\n // Update progress\n if (i % 50 === 0 || i === parseResult.data.length - 1) {\n const progress = Math.round((i / parseResult.data.length) * 100);\n this.config.progressCallback({\n stage: \"converting\",\n progress,\n rowsProcessed: i + 1,\n totalRows: parseResult.data.length,\n errors: errors.length,\n validEntries,\n invalidEntries,\n });\n }\n }\n\n // Sort entries by date\n entries.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime());\n\n // Calculate statistics\n const processingTimeMs = Date.now() - startTime;\n const dates = entries.map((e) => new Date(e.date));\n const dateRange = {\n start: dates.length > 0 ? new Date(Math.min(...dates.map((d) => d.getTime()))) : null,\n end: dates.length > 0 ? new Date(Math.max(...dates.map((d) => d.getTime()))) : null,\n };\n const finalPortfolioValue = entries.length > 0 ? entries[entries.length - 1].netLiquidity : 0;\n const maxDrawdown = entries.length > 0 ? Math.min(...entries.map((e) => e.drawdownPct)) : 0;\n const totalPL = entries.reduce((sum, e) => sum + e.dailyPl, 0);\n\n // Final progress update\n this.config.progressCallback({\n stage: \"completed\",\n progress: 100,\n rowsProcessed: parseResult.data.length,\n totalRows: parseResult.data.length,\n errors: errors.length,\n validEntries,\n invalidEntries,\n });\n\n return {\n entries,\n totalRows: parseResult.totalRows,\n validEntries,\n invalidEntries,\n errors,\n warnings,\n stats: {\n processingTimeMs,\n dateRange,\n finalPortfolioValue,\n maxDrawdown,\n totalPL,\n },\n };\n } catch (error) {\n const processingError: ProcessingError = {\n type: \"parsing\",\n message: `Daily log processing failed: ${error instanceof Error ? error.message : String(error)}`,\n details: { fileName: file.name, fileSize: file.size },\n };\n\n return {\n entries: [],\n totalRows: 0,\n validEntries: 0,\n invalidEntries: 0,\n errors: [processingError, ...errors],\n warnings,\n stats: {\n processingTimeMs: Date.now() - startTime,\n dateRange: { start: null, end: null },\n finalPortfolioValue: 0,\n maxDrawdown: 0,\n totalPL: 0,\n },\n };\n }\n }\n\n /**\n * Validate raw daily log data from CSV\n */\n private validateRawDailyLogData(\n row: Record<string, string>,\n rowIndex: number,\n ): Record<string, string> | null {\n try {\n // Set default values for missing optional fields\n const normalizedRow = { ...row };\n if (!normalizedRow[\"Withdrawn\"]) {\n normalizedRow[\"Withdrawn\"] = \"0\";\n }\n\n // Ensure required columns have values\n for (const field of REQUIRED_DAILY_LOG_COLUMNS) {\n if (!normalizedRow[field] || normalizedRow[field].trim() === \"\") {\n throw new Error(`Missing required field: ${field}`);\n }\n }\n\n // Basic format validation (detailed validation happens in conversion)\n rawDailyLogDataSchema.parse(normalizedRow);\n\n return normalizedRow;\n } catch (error) {\n // Log validation errors to console for debugging\n console.warn(\n `[DailyLogProcessor] Row ${rowIndex + 2} validation failed:`,\n error instanceof Error ? error.message : error,\n );\n // Return null for invalid rows - they'll be counted as invalid\n return null;\n }\n }\n\n /**\n * Parse a YYYY-MM-DD date string preserving the calendar date.\n * Same approach as trade-processor.ts for consistency.\n */\n private parseDatePreservingCalendarDay(dateStr: string): Date {\n const match = dateStr.match(/^(\\d{4})-(\\d{2})-(\\d{2})$/);\n if (match) {\n const [, year, month, day] = match;\n // Create date at midnight local time - this preserves the calendar date\n return new Date(parseInt(year), parseInt(month) - 1, parseInt(day));\n }\n // Fall back to default parsing for other formats\n return new Date(dateStr);\n }\n\n /**\n * Convert validated CSV row to DailyLogEntry object\n */\n private convertToDailyLogEntry(rawData: Record<string, string>, blockId?: string): DailyLogEntry {\n try {\n // Parse date preserving calendar day (same as trade processor)\n const date = this.parseDatePreservingCalendarDay(rawData[\"Date\"]);\n if (isNaN(date.getTime())) {\n throw new Error(`Invalid Date: ${rawData[\"Date\"]}`);\n }\n\n // Parse numeric values with error handling\n const parseNumber = (\n value: string | undefined,\n fieldName: string,\n defaultValue?: number,\n ): number => {\n if (!value || value.trim() === \"\") {\n if (defaultValue !== undefined) return defaultValue;\n throw new Error(`Missing required numeric field: ${fieldName}`);\n }\n\n // Remove currency symbols, commas, and percentage signs\n const cleaned = value.replace(/[$,%]/g, \"\").trim();\n const parsed = parseFloat(cleaned);\n\n if (isNaN(parsed)) {\n throw new Error(`Invalid numeric value for ${fieldName}: ${value}`);\n }\n\n return parsed;\n };\n\n // Build daily log entry object\n const entry: DailyLogEntry = {\n date,\n netLiquidity: parseNumber(rawData[\"Net Liquidity\"], \"Net Liquidity\"),\n currentFunds: parseNumber(rawData[\"Current Funds\"], \"Current Funds\"),\n withdrawn: parseNumber(rawData[\"Withdrawn\"], \"Withdrawn\", 0),\n tradingFunds: parseNumber(rawData[\"Trading Funds\"], \"Trading Funds\"),\n dailyPl: parseNumber(rawData[\"P/L\"], \"P/L\"),\n dailyPlPct: parseNumber(rawData[\"P/L %\"], \"P/L %\"),\n drawdownPct: parseNumber(rawData[\"Drawdown %\"], \"Drawdown %\"),\n blockId,\n };\n\n // Keep percentage values as they are from CSV to match legacy behavior\n // Legacy Python expects percentage values (e.g., -5.55), not decimals (e.g., -0.0555)\n\n // Extract custom fields (columns not in KNOWN_DAILY_LOG_COLUMNS)\n const customFields: Record<string, number | string> = {};\n for (const [key, value] of Object.entries(rawData)) {\n if (!KNOWN_DAILY_LOG_COLUMNS.has(key) && value !== undefined && value.trim() !== \"\") {\n // Auto-detect type: try to parse as number\n const cleaned = value.replace(/[$,%]/g, \"\").trim();\n const parsed = parseFloat(cleaned);\n if (!isNaN(parsed) && isFinite(parsed)) {\n customFields[key] = parsed;\n } else {\n customFields[key] = value.trim();\n }\n }\n }\n\n // Only add customFields if there are any\n if (Object.keys(customFields).length > 0) {\n entry.customFields = customFields;\n }\n\n // Final validation with Zod schema\n const validatedEntry = dailyLogEntrySchema.parse(entry);\n return validatedEntry;\n } catch (error) {\n throw new Error(\n `Daily log entry conversion failed: ${error instanceof Error ? error.message : String(error)}`,\n );\n }\n }\n\n /**\n * Process CSV content directly (for testing)\n */\n async processCSVContent(content: string, blockId?: string): Promise<DailyLogProcessingResult> {\n // Create a mock File object for testing\n const blob = new Blob([content], { type: \"text/csv\" });\n const file = new File([blob], \"test-daily.csv\", { type: \"text/csv\" });\n return this.processFile(file, blockId);\n }\n\n /**\n * Validate daily log data consistency\n */\n static validateDataConsistency(entries: DailyLogEntry[]): string[] {\n const warnings: string[] = [];\n\n if (entries.length === 0) return warnings;\n\n // Sort by date for chronological validation\n const sortedEntries = [...entries].sort(\n (a, b) => new Date(a.date).getTime() - new Date(b.date).getTime(),\n );\n\n // Check for gaps in dates (more than 7 days)\n for (let i = 1; i < sortedEntries.length; i++) {\n const prevDate = new Date(sortedEntries[i - 1].date);\n const currentDate = new Date(sortedEntries[i].date);\n const daysDiff = (currentDate.getTime() - prevDate.getTime()) / (1000 * 60 * 60 * 24);\n\n if (daysDiff > 7) {\n warnings.push(\n `Large date gap detected: ${daysDiff.toFixed(0)} days between ${prevDate.toISOString().split(\"T\")[0]} and ${currentDate.toISOString().split(\"T\")[0]}`,\n );\n }\n }\n\n // Check for negative net liquidity\n const negativeEntries = sortedEntries.filter((entry) => entry.netLiquidity < 0);\n if (negativeEntries.length > 0) {\n warnings.push(`${negativeEntries.length} entries have negative net liquidity`);\n }\n\n // Check for extreme drawdowns (> 50%)\n const extremeDrawdowns = sortedEntries.filter((entry) => entry.drawdownPct < -0.5);\n if (extremeDrawdowns.length > 0) {\n warnings.push(`${extremeDrawdowns.length} entries have extreme drawdowns (> 50%)`);\n }\n\n return warnings;\n }\n}\n"],"mappings":";;;;;;;;AAsDO,IAAM,2BAA2B;AAAA,EACtC,MAAM;AAAA,EACN,iBAAiB;AAAA,EACjB,iBAAiB;AAAA,EACjB,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,OAAO;AAAA,EACP,SAAS;AAAA,EACT,cAAc;AAChB;AAKO,IAAM,6BAA6B;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;;;AC3DA,IAAM,0BAA0B,oBAAI,IAAI;AAAA,EACtC,GAAG,OAAO,KAAK,wBAAwB;AAAA,EACvC;AAAA;AACF,CAAC;AA+CM,IAAM,oBAAN,MAAwB;AAAA,EACrB;AAAA,EAER,YAAY,SAAmC,CAAC,GAAG;AACjD,SAAK,SAAS;AAAA,MACZ,YAAY;AAAA;AAAA,MACZ,kBAAkB;AAAA,MAClB,kBAAkB,MAAM;AAAA,MAAC;AAAA,MACzB,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,YAAY,MAAY,SAAqD;AACjF,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,SAA4B,CAAC;AACnC,UAAM,WAAqB,CAAC;AAE5B,QAAI;AAEF,YAAM,iBAAiB,UAAU,gBAAgB,IAAI;AACrD,UAAI,CAAC,eAAe,OAAO;AACzB,cAAM,IAAI,MAAM,eAAe,KAAK;AAAA,MACtC;AAGA,YAAM,YAAY,IAAI,UAAU;AAAA,QAC9B,SAAS,KAAK,OAAO;AAAA,QACrB,kBAAkB,CAAC,UAAU,kBAAkB;AAC7C,eAAK,OAAO,iBAAiB;AAAA,YAC3B,OAAO;AAAA,YACP;AAAA,YACA;AAAA,YACA,WAAW;AAAA,YACX,QAAQ;AAAA,YACR,cAAc;AAAA,YACd,gBAAgB;AAAA,UAClB,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AAGD,YAAM,cAAc,MAAM,UAAU;AAAA,QAClC;AAAA,QACA,CAAC,KAAK,aAAa,KAAK,wBAAwB,KAAK,QAAQ;AAAA,QAC7D,CAAC,aAAa;AACZ,eAAK,OAAO,iBAAiB;AAAA,YAC3B,GAAG;AAAA,YACH,cAAc;AAAA,YACd,gBAAgB;AAAA,UAClB,CAAC;AAAA,QACH;AAAA,MACF;AAGA,aAAO,KAAK,GAAG,YAAY,MAAM;AACjC,eAAS,KAAK,GAAG,YAAY,QAAQ;AAGrC,YAAM,iBAAiB,mBAAmB,YAAY,SAAS,0BAA0B;AACzF,UAAI,eAAe,SAAS,GAAG;AAC7B,cAAM,IAAI,MAAM,6BAA6B,eAAe,KAAK,IAAI,CAAC,EAAE;AAAA,MAC1E;AAGA,WAAK,OAAO,iBAAiB;AAAA,QAC3B,OAAO;AAAA,QACP,UAAU;AAAA,QACV,eAAe;AAAA,QACf,WAAW,YAAY,KAAK;AAAA,QAC5B,QAAQ,OAAO;AAAA,QACf,cAAc;AAAA,QACd,gBAAgB;AAAA,MAClB,CAAC;AAGD,YAAM,UAA2B,CAAC;AAClC,UAAI,eAAe;AACnB,UAAI,iBAAiB;AAErB,eAAS,IAAI,GAAG,IAAI,YAAY,KAAK,QAAQ,KAAK;AAChD,YAAI;AACF,gBAAM,QAAQ,KAAK,uBAAuB,YAAY,KAAK,CAAC,GAAG,OAAO;AACtE,kBAAQ,KAAK,KAAK;AAClB;AAAA,QACF,SAAS,OAAO;AACd;AACA,gBAAM,eAAe,4CAA4C,IAAI,CAAC,KAAK,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AAGjI,kBAAQ,KAAK,uBAAuB,YAAY,EAAE;AAElD,gBAAM,kBAAmC;AAAA,YACvC,MAAM;AAAA,YACN,SAAS;AAAA,YACT,SAAS,EAAE,KAAK,YAAY,KAAK,CAAC,GAAG,UAAU,IAAI,EAAE;AAAA,YACrD,OAAO;AAAA,YACP,OAAO,YAAY,KAAK,CAAC;AAAA,YACzB,UAAU;AAAA,UACZ;AACA,iBAAO,KAAK,eAAe;AAE3B,cAAI,CAAC,KAAK,OAAO,kBAAkB;AACjC;AAAA,UACF,OAAO;AACL,kBAAM;AAAA,UACR;AAAA,QACF;AAGA,YAAI,IAAI,OAAO,KAAK,MAAM,YAAY,KAAK,SAAS,GAAG;AACrD,gBAAM,WAAW,KAAK,MAAO,IAAI,YAAY,KAAK,SAAU,GAAG;AAC/D,eAAK,OAAO,iBAAiB;AAAA,YAC3B,OAAO;AAAA,YACP;AAAA,YACA,eAAe,IAAI;AAAA,YACnB,WAAW,YAAY,KAAK;AAAA,YAC5B,QAAQ,OAAO;AAAA,YACf;AAAA,YACA;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAGA,cAAQ,KAAK,CAAC,GAAG,MAAM,IAAI,KAAK,EAAE,IAAI,EAAE,QAAQ,IAAI,IAAI,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC;AAG9E,YAAM,mBAAmB,KAAK,IAAI,IAAI;AACtC,YAAM,QAAQ,QAAQ,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,IAAI,CAAC;AACjD,YAAM,YAAY;AAAA,QAChB,OAAO,MAAM,SAAS,IAAI,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI;AAAA,QACjF,KAAK,MAAM,SAAS,IAAI,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI;AAAA,MACjF;AACA,YAAM,sBAAsB,QAAQ,SAAS,IAAI,QAAQ,QAAQ,SAAS,CAAC,EAAE,eAAe;AAC5F,YAAM,cAAc,QAAQ,SAAS,IAAI,KAAK,IAAI,GAAG,QAAQ,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI;AAC1F,YAAM,UAAU,QAAQ,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,SAAS,CAAC;AAG7D,WAAK,OAAO,iBAAiB;AAAA,QAC3B,OAAO;AAAA,QACP,UAAU;AAAA,QACV,eAAe,YAAY,KAAK;AAAA,QAChC,WAAW,YAAY,KAAK;AAAA,QAC5B,QAAQ,OAAO;AAAA,QACf;AAAA,QACA;AAAA,MACF,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,WAAW,YAAY;AAAA,QACvB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,OAAO;AAAA,UACL;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,YAAM,kBAAmC;AAAA,QACvC,MAAM;AAAA,QACN,SAAS,gCAAgC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AAAA,QAC/F,SAAS,EAAE,UAAU,KAAK,MAAM,UAAU,KAAK,KAAK;AAAA,MACtD;AAEA,aAAO;AAAA,QACL,SAAS,CAAC;AAAA,QACV,WAAW;AAAA,QACX,cAAc;AAAA,QACd,gBAAgB;AAAA,QAChB,QAAQ,CAAC,iBAAiB,GAAG,MAAM;AAAA,QACnC;AAAA,QACA,OAAO;AAAA,UACL,kBAAkB,KAAK,IAAI,IAAI;AAAA,UAC/B,WAAW,EAAE,OAAO,MAAM,KAAK,KAAK;AAAA,UACpC,qBAAqB;AAAA,UACrB,aAAa;AAAA,UACb,SAAS;AAAA,QACX;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,wBACN,KACA,UAC+B;AAC/B,QAAI;AAEF,YAAM,gBAAgB,EAAE,GAAG,IAAI;AAC/B,UAAI,CAAC,cAAc,WAAW,GAAG;AAC/B,sBAAc,WAAW,IAAI;AAAA,MAC/B;AAGA,iBAAW,SAAS,4BAA4B;AAC9C,YAAI,CAAC,cAAc,KAAK,KAAK,cAAc,KAAK,EAAE,KAAK,MAAM,IAAI;AAC/D,gBAAM,IAAI,MAAM,2BAA2B,KAAK,EAAE;AAAA,QACpD;AAAA,MACF;AAGA,4BAAsB,MAAM,aAAa;AAEzC,aAAO;AAAA,IACT,SAAS,OAAO;AAEd,cAAQ;AAAA,QACN,2BAA2B,WAAW,CAAC;AAAA,QACvC,iBAAiB,QAAQ,MAAM,UAAU;AAAA,MAC3C;AAEA,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,+BAA+B,SAAuB;AAC5D,UAAM,QAAQ,QAAQ,MAAM,2BAA2B;AACvD,QAAI,OAAO;AACT,YAAM,CAAC,EAAE,MAAM,OAAO,GAAG,IAAI;AAE7B,aAAO,IAAI,KAAK,SAAS,IAAI,GAAG,SAAS,KAAK,IAAI,GAAG,SAAS,GAAG,CAAC;AAAA,IACpE;AAEA,WAAO,IAAI,KAAK,OAAO;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA,EAKQ,uBAAuB,SAAiC,SAAiC;AAC/F,QAAI;AAEF,YAAM,OAAO,KAAK,+BAA+B,QAAQ,MAAM,CAAC;AAChE,UAAI,MAAM,KAAK,QAAQ,CAAC,GAAG;AACzB,cAAM,IAAI,MAAM,iBAAiB,QAAQ,MAAM,CAAC,EAAE;AAAA,MACpD;AAGA,YAAM,cAAc,CAClB,OACA,WACA,iBACW;AACX,YAAI,CAAC,SAAS,MAAM,KAAK,MAAM,IAAI;AACjC,cAAI,iBAAiB,OAAW,QAAO;AACvC,gBAAM,IAAI,MAAM,mCAAmC,SAAS,EAAE;AAAA,QAChE;AAGA,cAAM,UAAU,MAAM,QAAQ,UAAU,EAAE,EAAE,KAAK;AACjD,cAAM,SAAS,WAAW,OAAO;AAEjC,YAAI,MAAM,MAAM,GAAG;AACjB,gBAAM,IAAI,MAAM,6BAA6B,SAAS,KAAK,KAAK,EAAE;AAAA,QACpE;AAEA,eAAO;AAAA,MACT;AAGA,YAAM,QAAuB;AAAA,QAC3B;AAAA,QACA,cAAc,YAAY,QAAQ,eAAe,GAAG,eAAe;AAAA,QACnE,cAAc,YAAY,QAAQ,eAAe,GAAG,eAAe;AAAA,QACnE,WAAW,YAAY,QAAQ,WAAW,GAAG,aAAa,CAAC;AAAA,QAC3D,cAAc,YAAY,QAAQ,eAAe,GAAG,eAAe;AAAA,QACnE,SAAS,YAAY,QAAQ,KAAK,GAAG,KAAK;AAAA,QAC1C,YAAY,YAAY,QAAQ,OAAO,GAAG,OAAO;AAAA,QACjD,aAAa,YAAY,QAAQ,YAAY,GAAG,YAAY;AAAA,QAC5D;AAAA,MACF;AAMA,YAAM,eAAgD,CAAC;AACvD,iBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,OAAO,GAAG;AAClD,YAAI,CAAC,wBAAwB,IAAI,GAAG,KAAK,UAAU,UAAa,MAAM,KAAK,MAAM,IAAI;AAEnF,gBAAM,UAAU,MAAM,QAAQ,UAAU,EAAE,EAAE,KAAK;AACjD,gBAAM,SAAS,WAAW,OAAO;AACjC,cAAI,CAAC,MAAM,MAAM,KAAK,SAAS,MAAM,GAAG;AACtC,yBAAa,GAAG,IAAI;AAAA,UACtB,OAAO;AACL,yBAAa,GAAG,IAAI,MAAM,KAAK;AAAA,UACjC;AAAA,QACF;AAAA,MACF;AAGA,UAAI,OAAO,KAAK,YAAY,EAAE,SAAS,GAAG;AACxC,cAAM,eAAe;AAAA,MACvB;AAGA,YAAM,iBAAiB,oBAAoB,MAAM,KAAK;AACtD,aAAO;AAAA,IACT,SAAS,OAAO;AACd,YAAM,IAAI;AAAA,QACR,sCAAsC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AAAA,MAC9F;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,kBAAkB,SAAiB,SAAqD;AAE5F,UAAM,OAAO,IAAI,KAAK,CAAC,OAAO,GAAG,EAAE,MAAM,WAAW,CAAC;AACrD,UAAM,OAAO,IAAI,KAAK,CAAC,IAAI,GAAG,kBAAkB,EAAE,MAAM,WAAW,CAAC;AACpE,WAAO,KAAK,YAAY,MAAM,OAAO;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA,EAKA,OAAO,wBAAwB,SAAoC;AACjE,UAAM,WAAqB,CAAC;AAE5B,QAAI,QAAQ,WAAW,EAAG,QAAO;AAGjC,UAAM,gBAAgB,CAAC,GAAG,OAAO,EAAE;AAAA,MACjC,CAAC,GAAG,MAAM,IAAI,KAAK,EAAE,IAAI,EAAE,QAAQ,IAAI,IAAI,KAAK,EAAE,IAAI,EAAE,QAAQ;AAAA,IAClE;AAGA,aAAS,IAAI,GAAG,IAAI,cAAc,QAAQ,KAAK;AAC7C,YAAM,WAAW,IAAI,KAAK,cAAc,IAAI,CAAC,EAAE,IAAI;AACnD,YAAM,cAAc,IAAI,KAAK,cAAc,CAAC,EAAE,IAAI;AAClD,YAAM,YAAY,YAAY,QAAQ,IAAI,SAAS,QAAQ,MAAM,MAAO,KAAK,KAAK;AAElF,UAAI,WAAW,GAAG;AAChB,iBAAS;AAAA,UACP,4BAA4B,SAAS,QAAQ,CAAC,CAAC,iBAAiB,SAAS,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC,CAAC,QAAQ,YAAY,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC,CAAC;AAAA,QACrJ;AAAA,MACF;AAAA,IACF;AAGA,UAAM,kBAAkB,cAAc,OAAO,CAAC,UAAU,MAAM,eAAe,CAAC;AAC9E,QAAI,gBAAgB,SAAS,GAAG;AAC9B,eAAS,KAAK,GAAG,gBAAgB,MAAM,sCAAsC;AAAA,IAC/E;AAGA,UAAM,mBAAmB,cAAc,OAAO,CAAC,UAAU,MAAM,cAAc,IAAI;AACjF,QAAI,iBAAiB,SAAS,GAAG;AAC/B,eAAS,KAAK,GAAG,iBAAiB,MAAM,yCAAyC;AAAA,IACnF;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}
|
|
@@ -2,7 +2,7 @@ import {
|
|
|
2
2
|
REPORTING_TRADE_COLUMN_ALIASES,
|
|
3
3
|
convertTatRowToReportingTrade,
|
|
4
4
|
isTatFormat
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-C6LL746C.js";
|
|
6
6
|
|
|
7
7
|
// src/sync/index.ts
|
|
8
8
|
import { existsSync as existsSync2 } from "fs";
|
|
@@ -259,9 +259,21 @@ var DATASETS_V3 = {
|
|
|
259
259
|
spot: { subdir: "spot", partitionKeys: ["ticker", "date"], filename: "data.parquet" },
|
|
260
260
|
enriched: { subdir: "enriched", partitionKeys: ["ticker"], filename: "data.parquet" },
|
|
261
261
|
enriched_context: { subdir: "enriched/context", partitionKeys: [], filename: "data.parquet" },
|
|
262
|
-
option_chain: {
|
|
263
|
-
|
|
264
|
-
|
|
262
|
+
option_chain: {
|
|
263
|
+
subdir: "option_chain",
|
|
264
|
+
partitionKeys: ["underlying", "date"],
|
|
265
|
+
filename: "data.parquet"
|
|
266
|
+
},
|
|
267
|
+
option_quote_minutes: {
|
|
268
|
+
subdir: "option_quote_minutes",
|
|
269
|
+
partitionKeys: ["underlying", "date"],
|
|
270
|
+
filename: "data.parquet"
|
|
271
|
+
},
|
|
272
|
+
option_oi_daily: {
|
|
273
|
+
subdir: "option_oi_daily",
|
|
274
|
+
partitionKeys: ["underlying", "date"],
|
|
275
|
+
filename: "data.parquet"
|
|
276
|
+
}
|
|
265
277
|
};
|
|
266
278
|
async function writeSpotPartition(conn, args) {
|
|
267
279
|
const def = DATASETS_V3.spot;
|
|
@@ -371,9 +383,7 @@ async function writeParquetPartition(conn, opts) {
|
|
|
371
383
|
const filename = isLegacy ? "data.parquet" : opts.filename ?? "data.parquet";
|
|
372
384
|
for (const [k, v] of Object.entries(partitions)) {
|
|
373
385
|
if (!PARTITION_KEY_RE.test(k)) {
|
|
374
|
-
throw new Error(
|
|
375
|
-
`writeParquetPartition: unsafe partition key: ${JSON.stringify(k)}`
|
|
376
|
-
);
|
|
386
|
+
throw new Error(`writeParquetPartition: unsafe partition key: ${JSON.stringify(k)}`);
|
|
377
387
|
}
|
|
378
388
|
if (!PARTITION_VALUE_RE.test(v)) {
|
|
379
389
|
throw new Error(
|
|
@@ -381,9 +391,7 @@ async function writeParquetPartition(conn, opts) {
|
|
|
381
391
|
);
|
|
382
392
|
}
|
|
383
393
|
}
|
|
384
|
-
const partitionSegments = Object.entries(partitions).map(
|
|
385
|
-
([k, v]) => `${k}=${v}`
|
|
386
|
-
);
|
|
394
|
+
const partitionSegments = Object.entries(partitions).map(([k, v]) => `${k}=${v}`);
|
|
387
395
|
const targetPath = path2.join(opts.baseDir, ...partitionSegments, filename);
|
|
388
396
|
return writeParquetAtomic(conn, {
|
|
389
397
|
targetPath,
|
|
@@ -635,9 +643,7 @@ async function loadEnrichmentWatermarks(dataDir) {
|
|
|
635
643
|
if (raw === null) return { version: 1, watermarks: {} };
|
|
636
644
|
const parsed = EnrichmentWatermarksSchema.safeParse(raw);
|
|
637
645
|
if (!parsed.success) {
|
|
638
|
-
throw new Error(
|
|
639
|
-
`enrichment-watermarks.json is malformed: ${parsed.error.message}`
|
|
640
|
-
);
|
|
646
|
+
throw new Error(`enrichment-watermarks.json is malformed: ${parsed.error.message}`);
|
|
641
647
|
}
|
|
642
648
|
return parsed.data;
|
|
643
649
|
}
|
|
@@ -678,13 +684,27 @@ async function ensureProfilesSchema(conn) {
|
|
|
678
684
|
await conn.run(`
|
|
679
685
|
ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS position_sizing JSON
|
|
680
686
|
`);
|
|
681
|
-
await conn.run(
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
await conn.run(
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
await conn.run(
|
|
687
|
+
await conn.run(
|
|
688
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS underlying VARCHAR`
|
|
689
|
+
);
|
|
690
|
+
await conn.run(
|
|
691
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS re_entry BOOLEAN`
|
|
692
|
+
);
|
|
693
|
+
await conn.run(
|
|
694
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS cap_profits BOOLEAN`
|
|
695
|
+
);
|
|
696
|
+
await conn.run(
|
|
697
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS cap_losses BOOLEAN`
|
|
698
|
+
);
|
|
699
|
+
await conn.run(
|
|
700
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS require_two_prices_pt BOOLEAN`
|
|
701
|
+
);
|
|
702
|
+
await conn.run(
|
|
703
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS close_on_completion BOOLEAN`
|
|
704
|
+
);
|
|
705
|
+
await conn.run(
|
|
706
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS ignore_margin_req BOOLEAN`
|
|
707
|
+
);
|
|
688
708
|
const strategyCols = [
|
|
689
709
|
{ name: "slippage_entry", type: "DOUBLE" },
|
|
690
710
|
{ name: "slippage_exit", type: "DOUBLE" },
|
|
@@ -698,7 +718,9 @@ async function ensureProfilesSchema(conn) {
|
|
|
698
718
|
{ name: "default_to_date", type: "VARCHAR" }
|
|
699
719
|
];
|
|
700
720
|
for (const col of strategyCols) {
|
|
701
|
-
await conn.run(
|
|
721
|
+
await conn.run(
|
|
722
|
+
`ALTER TABLE profiles.strategy_profiles ADD COLUMN IF NOT EXISTS ${col.name} ${col.type}`
|
|
723
|
+
);
|
|
702
724
|
}
|
|
703
725
|
await migrateExpectedRegimes(conn);
|
|
704
726
|
}
|
|
@@ -1068,7 +1090,11 @@ async function createMarketParquetViews(conn, dataDir) {
|
|
|
1068
1090
|
const viewsCreated = [];
|
|
1069
1091
|
const tablesKept = [];
|
|
1070
1092
|
const hiveViews = [
|
|
1071
|
-
{
|
|
1093
|
+
{
|
|
1094
|
+
name: "option_chain",
|
|
1095
|
+
subdir: resolveCanonicalMarketPartitionDir(dataDir, "option_chain"),
|
|
1096
|
+
partitionKey: "underlying"
|
|
1097
|
+
}
|
|
1072
1098
|
];
|
|
1073
1099
|
for (const { name, subdir, partitionKey } of hiveViews) {
|
|
1074
1100
|
const dirPath = subdir;
|
|
@@ -1336,12 +1362,7 @@ async function migrateProfiles(conn, blocksDir) {
|
|
|
1336
1362
|
createdAt: createdAt.toISOString(),
|
|
1337
1363
|
updatedAt: updatedAt.toISOString()
|
|
1338
1364
|
};
|
|
1339
|
-
const filePath = path6.join(
|
|
1340
|
-
blocksDir,
|
|
1341
|
-
blockId,
|
|
1342
|
-
"profiles",
|
|
1343
|
-
toFileSlug(strategyName) + ".json"
|
|
1344
|
-
);
|
|
1365
|
+
const filePath = path6.join(blocksDir, blockId, "profiles", toFileSlug(strategyName) + ".json");
|
|
1345
1366
|
await writeJsonFile(filePath, profileJson);
|
|
1346
1367
|
}
|
|
1347
1368
|
return rows.length;
|
|
@@ -1400,7 +1421,9 @@ async function migrateMarketImportMetadata(conn, dataDir) {
|
|
|
1400
1421
|
return rows.length;
|
|
1401
1422
|
}
|
|
1402
1423
|
async function migrateFlatImportLog(conn, dataDir) {
|
|
1403
|
-
const existingFile = await readJsonFile(
|
|
1424
|
+
const existingFile = await readJsonFile(
|
|
1425
|
+
path6.join(dataDir, "market-meta", "flat-import-log.json")
|
|
1426
|
+
);
|
|
1404
1427
|
if (existingFile !== null) return 0;
|
|
1405
1428
|
let rows;
|
|
1406
1429
|
try {
|
|
@@ -1639,10 +1662,13 @@ async function openReadWriteConnection(dbPath, threads, memoryLimit) {
|
|
|
1639
1662
|
await createMarketParquetViews(connection, dataRoot);
|
|
1640
1663
|
await ensureMarketDataTables(connection);
|
|
1641
1664
|
try {
|
|
1642
|
-
const blocksDir = (await import("./sync-
|
|
1665
|
+
const blocksDir = (await import("./sync-EVLKB3ZJ.js")).getBlocksDir(dataRoot);
|
|
1643
1666
|
await migrateMetadataToJson(connection, dataRoot, blocksDir);
|
|
1644
1667
|
} catch (err) {
|
|
1645
|
-
console.warn(
|
|
1668
|
+
console.warn(
|
|
1669
|
+
"[json-migration] Migration failed (non-fatal):",
|
|
1670
|
+
err instanceof Error ? err.message : err
|
|
1671
|
+
);
|
|
1646
1672
|
}
|
|
1647
1673
|
connectionMode = "read_write";
|
|
1648
1674
|
return connection;
|
|
@@ -1697,7 +1723,11 @@ async function getConnection(dataDir) {
|
|
|
1697
1723
|
} catch (error) {
|
|
1698
1724
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
1699
1725
|
if (isLockError(errorMessage)) {
|
|
1700
|
-
const recovered = await tryRecoverLockByTerminatingStaleProcess(
|
|
1726
|
+
const recovered = await tryRecoverLockByTerminatingStaleProcess(
|
|
1727
|
+
errorMessage,
|
|
1728
|
+
dbPath,
|
|
1729
|
+
forceRecovery
|
|
1730
|
+
);
|
|
1701
1731
|
if (recovered) {
|
|
1702
1732
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
1703
1733
|
await new Promise((r) => setTimeout(r, 500 * (attempt + 1)));
|
|
@@ -1940,18 +1970,13 @@ async function deleteSyncMetadata(conn, blockId, blocksDir) {
|
|
|
1940
1970
|
await deleteSyncMetadataJson(blockId, blocksDir);
|
|
1941
1971
|
return;
|
|
1942
1972
|
}
|
|
1943
|
-
await conn.run(
|
|
1944
|
-
`DELETE FROM trades._sync_metadata WHERE block_id = $1`,
|
|
1945
|
-
[blockId]
|
|
1946
|
-
);
|
|
1973
|
+
await conn.run(`DELETE FROM trades._sync_metadata WHERE block_id = $1`, [blockId]);
|
|
1947
1974
|
}
|
|
1948
1975
|
async function getAllSyncedBlockIds(conn, blocksDir) {
|
|
1949
1976
|
if (isParquetMode() && blocksDir) {
|
|
1950
1977
|
return getAllSyncedBlockIdsJson(blocksDir);
|
|
1951
1978
|
}
|
|
1952
|
-
const reader = await conn.runAndReadAll(
|
|
1953
|
-
`SELECT block_id FROM trades._sync_metadata`
|
|
1954
|
-
);
|
|
1979
|
+
const reader = await conn.runAndReadAll(`SELECT block_id FROM trades._sync_metadata`);
|
|
1955
1980
|
const rows = reader.getRows();
|
|
1956
1981
|
return rows.map((row) => row[0]);
|
|
1957
1982
|
}
|
|
@@ -2129,12 +2154,8 @@ async function detectCsvType(filePath) {
|
|
|
2129
2154
|
const reportingAliases = Object.keys(REPORTING_TRADE_COLUMN_ALIASES).map(
|
|
2130
2155
|
(k) => k.toLowerCase()
|
|
2131
2156
|
);
|
|
2132
|
-
const hasReportingColumns = reportingAliases.some(
|
|
2133
|
-
|
|
2134
|
-
);
|
|
2135
|
-
const hasActualPl = headers.some(
|
|
2136
|
-
(h) => h.includes("actual") && h.includes("p")
|
|
2137
|
-
);
|
|
2157
|
+
const hasReportingColumns = reportingAliases.some((alias) => headers.includes(alias));
|
|
2158
|
+
const hasActualPl = headers.some((h) => h.includes("actual") && h.includes("p"));
|
|
2138
2159
|
const hasReportedStyle = headers.includes("trade id") || headers.some((h) => h.includes("reported"));
|
|
2139
2160
|
if (hasActualPl || hasReportingColumns || hasReportedStyle) {
|
|
2140
2161
|
if (!hasPl || hasActualPl) {
|
|
@@ -2198,7 +2219,9 @@ async function discoverCsvFiles(folderPath) {
|
|
|
2198
2219
|
return { mappings, unrecognized };
|
|
2199
2220
|
}
|
|
2200
2221
|
function logCsvDiscoveryWarning(folderName, csvFiles) {
|
|
2201
|
-
console.error(
|
|
2222
|
+
console.error(
|
|
2223
|
+
`Warning: Folder '${folderName}' has CSV files but none match expected trade log format.`
|
|
2224
|
+
);
|
|
2202
2225
|
console.error(` Found: ${csvFiles.join(", ")}`);
|
|
2203
2226
|
console.error(` Expected columns: P/L, Date Opened, Date Closed, Symbol, Strategy`);
|
|
2204
2227
|
}
|
|
@@ -2401,9 +2424,7 @@ async function loadDailyLogs(blockPath, blockId, filename = "dailylog.csv") {
|
|
|
2401
2424
|
entries.push(entry);
|
|
2402
2425
|
}
|
|
2403
2426
|
}
|
|
2404
|
-
entries.sort(
|
|
2405
|
-
(a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()
|
|
2406
|
-
);
|
|
2427
|
+
entries.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime());
|
|
2407
2428
|
return entries.length > 0 ? entries : void 0;
|
|
2408
2429
|
} catch {
|
|
2409
2430
|
return void 0;
|
|
@@ -2612,14 +2633,12 @@ async function listBlocks(baseDir) {
|
|
|
2612
2633
|
}
|
|
2613
2634
|
function normalizeRecordHeaders(raw) {
|
|
2614
2635
|
const normalized = { ...raw };
|
|
2615
|
-
Object.entries(REPORTING_TRADE_COLUMN_ALIASES).forEach(
|
|
2616
|
-
([alias
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
delete normalized[alias];
|
|
2620
|
-
}
|
|
2636
|
+
Object.entries(REPORTING_TRADE_COLUMN_ALIASES).forEach(([alias, canonical]) => {
|
|
2637
|
+
if (normalized[alias] !== void 0) {
|
|
2638
|
+
normalized[canonical] = normalized[alias];
|
|
2639
|
+
delete normalized[alias];
|
|
2621
2640
|
}
|
|
2622
|
-
);
|
|
2641
|
+
});
|
|
2623
2642
|
return normalized;
|
|
2624
2643
|
}
|
|
2625
2644
|
function convertToReportingTrade(raw) {
|
|
@@ -2672,9 +2691,7 @@ async function loadReportingLog(baseDir, blockId) {
|
|
|
2672
2691
|
trades.push(trade);
|
|
2673
2692
|
}
|
|
2674
2693
|
}
|
|
2675
|
-
trades.sort(
|
|
2676
|
-
(a, b) => new Date(a.dateOpened).getTime() - new Date(b.dateOpened).getTime()
|
|
2677
|
-
);
|
|
2694
|
+
trades.sort((a, b) => new Date(a.dateOpened).getTime() - new Date(b.dateOpened).getTime());
|
|
2678
2695
|
return trades;
|
|
2679
2696
|
}
|
|
2680
2697
|
function toKebabCase(str) {
|
|
@@ -2714,9 +2731,7 @@ function validateCsvColumns(records, csvType) {
|
|
|
2714
2731
|
}
|
|
2715
2732
|
const dateOpenedAliases = ["Date Opened", "date_opened"];
|
|
2716
2733
|
const plAliases = ["P/L", "pl"];
|
|
2717
|
-
const hasDateOpened = dateOpenedAliases.some(
|
|
2718
|
-
(col) => headers.includes(col)
|
|
2719
|
-
);
|
|
2734
|
+
const hasDateOpened = dateOpenedAliases.some((col) => headers.includes(col));
|
|
2720
2735
|
const hasPl = plAliases.some((col) => headers.includes(col));
|
|
2721
2736
|
const missing = [];
|
|
2722
2737
|
if (!hasDateOpened) missing.push("Date Opened");
|
|
@@ -2756,9 +2771,7 @@ async function importCsv(baseDir, options) {
|
|
|
2756
2771
|
const name = blockName;
|
|
2757
2772
|
const blockId = toKebabCase(name);
|
|
2758
2773
|
if (!blockId) {
|
|
2759
|
-
throw new Error(
|
|
2760
|
-
"Could not derive a valid block ID from the filename or provided name"
|
|
2761
|
-
);
|
|
2774
|
+
throw new Error("Could not derive a valid block ID from the filename or provided name");
|
|
2762
2775
|
}
|
|
2763
2776
|
const blockPath = path9.join(blocksDir, blockId);
|
|
2764
2777
|
try {
|
|
@@ -2917,10 +2930,7 @@ async function insertTradeBatch(conn, blockId, records, startIdx, batchSize) {
|
|
|
2917
2930
|
for (let rowIdx = 0; rowIdx < batch.length; rowIdx++) {
|
|
2918
2931
|
const record = batch[rowIdx];
|
|
2919
2932
|
const baseParam = rowIdx * columnsPerRow + 1;
|
|
2920
|
-
const rowPlaceholders = Array.from(
|
|
2921
|
-
{ length: columnsPerRow },
|
|
2922
|
-
(_, i) => `$${baseParam + i}`
|
|
2923
|
-
);
|
|
2933
|
+
const rowPlaceholders = Array.from({ length: columnsPerRow }, (_, i) => `$${baseParam + i}`);
|
|
2924
2934
|
placeholders.push(`(${rowPlaceholders.join(", ")})`);
|
|
2925
2935
|
const premium = parseFloat(record["Premium"]);
|
|
2926
2936
|
const numContracts = parseInt(record["No. of Contracts"], 10);
|
|
@@ -2988,10 +2998,7 @@ async function insertReportingBatch(conn, blockId, trades, tickers, startIdx, ba
|
|
|
2988
2998
|
for (let rowIdx = 0; rowIdx < batch.length; rowIdx++) {
|
|
2989
2999
|
const trade = batch[rowIdx];
|
|
2990
3000
|
const baseParam = rowIdx * columnsPerRow + 1;
|
|
2991
|
-
const rowPlaceholders = Array.from(
|
|
2992
|
-
{ length: columnsPerRow },
|
|
2993
|
-
(_, i) => `$${baseParam + i}`
|
|
2994
|
-
);
|
|
3001
|
+
const rowPlaceholders = Array.from({ length: columnsPerRow }, (_, i) => `$${baseParam + i}`);
|
|
2995
3002
|
placeholders.push(`(${rowPlaceholders.join(", ")})`);
|
|
2996
3003
|
params.push(
|
|
2997
3004
|
blockId,
|
|
@@ -3033,14 +3040,8 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3033
3040
|
if (existingMetadata) {
|
|
3034
3041
|
await conn.run("BEGIN TRANSACTION");
|
|
3035
3042
|
try {
|
|
3036
|
-
await conn.run(
|
|
3037
|
-
|
|
3038
|
-
[blockId]
|
|
3039
|
-
);
|
|
3040
|
-
await conn.run(
|
|
3041
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
3042
|
-
[blockId]
|
|
3043
|
-
);
|
|
3043
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
3044
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
3044
3045
|
await deleteSyncMetadata(conn, blockId, blocksDir);
|
|
3045
3046
|
await conn.run("COMMIT");
|
|
3046
3047
|
return { blockId, status: "deleted" };
|
|
@@ -3075,10 +3076,7 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3075
3076
|
}
|
|
3076
3077
|
await conn.run("BEGIN TRANSACTION");
|
|
3077
3078
|
try {
|
|
3078
|
-
await conn.run(
|
|
3079
|
-
"DELETE FROM trades.trade_data WHERE block_id = $1",
|
|
3080
|
-
[blockId]
|
|
3081
|
-
);
|
|
3079
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
3082
3080
|
const csvContent = await fs8.readFile(tradelogPath, "utf-8");
|
|
3083
3081
|
const records = parseCSV2(csvContent);
|
|
3084
3082
|
const batchSize = 500;
|
|
@@ -3090,24 +3088,21 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3090
3088
|
let reportinglogHash = null;
|
|
3091
3089
|
if (optionalLogs.dailylog) {
|
|
3092
3090
|
try {
|
|
3093
|
-
dailylogHash = versionedHash(
|
|
3094
|
-
path10.join(blockPath, optionalLogs.dailylog)
|
|
3095
|
-
)
|
|
3091
|
+
dailylogHash = versionedHash(
|
|
3092
|
+
await hashFileContent(path10.join(blockPath, optionalLogs.dailylog))
|
|
3093
|
+
);
|
|
3096
3094
|
} catch {
|
|
3097
3095
|
}
|
|
3098
3096
|
}
|
|
3099
3097
|
if (optionalLogs.reportinglog) {
|
|
3100
3098
|
try {
|
|
3101
|
-
reportinglogHash = versionedHash(
|
|
3102
|
-
path10.join(blockPath, optionalLogs.reportinglog)
|
|
3103
|
-
)
|
|
3099
|
+
reportinglogHash = versionedHash(
|
|
3100
|
+
await hashFileContent(path10.join(blockPath, optionalLogs.reportinglog))
|
|
3101
|
+
);
|
|
3104
3102
|
} catch {
|
|
3105
3103
|
}
|
|
3106
3104
|
}
|
|
3107
|
-
await conn.run(
|
|
3108
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
3109
|
-
[blockId]
|
|
3110
|
-
);
|
|
3105
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
3111
3106
|
if (optionalLogs.reportinglog && reportinglogHash) {
|
|
3112
3107
|
const reportingPath = path10.join(blockPath, optionalLogs.reportinglog);
|
|
3113
3108
|
const reportingContent = await fs8.readFile(reportingPath, "utf-8");
|
|
@@ -3122,7 +3117,14 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3122
3117
|
}
|
|
3123
3118
|
}
|
|
3124
3119
|
for (let i = 0; i < reportingTrades.length; i += batchSize) {
|
|
3125
|
-
await insertReportingBatch(
|
|
3120
|
+
await insertReportingBatch(
|
|
3121
|
+
conn,
|
|
3122
|
+
blockId,
|
|
3123
|
+
reportingTrades,
|
|
3124
|
+
reportingTickers,
|
|
3125
|
+
i,
|
|
3126
|
+
batchSize
|
|
3127
|
+
);
|
|
3126
3128
|
}
|
|
3127
3129
|
}
|
|
3128
3130
|
const newMetadata = {
|
|
@@ -3145,14 +3147,8 @@ async function syncBlockInternal(conn, blockId, blockPath) {
|
|
|
3145
3147
|
if (existingMetadata) {
|
|
3146
3148
|
try {
|
|
3147
3149
|
await conn.run("BEGIN TRANSACTION");
|
|
3148
|
-
await conn.run(
|
|
3149
|
-
|
|
3150
|
-
[blockId]
|
|
3151
|
-
);
|
|
3152
|
-
await conn.run(
|
|
3153
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
3154
|
-
[blockId]
|
|
3155
|
-
);
|
|
3150
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
3151
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
3156
3152
|
await deleteSyncMetadata(conn, blockId, blocksDir);
|
|
3157
3153
|
await conn.run("COMMIT");
|
|
3158
3154
|
} catch {
|
|
@@ -3182,7 +3178,8 @@ async function detectBlockChanges(conn, baseDir) {
|
|
|
3182
3178
|
for (const entry of entries) {
|
|
3183
3179
|
if (!entry.isDirectory()) continue;
|
|
3184
3180
|
if (entry.name.startsWith(".") || entry.name.startsWith("_")) continue;
|
|
3185
|
-
if (entry.name.endsWith(".tmp") || entry.name.endsWith(".duckdb") || entry.name.endsWith(".duckdb.tmp"))
|
|
3181
|
+
if (entry.name.endsWith(".tmp") || entry.name.endsWith(".duckdb") || entry.name.endsWith(".duckdb.tmp"))
|
|
3182
|
+
continue;
|
|
3186
3183
|
const blockId = entry.name;
|
|
3187
3184
|
folderNames.add(blockId);
|
|
3188
3185
|
const blockPath = path10.join(baseDir, blockId);
|
|
@@ -3231,14 +3228,8 @@ async function cleanupDeletedBlocks(conn, deletedBlockIds, blocksDir) {
|
|
|
3231
3228
|
for (const blockId of deletedBlockIds) {
|
|
3232
3229
|
await conn.run("BEGIN TRANSACTION");
|
|
3233
3230
|
try {
|
|
3234
|
-
await conn.run(
|
|
3235
|
-
|
|
3236
|
-
[blockId]
|
|
3237
|
-
);
|
|
3238
|
-
await conn.run(
|
|
3239
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
3240
|
-
[blockId]
|
|
3241
|
-
);
|
|
3231
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
3232
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
3242
3233
|
await deleteSyncMetadata(conn, blockId, blocksDir);
|
|
3243
3234
|
await conn.run("COMMIT");
|
|
3244
3235
|
} catch (err) {
|
|
@@ -3397,4 +3388,4 @@ export {
|
|
|
3397
3388
|
loadReportingLog,
|
|
3398
3389
|
importCsv
|
|
3399
3390
|
};
|
|
3400
|
-
//# sourceMappingURL=chunk-
|
|
3391
|
+
//# sourceMappingURL=chunk-UBUC5A66.js.map
|