tradeblocks-mcp 3.0.2 → 3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -68
- package/dist/{chunk-LDKTV7GW.js → chunk-27S67XW3.js} +24 -52
- package/dist/chunk-27S67XW3.js.map +1 -0
- package/dist/{chunk-FGZH632F.js → chunk-2E63THNI.js} +13 -11
- package/dist/chunk-2E63THNI.js.map +1 -0
- package/dist/{chunk-BKQ4PM4Y.js → chunk-6S37CXUA.js} +11 -13
- package/dist/chunk-6S37CXUA.js.map +1 -0
- package/dist/{chunk-PRAYH3RT.js → chunk-AP7IHUUR.js} +29 -12
- package/dist/chunk-AP7IHUUR.js.map +1 -0
- package/dist/{chunk-QTTR7AAW.js → chunk-C6LL746C.js} +67 -84
- package/dist/{chunk-QTTR7AAW.js.map → chunk-C6LL746C.js.map} +1 -1
- package/dist/{chunk-W2PP3LEH.js → chunk-SEUZYQGQ.js} +17 -8
- package/dist/chunk-SEUZYQGQ.js.map +1 -0
- package/dist/{chunk-4BLCXNQ6.js → chunk-UBUC5A66.js} +105 -114
- package/dist/chunk-UBUC5A66.js.map +1 -0
- package/dist/{chunk-XXYOUIZY.js → chunk-VDU25Z6X.js} +63 -80
- package/dist/chunk-VDU25Z6X.js.map +1 -0
- package/dist/daily-log-processor-BY3ISY6K.js +9 -0
- package/dist/{daily-logs-store-D3QC2XPV.js → daily-logs-store-3C3OTGPK.js} +2 -2
- package/dist/iv-solver-worker.js +9 -1
- package/dist/market-provider-H3ARIVZ4.js +16 -0
- package/dist/{proto-3HZTCWK4.js → proto-V23GIZG2.js} +2 -2
- package/dist/{sync-PO4IPCYV.js → sync-EVLKB3ZJ.js} +7 -7
- package/dist/test-exports.js +358 -254
- package/dist/test-exports.js.map +1 -1
- package/dist/trade-processor-EYA3I3XB.js +9 -0
- package/dist/{trades-store-LOL3FQSF.js → trades-store-R3NL25T2.js} +2 -2
- package/manifest.json +3 -11
- package/package.json +1 -1
- package/server/{chunk-NZO6PT64.js → chunk-4P7D7YZP.js} +102 -113
- package/server/chunk-4P7D7YZP.js.map +1 -0
- package/server/{chunk-72GKJE2U.js → chunk-BOPHW5M6.js} +521 -235
- package/server/chunk-BOPHW5M6.js.map +1 -0
- package/server/{chunk-WA5AAPCH.js → chunk-GH2552SE.js} +17 -8
- package/server/chunk-GH2552SE.js.map +1 -0
- package/server/{chunk-5EBXHT6C.js → chunk-OBYKFW2B.js} +11 -13
- package/server/chunk-OBYKFW2B.js.map +1 -0
- package/server/{chunk-FBNDMCT5.js → chunk-T66KH2XH.js} +13 -11
- package/server/chunk-T66KH2XH.js.map +1 -0
- package/server/{chunk-NRFXAJF7.js → chunk-W5E7FHC4.js} +24 -52
- package/server/chunk-W5E7FHC4.js.map +1 -0
- package/server/{chunk-CSDVJPBB.js → chunk-YUCOAJ4Z.js} +29 -12
- package/server/chunk-YUCOAJ4Z.js.map +1 -0
- package/server/{chunk-PNKG7RY7.js → chunk-ZBJCF4ZG.js} +63 -80
- package/server/chunk-ZBJCF4ZG.js.map +1 -0
- package/server/{config-6IZXEFEX.js → config-DK7KOMNL.js} +1 -1
- package/server/config-DK7KOMNL.js.map +1 -0
- package/server/daily-log-processor-ENEUT22A.js +10 -0
- package/server/{daily-logs-store-NU7M4L5C.js → daily-logs-store-YFNXTB7J.js} +2 -2
- package/server/http-server.js +2 -5
- package/server/index.js +1771 -1538
- package/server/index.js.map +1 -1
- package/server/iv-solver-worker.js +9 -1
- package/server/market-provider-B437HKLW.js +17 -0
- package/server/{proto-DWRZJO4E.js → proto-YZ4TLX2N.js} +2 -2
- package/server/{sync-QFI5L7S7.js → sync-V25UQJA3.js} +7 -7
- package/server/trade-processor-L3PIQ5TG.js +10 -0
- package/server/{trades-store-VY26MWER.js → trades-store-RTTEYMXH.js} +2 -2
- package/src/auth/clients-store.ts +4 -4
- package/src/auth/code-store.ts +1 -1
- package/src/auth/config.ts +11 -12
- package/src/auth/login-page.ts +8 -10
- package/src/auth/provider.ts +35 -30
- package/src/auth/token.ts +17 -15
- package/src/db/backtest-schemas.ts +12 -4
- package/src/db/connection.ts +107 -40
- package/src/db/index.ts +37 -5
- package/src/db/json-adapters.ts +22 -31
- package/src/db/json-migration.ts +11 -15
- package/src/db/json-store.ts +5 -3
- package/src/db/market-datasets.ts +48 -15
- package/src/db/market-schemas.ts +11 -5
- package/src/db/market-views.ts +119 -32
- package/src/db/parquet-writer.ts +7 -7
- package/src/db/profile-schemas.ts +39 -22
- package/src/db/schemas.ts +2 -2
- package/src/http-server.ts +7 -12
- package/src/index.ts +8 -9
- package/src/market/ingestor/index.ts +1 -4
- package/src/market/ingestor/market-ingestor.ts +140 -67
- package/src/market/ingestor/types.ts +1 -3
- package/src/market/stores/chain-sql.ts +2 -8
- package/src/market/stores/chain-store.ts +1 -4
- package/src/market/stores/duckdb-chain-store.ts +3 -14
- package/src/market/stores/duckdb-enriched-store.ts +3 -13
- package/src/market/stores/duckdb-quote-store.ts +10 -27
- package/src/market/stores/duckdb-spot-store.ts +4 -20
- package/src/market/stores/enriched-sql.ts +3 -9
- package/src/market/stores/enriched-store.ts +2 -2
- package/src/market/stores/parquet-chain-store.ts +5 -22
- package/src/market/stores/parquet-enriched-store.ts +4 -16
- package/src/market/stores/parquet-oi-daily-store.ts +3 -14
- package/src/market/stores/parquet-quote-store.ts +9 -29
- package/src/market/stores/parquet-spot-store.ts +14 -33
- package/src/market/stores/quote-store.ts +2 -10
- package/src/market/stores/spot-sql.ts +3 -15
- package/src/market/stores/spot-store.ts +1 -1
- package/src/market/tickers/defaults.json +6 -6
- package/src/market/tickers/loader.ts +2 -7
- package/src/market/tickers/registry.ts +5 -15
- package/src/market/tickers/schemas.ts +2 -6
- package/src/models/strategy-profile.ts +39 -39
- package/src/sync/block-sync.ts +44 -65
- package/src/sync/index.ts +1 -4
- package/src/sync/metadata.ts +14 -20
- package/src/test-exports.ts +127 -123
- package/src/tools/analysis.ts +180 -159
- package/src/tools/batch-exit-analysis.ts +100 -59
- package/src/tools/blocks/analysis.ts +116 -101
- package/src/tools/blocks/comparison.ts +281 -364
- package/src/tools/blocks/core.ts +208 -253
- package/src/tools/blocks/health.ts +571 -563
- package/src/tools/blocks/similarity.ts +316 -283
- package/src/tools/edge-decay.ts +124 -197
- package/src/tools/exit-analysis.ts +130 -77
- package/src/tools/greeks-attribution.ts +84 -35
- package/src/tools/guides.ts +4 -6
- package/src/tools/imports.ts +9 -12
- package/src/tools/market-data.ts +1067 -757
- package/src/tools/market-enrichment.ts +3 -3
- package/src/tools/market-fetch.ts +148 -67
- package/src/tools/market-imports.ts +12 -12
- package/src/tools/middleware/sync-middleware.ts +5 -6
- package/src/tools/performance.ts +185 -302
- package/src/tools/profile-analysis.ts +52 -66
- package/src/tools/profiles.ts +106 -69
- package/src/tools/regime-advisor.ts +20 -45
- package/src/tools/replay.ts +81 -77
- package/src/tools/reports/discrepancies.ts +298 -328
- package/src/tools/reports/fields.ts +7 -25
- package/src/tools/reports/helpers.ts +18 -49
- package/src/tools/reports/predictive.ts +27 -70
- package/src/tools/reports/slippage-trends.ts +315 -345
- package/src/tools/reports/slippage.ts +1 -4
- package/src/tools/reports/strategy-matches.ts +399 -441
- package/src/tools/schema.ts +43 -40
- package/src/tools/shared/filters.ts +3 -9
- package/src/tools/snapshot.ts +9 -30
- package/src/tools/sql.ts +15 -14
- package/src/tools/tickers.ts +1 -4
- package/src/utils/batch-exit-analysis.ts +31 -42
- package/src/utils/black-scholes.ts +39 -29
- package/src/utils/block-loader.ts +68 -82
- package/src/utils/calibration-probe.ts +1 -2
- package/src/utils/chain-loader.ts +3 -3
- package/src/utils/csv-discovery.ts +16 -22
- package/src/utils/data-quality.ts +24 -36
- package/src/utils/exit-triggers.ts +91 -96
- package/src/utils/field-timing.ts +94 -79
- package/src/utils/filter-predicates.ts +13 -9
- package/src/utils/flatfile-importer.ts +94 -64
- package/src/utils/greeks-decomposition.ts +152 -100
- package/src/utils/iv-solver-pool.ts +55 -25
- package/src/utils/iv-solver-worker.ts +5 -5
- package/src/utils/market-enricher.ts +528 -497
- package/src/utils/market-importer.ts +31 -12
- package/src/utils/market-provider.ts +21 -23
- package/src/utils/massive-tier.ts +5 -7
- package/src/utils/migrate-option-data-helpers.ts +2 -8
- package/src/utils/option-quote-greeks.ts +25 -31
- package/src/utils/option-time.ts +4 -8
- package/src/utils/output-formatter.ts +1 -4
- package/src/utils/provider-capabilities.ts +1 -4
- package/src/utils/providers/massive.ts +59 -93
- package/src/utils/providers/thetadata/backfill.ts +14 -23
- package/src/utils/providers/thetadata/client.ts +12 -8
- package/src/utils/providers/thetadata/decode.ts +2 -20
- package/src/utils/providers/thetadata/endpoints.ts +32 -40
- package/src/utils/providers/thetadata/join.ts +11 -10
- package/src/utils/providers/thetadata/proto.ts +12 -10
- package/src/utils/providers/thetadata/quote-mid-greeks.ts +5 -5
- package/src/utils/providers/thetadata.ts +11 -10
- package/src/utils/quote-enricher.ts +4 -4
- package/src/utils/quote-parquet-projection.ts +3 -11
- package/src/utils/sample-date-selector.ts +3 -5
- package/src/utils/schema-metadata.ts +102 -70
- package/src/utils/ticker.ts +5 -9
- package/src/utils/trade-replay.ts +77 -68
- package/dist/chunk-4BLCXNQ6.js.map +0 -1
- package/dist/chunk-BKQ4PM4Y.js.map +0 -1
- package/dist/chunk-FGZH632F.js.map +0 -1
- package/dist/chunk-LDKTV7GW.js.map +0 -1
- package/dist/chunk-PRAYH3RT.js.map +0 -1
- package/dist/chunk-W2PP3LEH.js.map +0 -1
- package/dist/chunk-XXYOUIZY.js.map +0 -1
- package/dist/daily-log-processor-6MWJ23JK.js +0 -9
- package/dist/market-provider-VDRJUEF2.js +0 -16
- package/dist/trade-processor-NHU4VWRX.js +0 -9
- package/server/chunk-5EBXHT6C.js.map +0 -1
- package/server/chunk-72GKJE2U.js.map +0 -1
- package/server/chunk-CSDVJPBB.js.map +0 -1
- package/server/chunk-FBNDMCT5.js.map +0 -1
- package/server/chunk-NRFXAJF7.js.map +0 -1
- package/server/chunk-NZO6PT64.js.map +0 -1
- package/server/chunk-PNKG7RY7.js.map +0 -1
- package/server/chunk-WA5AAPCH.js.map +0 -1
- package/server/config-6IZXEFEX.js.map +0 -1
- package/server/daily-log-processor-Y3PVSVBM.js +0 -10
- package/server/market-provider-VOYYVYWT.js +0 -17
- package/server/trade-processor-JWVS37KM.js +0 -10
- /package/dist/{daily-log-processor-6MWJ23JK.js.map → daily-log-processor-BY3ISY6K.js.map} +0 -0
- /package/dist/{daily-logs-store-D3QC2XPV.js.map → daily-logs-store-3C3OTGPK.js.map} +0 -0
- /package/dist/{market-provider-VDRJUEF2.js.map → market-provider-H3ARIVZ4.js.map} +0 -0
- /package/dist/{proto-3HZTCWK4.js.map → proto-V23GIZG2.js.map} +0 -0
- /package/dist/{sync-PO4IPCYV.js.map → sync-EVLKB3ZJ.js.map} +0 -0
- /package/dist/{trade-processor-NHU4VWRX.js.map → trade-processor-EYA3I3XB.js.map} +0 -0
- /package/dist/{trades-store-LOL3FQSF.js.map → trades-store-R3NL25T2.js.map} +0 -0
- /package/server/{daily-log-processor-Y3PVSVBM.js.map → daily-log-processor-ENEUT22A.js.map} +0 -0
- /package/server/{daily-logs-store-NU7M4L5C.js.map → daily-logs-store-YFNXTB7J.js.map} +0 -0
- /package/server/{market-provider-VOYYVYWT.js.map → market-provider-B437HKLW.js.map} +0 -0
- /package/server/{proto-DWRZJO4E.js.map → proto-YZ4TLX2N.js.map} +0 -0
- /package/server/{sync-QFI5L7S7.js.map → sync-V25UQJA3.js.map} +0 -0
- /package/server/{trade-processor-JWVS37KM.js.map → trade-processor-L3PIQ5TG.js.map} +0 -0
- /package/server/{trades-store-VY26MWER.js.map → trades-store-RTTEYMXH.js.map} +0 -0
|
@@ -7,30 +7,30 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
export interface PositionSizing {
|
|
10
|
-
method: string;
|
|
11
|
-
allocationPct?: number;
|
|
12
|
-
maxContracts?: number;
|
|
10
|
+
method: string; // "pct_of_portfolio" | "fixed_contracts" | "fixed_dollar" | "discretionary"
|
|
11
|
+
allocationPct?: number; // e.g., 2, 10
|
|
12
|
+
maxContracts?: number; // hard cap per trade
|
|
13
13
|
maxAllocationDollar?: number; // hard dollar cap per trade
|
|
14
|
-
maxOpenPositions?: number;
|
|
15
|
-
description?: string;
|
|
16
|
-
backtestAllocationPct?: number;
|
|
17
|
-
liveAllocationPct?: number;
|
|
18
|
-
maxContractsPerTrade?: number;
|
|
14
|
+
maxOpenPositions?: number; // concurrency limit
|
|
15
|
+
description?: string; // free text for anything unusual
|
|
16
|
+
backtestAllocationPct?: number; // allocation % used in backtest
|
|
17
|
+
liveAllocationPct?: number; // allocation % used in live portfolio
|
|
18
|
+
maxContractsPerTrade?: number; // per-entry cap (distinct from maxContracts hard cap)
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
export interface StrategyProfile {
|
|
22
22
|
blockId: string;
|
|
23
23
|
strategyName: string;
|
|
24
|
-
structureType: string;
|
|
25
|
-
greeksBias: string;
|
|
26
|
-
thesis: string;
|
|
27
|
-
legs: LegDetail[];
|
|
28
|
-
entryFilters: EntryFilter[];
|
|
29
|
-
exitRules: ExitRule[];
|
|
30
|
-
expectedRegimes: string[];
|
|
31
|
-
keyMetrics: KeyMetrics;
|
|
24
|
+
structureType: string; // e.g., "iron_condor", "calendar_spread", "reverse_iron_condor"
|
|
25
|
+
greeksBias: string; // e.g., "theta_positive", "vega_negative", "delta_neutral"
|
|
26
|
+
thesis: string; // Free-text description of the strategy thesis
|
|
27
|
+
legs: LegDetail[]; // Structured leg descriptions
|
|
28
|
+
entryFilters: EntryFilter[]; // Conditions for entry
|
|
29
|
+
exitRules: ExitRule[]; // Exit criteria
|
|
30
|
+
expectedRegimes: string[]; // Market regimes this strategy targets
|
|
31
|
+
keyMetrics: KeyMetrics; // Performance benchmarks
|
|
32
32
|
positionSizing?: PositionSizing; // Per-block position sizing rules
|
|
33
|
-
underlying?: string;
|
|
33
|
+
underlying?: string; // e.g., "SPX", "QQQ"
|
|
34
34
|
reEntry?: boolean;
|
|
35
35
|
capProfits?: boolean;
|
|
36
36
|
capLosses?: boolean;
|
|
@@ -42,43 +42,43 @@ export interface StrategyProfile {
|
|
|
42
42
|
}
|
|
43
43
|
|
|
44
44
|
export interface LegDetail {
|
|
45
|
-
type: string;
|
|
46
|
-
strike: string;
|
|
47
|
-
expiry: string;
|
|
45
|
+
type: string; // "long_put", "short_call", etc.
|
|
46
|
+
strike: string; // Relative description: "ATM", "5-delta", "30-delta"
|
|
47
|
+
expiry: string; // Relative: "same-day", "weekly", "45-DTE"
|
|
48
48
|
quantity: number; // Positive = long, negative = short
|
|
49
|
-
strikeMethod?:
|
|
49
|
+
strikeMethod?: "delta" | "dollar_price" | "offset" | "percentage";
|
|
50
50
|
strikeValue?: number; // numeric value (e.g., 25 for 25-delta, 3.50 for dollar_price)
|
|
51
51
|
}
|
|
52
52
|
|
|
53
53
|
export interface EntryFilter {
|
|
54
|
-
field: string;
|
|
55
|
-
operator: string;
|
|
54
|
+
field: string; // e.g., "VIX_Close", "RSI_14", "Vol_Regime"
|
|
55
|
+
operator: string; // ">", "<", ">=", "<=", "==", "between", "in"
|
|
56
56
|
value: string | number | (string | number)[];
|
|
57
57
|
description?: string;
|
|
58
58
|
source?: "market" | "execution"; // "market" = testable against market data, "execution" = OO/platform-level
|
|
59
59
|
}
|
|
60
60
|
|
|
61
61
|
export interface ExitRuleMonitoring {
|
|
62
|
-
granularity?:
|
|
63
|
-
priceSource?:
|
|
62
|
+
granularity?: "intra_minute" | "candle_close" | "end_of_bar";
|
|
63
|
+
priceSource?: "nbbo" | "mid" | "last";
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
export interface ExitRule {
|
|
67
|
-
type: string;
|
|
68
|
-
trigger: string;
|
|
67
|
+
type: string; // "stop_loss", "profit_target", "time_exit", "conditional"
|
|
68
|
+
trigger: string; // e.g., "200% of credit", "50% of max profit", "15:00 ET"
|
|
69
69
|
description?: string;
|
|
70
|
-
stopLossType?:
|
|
70
|
+
stopLossType?: "percentage" | "dollar" | "sl_ratio" | "debit_percentage";
|
|
71
71
|
stopLossValue?: number;
|
|
72
72
|
monitoring?: ExitRuleMonitoring;
|
|
73
|
-
slippage?: number;
|
|
73
|
+
slippage?: number; // per-rule slippage override
|
|
74
74
|
}
|
|
75
75
|
|
|
76
76
|
export interface KeyMetrics {
|
|
77
|
-
expectedWinRate?: number;
|
|
78
|
-
targetPremium?: number;
|
|
79
|
-
maxLoss?: number;
|
|
80
|
-
profitTarget?: number;
|
|
81
|
-
[key: string]: unknown;
|
|
77
|
+
expectedWinRate?: number; // 0-1
|
|
78
|
+
targetPremium?: number; // Dollar amount
|
|
79
|
+
maxLoss?: number; // Dollar amount per contract
|
|
80
|
+
profitTarget?: number; // Dollar amount or percentage
|
|
81
|
+
[key: string]: unknown; // Extensible for strategy-specific metrics
|
|
82
82
|
}
|
|
83
83
|
|
|
84
84
|
/**
|
|
@@ -91,12 +91,12 @@ export interface StrategyProfileRow {
|
|
|
91
91
|
structure_type: string;
|
|
92
92
|
greeks_bias: string;
|
|
93
93
|
thesis: string;
|
|
94
|
-
legs: string;
|
|
95
|
-
entry_filters: string;
|
|
96
|
-
exit_rules: string;
|
|
94
|
+
legs: string; // JSON string
|
|
95
|
+
entry_filters: string; // JSON string
|
|
96
|
+
exit_rules: string; // JSON string
|
|
97
97
|
expected_regimes: string; // JSON string
|
|
98
|
-
key_metrics: string;
|
|
99
|
-
position_sizing: string;
|
|
98
|
+
key_metrics: string; // JSON string
|
|
99
|
+
position_sizing: string; // JSON string
|
|
100
100
|
underlying: string | null;
|
|
101
101
|
re_entry: boolean | null;
|
|
102
102
|
cap_profits: boolean | null;
|
package/src/sync/block-sync.ts
CHANGED
|
@@ -38,7 +38,10 @@ export interface BlockSyncResult {
|
|
|
38
38
|
*/
|
|
39
39
|
function parseCSV(content: string): Record<string, string>[] {
|
|
40
40
|
// Strip UTF-8 BOM if present (common in Windows/Excel CSV exports)
|
|
41
|
-
const lines = content
|
|
41
|
+
const lines = content
|
|
42
|
+
.replace(/^\uFEFF/, "")
|
|
43
|
+
.trim()
|
|
44
|
+
.split("\n");
|
|
42
45
|
if (lines.length < 2) return [];
|
|
43
46
|
|
|
44
47
|
const headers = parseCSVLine(lines[0]);
|
|
@@ -118,11 +121,7 @@ function normalizeCsvDate(value: string | undefined): string | null {
|
|
|
118
121
|
}
|
|
119
122
|
|
|
120
123
|
const dt = new Date(Date.UTC(year, month - 1, day));
|
|
121
|
-
if (
|
|
122
|
-
dt.getUTCFullYear() !== year ||
|
|
123
|
-
dt.getUTCMonth() + 1 !== month ||
|
|
124
|
-
dt.getUTCDate() !== day
|
|
125
|
-
) {
|
|
124
|
+
if (dt.getUTCFullYear() !== year || dt.getUTCMonth() + 1 !== month || dt.getUTCDate() !== day) {
|
|
126
125
|
return raw;
|
|
127
126
|
}
|
|
128
127
|
|
|
@@ -137,9 +136,7 @@ function normalizeCsvDate(value: string | undefined): string | null {
|
|
|
137
136
|
* Find the tradelog CSV file for a block using header-sniffing discovery.
|
|
138
137
|
* No longer reads block.json — uses discoverCsvFiles from csv-discovery.ts.
|
|
139
138
|
*/
|
|
140
|
-
async function findTradelogFile(
|
|
141
|
-
blockPath: string
|
|
142
|
-
): Promise<string | null> {
|
|
139
|
+
async function findTradelogFile(blockPath: string): Promise<string | null> {
|
|
143
140
|
const { mappings } = await discoverCsvFiles(blockPath);
|
|
144
141
|
return mappings.tradelog || null;
|
|
145
142
|
}
|
|
@@ -149,7 +146,7 @@ async function findTradelogFile(
|
|
|
149
146
|
* No longer reads block.json — uses discoverCsvFiles from csv-discovery.ts.
|
|
150
147
|
*/
|
|
151
148
|
async function findOptionalLogFiles(
|
|
152
|
-
blockPath: string
|
|
149
|
+
blockPath: string,
|
|
153
150
|
): Promise<{ dailylog: string | null; reportinglog: string | null }> {
|
|
154
151
|
const { mappings } = await discoverCsvFiles(blockPath);
|
|
155
152
|
return {
|
|
@@ -174,7 +171,7 @@ async function insertTradeBatch(
|
|
|
174
171
|
blockId: string,
|
|
175
172
|
records: Record<string, string>[],
|
|
176
173
|
startIdx: number,
|
|
177
|
-
batchSize: number
|
|
174
|
+
batchSize: number,
|
|
178
175
|
): Promise<void> {
|
|
179
176
|
const batch = records.slice(startIdx, startIdx + batchSize);
|
|
180
177
|
if (batch.length === 0) return;
|
|
@@ -188,10 +185,7 @@ async function insertTradeBatch(
|
|
|
188
185
|
for (let rowIdx = 0; rowIdx < batch.length; rowIdx++) {
|
|
189
186
|
const record = batch[rowIdx];
|
|
190
187
|
const baseParam = rowIdx * columnsPerRow + 1;
|
|
191
|
-
const rowPlaceholders = Array.from(
|
|
192
|
-
{ length: columnsPerRow },
|
|
193
|
-
(_, i) => `$${baseParam + i}`
|
|
194
|
-
);
|
|
188
|
+
const rowPlaceholders = Array.from({ length: columnsPerRow }, (_, i) => `$${baseParam + i}`);
|
|
195
189
|
placeholders.push(`(${rowPlaceholders.join(", ")})`);
|
|
196
190
|
|
|
197
191
|
// Parse numeric values safely
|
|
@@ -219,7 +213,7 @@ async function insertTradeBatch(
|
|
|
219
213
|
isNaN(marginReq) ? null : marginReq, // margin_req
|
|
220
214
|
isNaN(openingCommissions) ? 0 : openingCommissions, // opening_commissions
|
|
221
215
|
isNaN(closingCommissions) ? 0 : closingCommissions, // closing_commissions
|
|
222
|
-
ticker // ticker
|
|
216
|
+
ticker, // ticker
|
|
223
217
|
);
|
|
224
218
|
}
|
|
225
219
|
|
|
@@ -264,7 +258,7 @@ async function insertReportingBatch(
|
|
|
264
258
|
trades: ReportingTrade[],
|
|
265
259
|
tickers: string[],
|
|
266
260
|
startIdx: number,
|
|
267
|
-
batchSize: number
|
|
261
|
+
batchSize: number,
|
|
268
262
|
): Promise<void> {
|
|
269
263
|
const batch = trades.slice(startIdx, startIdx + batchSize);
|
|
270
264
|
const batchTickers = tickers.slice(startIdx, startIdx + batchSize);
|
|
@@ -278,10 +272,7 @@ async function insertReportingBatch(
|
|
|
278
272
|
for (let rowIdx = 0; rowIdx < batch.length; rowIdx++) {
|
|
279
273
|
const trade = batch[rowIdx];
|
|
280
274
|
const baseParam = rowIdx * columnsPerRow + 1;
|
|
281
|
-
const rowPlaceholders = Array.from(
|
|
282
|
-
{ length: columnsPerRow },
|
|
283
|
-
(_, i) => `$${baseParam + i}`
|
|
284
|
-
);
|
|
275
|
+
const rowPlaceholders = Array.from({ length: columnsPerRow }, (_, i) => `$${baseParam + i}`);
|
|
285
276
|
placeholders.push(`(${rowPlaceholders.join(", ")})`);
|
|
286
277
|
|
|
287
278
|
params.push(
|
|
@@ -299,7 +290,7 @@ async function insertReportingBatch(
|
|
|
299
290
|
trade.avgClosingCost ?? null,
|
|
300
291
|
trade.reasonForClose || null,
|
|
301
292
|
trade.openingPrice ?? null,
|
|
302
|
-
batchTickers[rowIdx]
|
|
293
|
+
batchTickers[rowIdx],
|
|
303
294
|
);
|
|
304
295
|
}
|
|
305
296
|
|
|
@@ -347,7 +338,7 @@ function versionedHash(hash: string): string {
|
|
|
347
338
|
export async function syncBlockInternal(
|
|
348
339
|
conn: DuckDBConnection,
|
|
349
340
|
blockId: string,
|
|
350
|
-
blockPath: string
|
|
341
|
+
blockPath: string,
|
|
351
342
|
): Promise<BlockSyncResult> {
|
|
352
343
|
const blocksDir = path.dirname(blockPath);
|
|
353
344
|
try {
|
|
@@ -361,14 +352,8 @@ export async function syncBlockInternal(
|
|
|
361
352
|
if (existingMetadata) {
|
|
362
353
|
await conn.run("BEGIN TRANSACTION");
|
|
363
354
|
try {
|
|
364
|
-
await conn.run(
|
|
365
|
-
|
|
366
|
-
[blockId]
|
|
367
|
-
);
|
|
368
|
-
await conn.run(
|
|
369
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
370
|
-
[blockId]
|
|
371
|
-
);
|
|
355
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
356
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
372
357
|
await deleteSyncMetadata(conn, blockId, blocksDir);
|
|
373
358
|
await conn.run("COMMIT");
|
|
374
359
|
return { blockId, status: "deleted" };
|
|
@@ -419,10 +404,7 @@ export async function syncBlockInternal(
|
|
|
419
404
|
|
|
420
405
|
try {
|
|
421
406
|
// Delete old trade data for this block
|
|
422
|
-
await conn.run(
|
|
423
|
-
"DELETE FROM trades.trade_data WHERE block_id = $1",
|
|
424
|
-
[blockId]
|
|
425
|
-
);
|
|
407
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
426
408
|
|
|
427
409
|
// Read and parse CSV
|
|
428
410
|
const csvContent = await fs.readFile(tradelogPath, "utf-8");
|
|
@@ -441,9 +423,9 @@ export async function syncBlockInternal(
|
|
|
441
423
|
|
|
442
424
|
if (optionalLogs.dailylog) {
|
|
443
425
|
try {
|
|
444
|
-
dailylogHash = versionedHash(
|
|
445
|
-
path.join(blockPath, optionalLogs.dailylog)
|
|
446
|
-
)
|
|
426
|
+
dailylogHash = versionedHash(
|
|
427
|
+
await hashFileContent(path.join(blockPath, optionalLogs.dailylog)),
|
|
428
|
+
);
|
|
447
429
|
} catch {
|
|
448
430
|
// Dailylog file can't be read, leave hash null
|
|
449
431
|
}
|
|
@@ -451,9 +433,9 @@ export async function syncBlockInternal(
|
|
|
451
433
|
|
|
452
434
|
if (optionalLogs.reportinglog) {
|
|
453
435
|
try {
|
|
454
|
-
reportinglogHash = versionedHash(
|
|
455
|
-
path.join(blockPath, optionalLogs.reportinglog)
|
|
456
|
-
)
|
|
436
|
+
reportinglogHash = versionedHash(
|
|
437
|
+
await hashFileContent(path.join(blockPath, optionalLogs.reportinglog)),
|
|
438
|
+
);
|
|
457
439
|
} catch {
|
|
458
440
|
// Reportinglog file can't be read, leave hash null
|
|
459
441
|
}
|
|
@@ -461,10 +443,7 @@ export async function syncBlockInternal(
|
|
|
461
443
|
|
|
462
444
|
// Sync reporting log if it exists and has changed
|
|
463
445
|
// Always delete old reporting data for this block (same pattern as trade_data)
|
|
464
|
-
await conn.run(
|
|
465
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
466
|
-
[blockId]
|
|
467
|
-
);
|
|
446
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
468
447
|
|
|
469
448
|
if (optionalLogs.reportinglog && reportinglogHash) {
|
|
470
449
|
// Read and parse reporting CSV, then convert to ReportingTrade objects.
|
|
@@ -485,7 +464,14 @@ export async function syncBlockInternal(
|
|
|
485
464
|
|
|
486
465
|
// Insert reporting trades in batches of 500
|
|
487
466
|
for (let i = 0; i < reportingTrades.length; i += batchSize) {
|
|
488
|
-
await insertReportingBatch(
|
|
467
|
+
await insertReportingBatch(
|
|
468
|
+
conn,
|
|
469
|
+
blockId,
|
|
470
|
+
reportingTrades,
|
|
471
|
+
reportingTickers,
|
|
472
|
+
i,
|
|
473
|
+
batchSize,
|
|
474
|
+
);
|
|
489
475
|
}
|
|
490
476
|
}
|
|
491
477
|
|
|
@@ -517,14 +503,8 @@ export async function syncBlockInternal(
|
|
|
517
503
|
if (existingMetadata) {
|
|
518
504
|
try {
|
|
519
505
|
await conn.run("BEGIN TRANSACTION");
|
|
520
|
-
await conn.run(
|
|
521
|
-
|
|
522
|
-
[blockId]
|
|
523
|
-
);
|
|
524
|
-
await conn.run(
|
|
525
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
526
|
-
[blockId]
|
|
527
|
-
);
|
|
506
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
507
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
528
508
|
await deleteSyncMetadata(conn, blockId, blocksDir);
|
|
529
509
|
await conn.run("COMMIT");
|
|
530
510
|
} catch {
|
|
@@ -562,7 +542,7 @@ export async function syncBlockInternal(
|
|
|
562
542
|
*/
|
|
563
543
|
export async function detectBlockChanges(
|
|
564
544
|
conn: DuckDBConnection,
|
|
565
|
-
baseDir: string
|
|
545
|
+
baseDir: string,
|
|
566
546
|
): Promise<{ toSync: string[]; toDelete: string[] }> {
|
|
567
547
|
const toSync: string[] = [];
|
|
568
548
|
const toDelete: string[] = [];
|
|
@@ -577,7 +557,12 @@ export async function detectBlockChanges(
|
|
|
577
557
|
for (const entry of entries) {
|
|
578
558
|
if (!entry.isDirectory()) continue;
|
|
579
559
|
if (entry.name.startsWith(".") || entry.name.startsWith("_")) continue; // Skip hidden/internal folders
|
|
580
|
-
if (
|
|
560
|
+
if (
|
|
561
|
+
entry.name.endsWith(".tmp") ||
|
|
562
|
+
entry.name.endsWith(".duckdb") ||
|
|
563
|
+
entry.name.endsWith(".duckdb.tmp")
|
|
564
|
+
)
|
|
565
|
+
continue; // Skip DuckDB/temp files
|
|
581
566
|
|
|
582
567
|
const blockId = entry.name;
|
|
583
568
|
folderNames.add(blockId);
|
|
@@ -651,19 +636,13 @@ export async function detectBlockChanges(
|
|
|
651
636
|
export async function cleanupDeletedBlocks(
|
|
652
637
|
conn: DuckDBConnection,
|
|
653
638
|
deletedBlockIds: string[],
|
|
654
|
-
blocksDir?: string
|
|
639
|
+
blocksDir?: string,
|
|
655
640
|
): Promise<void> {
|
|
656
641
|
for (const blockId of deletedBlockIds) {
|
|
657
642
|
await conn.run("BEGIN TRANSACTION");
|
|
658
643
|
try {
|
|
659
|
-
await conn.run(
|
|
660
|
-
|
|
661
|
-
[blockId]
|
|
662
|
-
);
|
|
663
|
-
await conn.run(
|
|
664
|
-
"DELETE FROM trades.reporting_data WHERE block_id = $1",
|
|
665
|
-
[blockId]
|
|
666
|
-
);
|
|
644
|
+
await conn.run("DELETE FROM trades.trade_data WHERE block_id = $1", [blockId]);
|
|
645
|
+
await conn.run("DELETE FROM trades.reporting_data WHERE block_id = $1", [blockId]);
|
|
667
646
|
await deleteSyncMetadata(conn, blockId, blocksDir);
|
|
668
647
|
await conn.run("COMMIT");
|
|
669
648
|
} catch (err) {
|
package/src/sync/index.ts
CHANGED
|
@@ -133,10 +133,7 @@ export async function syncAllBlocks(baseDir: string): Promise<SyncResult> {
|
|
|
133
133
|
* @param baseDir - Base data directory containing block folders
|
|
134
134
|
* @returns Sync result for the block
|
|
135
135
|
*/
|
|
136
|
-
export async function syncBlock(
|
|
137
|
-
blockId: string,
|
|
138
|
-
baseDir: string
|
|
139
|
-
): Promise<BlockSyncResult> {
|
|
136
|
+
export async function syncBlock(blockId: string, baseDir: string): Promise<BlockSyncResult> {
|
|
140
137
|
const conn = await getConnection(baseDir);
|
|
141
138
|
const blocksDir = getBlocksDir(baseDir);
|
|
142
139
|
const blockPath = path.join(blocksDir, blockId);
|
package/src/sync/metadata.ts
CHANGED
|
@@ -28,7 +28,6 @@ export interface BlockSyncMetadata {
|
|
|
28
28
|
sync_version: number;
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
-
|
|
32
31
|
/**
|
|
33
32
|
* Get sync metadata for a block.
|
|
34
33
|
*
|
|
@@ -39,7 +38,7 @@ export interface BlockSyncMetadata {
|
|
|
39
38
|
export async function getSyncMetadata(
|
|
40
39
|
conn: DuckDBConnection,
|
|
41
40
|
blockId: string,
|
|
42
|
-
blocksDir?: string
|
|
41
|
+
blocksDir?: string,
|
|
43
42
|
): Promise<BlockSyncMetadata | null> {
|
|
44
43
|
if (isParquetMode() && blocksDir) {
|
|
45
44
|
return getSyncMetadataJson(blockId, blocksDir);
|
|
@@ -48,7 +47,7 @@ export async function getSyncMetadata(
|
|
|
48
47
|
`SELECT block_id, tradelog_hash, dailylog_hash, reportinglog_hash, synced_at, sync_version
|
|
49
48
|
FROM trades._sync_metadata
|
|
50
49
|
WHERE block_id = $1`,
|
|
51
|
-
[blockId]
|
|
50
|
+
[blockId],
|
|
52
51
|
);
|
|
53
52
|
|
|
54
53
|
const rows = reader.getRows();
|
|
@@ -76,7 +75,7 @@ export async function getSyncMetadata(
|
|
|
76
75
|
export async function upsertSyncMetadata(
|
|
77
76
|
conn: DuckDBConnection,
|
|
78
77
|
metadata: BlockSyncMetadata,
|
|
79
|
-
blocksDir?: string
|
|
78
|
+
blocksDir?: string,
|
|
80
79
|
): Promise<void> {
|
|
81
80
|
if (isParquetMode() && blocksDir) {
|
|
82
81
|
return upsertSyncMetadataJson(metadata, blocksDir);
|
|
@@ -92,7 +91,7 @@ export async function upsertSyncMetadata(
|
|
|
92
91
|
metadata.reportinglog_hash,
|
|
93
92
|
metadata.synced_at.toISOString(),
|
|
94
93
|
metadata.sync_version,
|
|
95
|
-
]
|
|
94
|
+
],
|
|
96
95
|
);
|
|
97
96
|
}
|
|
98
97
|
|
|
@@ -105,16 +104,13 @@ export async function upsertSyncMetadata(
|
|
|
105
104
|
export async function deleteSyncMetadata(
|
|
106
105
|
conn: DuckDBConnection,
|
|
107
106
|
blockId: string,
|
|
108
|
-
blocksDir?: string
|
|
107
|
+
blocksDir?: string,
|
|
109
108
|
): Promise<void> {
|
|
110
109
|
if (isParquetMode() && blocksDir) {
|
|
111
110
|
await deleteSyncMetadataJson(blockId, blocksDir);
|
|
112
111
|
return;
|
|
113
112
|
}
|
|
114
|
-
await conn.run(
|
|
115
|
-
`DELETE FROM trades._sync_metadata WHERE block_id = $1`,
|
|
116
|
-
[blockId]
|
|
117
|
-
);
|
|
113
|
+
await conn.run(`DELETE FROM trades._sync_metadata WHERE block_id = $1`, [blockId]);
|
|
118
114
|
}
|
|
119
115
|
|
|
120
116
|
/**
|
|
@@ -125,14 +121,12 @@ export async function deleteSyncMetadata(
|
|
|
125
121
|
*/
|
|
126
122
|
export async function getAllSyncedBlockIds(
|
|
127
123
|
conn: DuckDBConnection,
|
|
128
|
-
blocksDir?: string
|
|
124
|
+
blocksDir?: string,
|
|
129
125
|
): Promise<string[]> {
|
|
130
126
|
if (isParquetMode() && blocksDir) {
|
|
131
127
|
return getAllSyncedBlockIdsJson(blocksDir);
|
|
132
128
|
}
|
|
133
|
-
const reader = await conn.runAndReadAll(
|
|
134
|
-
`SELECT block_id FROM trades._sync_metadata`
|
|
135
|
-
);
|
|
129
|
+
const reader = await conn.runAndReadAll(`SELECT block_id FROM trades._sync_metadata`);
|
|
136
130
|
|
|
137
131
|
const rows = reader.getRows();
|
|
138
132
|
return rows.map((row) => row[0] as string);
|
|
@@ -150,8 +144,8 @@ export async function getAllSyncedBlockIds(
|
|
|
150
144
|
* Use these helpers for all Phase 61+ import tool calls.
|
|
151
145
|
*/
|
|
152
146
|
export interface MarketImportMetadata {
|
|
153
|
-
source: string;
|
|
154
|
-
ticker: string;
|
|
147
|
+
source: string; // e.g., "import_market_csv:/abs/path/to/file.csv"
|
|
148
|
+
ticker: string; // Normalized ticker e.g. "SPX"
|
|
155
149
|
target_table: string; // "daily" | "date_context" | "intraday" | other canonical datasets
|
|
156
150
|
max_date: string | null;
|
|
157
151
|
synced_at: Date;
|
|
@@ -165,7 +159,7 @@ export async function getMarketImportMetadata(
|
|
|
165
159
|
source: string,
|
|
166
160
|
ticker: string,
|
|
167
161
|
targetTable: string,
|
|
168
|
-
dataDir?: string
|
|
162
|
+
dataDir?: string,
|
|
169
163
|
): Promise<MarketImportMetadata | null> {
|
|
170
164
|
if (isParquetMode() && dataDir) {
|
|
171
165
|
return getMarketImportMetadataJson(source, ticker, targetTable, dataDir);
|
|
@@ -174,7 +168,7 @@ export async function getMarketImportMetadata(
|
|
|
174
168
|
`SELECT source, ticker, target_table, max_date, synced_at
|
|
175
169
|
FROM market._sync_metadata
|
|
176
170
|
WHERE source = $1 AND ticker = $2 AND target_table = $3`,
|
|
177
|
-
[source, ticker, targetTable]
|
|
171
|
+
[source, ticker, targetTable],
|
|
178
172
|
);
|
|
179
173
|
const rows = reader.getRows();
|
|
180
174
|
if (rows.length === 0) return null;
|
|
@@ -195,7 +189,7 @@ export async function getMarketImportMetadata(
|
|
|
195
189
|
export async function upsertMarketImportMetadata(
|
|
196
190
|
conn: DuckDBConnection,
|
|
197
191
|
metadata: MarketImportMetadata,
|
|
198
|
-
dataDir?: string
|
|
192
|
+
dataDir?: string,
|
|
199
193
|
): Promise<void> {
|
|
200
194
|
if (isParquetMode() && dataDir) {
|
|
201
195
|
return upsertMarketImportMetadataJson(metadata, dataDir);
|
|
@@ -213,6 +207,6 @@ export async function upsertMarketImportMetadata(
|
|
|
213
207
|
metadata.target_table,
|
|
214
208
|
metadata.max_date,
|
|
215
209
|
metadata.synced_at.toISOString(),
|
|
216
|
-
]
|
|
210
|
+
],
|
|
217
211
|
);
|
|
218
212
|
}
|