tradeblocks-mcp 3.0.2 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -80
- package/dist/{chunk-LDKTV7GW.js → chunk-27S67XW3.js} +24 -52
- package/dist/chunk-27S67XW3.js.map +1 -0
- package/dist/{chunk-FGZH632F.js → chunk-2E63THNI.js} +13 -11
- package/dist/chunk-2E63THNI.js.map +1 -0
- package/dist/{chunk-QTTR7AAW.js → chunk-2I3S2ZLT.js} +325 -84
- package/dist/chunk-2I3S2ZLT.js.map +1 -0
- package/dist/{chunk-4BLCXNQ6.js → chunk-4HZLVUOZ.js} +179 -183
- package/dist/chunk-4HZLVUOZ.js.map +1 -0
- package/dist/{chunk-XXYOUIZY.js → chunk-NWNSKALN.js} +63 -80
- package/dist/chunk-NWNSKALN.js.map +1 -0
- package/dist/{chunk-PRAYH3RT.js → chunk-WNTYA7ER.js} +29 -12
- package/dist/chunk-WNTYA7ER.js.map +1 -0
- package/dist/{chunk-BKQ4PM4Y.js → chunk-YZA4RS76.js} +24 -18
- package/dist/chunk-YZA4RS76.js.map +1 -0
- package/dist/{chunk-W2PP3LEH.js → chunk-ZTJXC3LS.js} +17 -8
- package/dist/chunk-ZTJXC3LS.js.map +1 -0
- package/dist/daily-log-processor-MZW2XBZY.js +9 -0
- package/dist/{daily-logs-store-D3QC2XPV.js → daily-logs-store-3C3OTGPK.js} +2 -2
- package/dist/iv-solver-worker.js +9 -1
- package/dist/market-provider-6U33TQUT.js +16 -0
- package/dist/{proto-3HZTCWK4.js → proto-V23GIZG2.js} +2 -2
- package/dist/{sync-PO4IPCYV.js → sync-3UBC37VW.js} +7 -7
- package/dist/test-exports.js +876 -483
- package/dist/test-exports.js.map +1 -1
- package/dist/trade-processor-DKDCWGKD.js +9 -0
- package/dist/{trades-store-LOL3FQSF.js → trades-store-R3NL25T2.js} +2 -2
- package/package.json +2 -6
- package/server/{chunk-CSDVJPBB.js → chunk-536NSFED.js} +29 -12
- package/server/chunk-536NSFED.js.map +1 -0
- package/server/{chunk-WA5AAPCH.js → chunk-A3UKD64A.js} +17 -8
- package/server/chunk-A3UKD64A.js.map +1 -0
- package/server/{chunk-PNKG7RY7.js → chunk-OMUDNJBJ.js} +63 -80
- package/server/chunk-OMUDNJBJ.js.map +1 -0
- package/server/{chunk-72GKJE2U.js → chunk-RJDJHZ5Y.js} +934 -335
- package/server/chunk-RJDJHZ5Y.js.map +1 -0
- package/server/{chunk-NZO6PT64.js → chunk-RM4B2VKS.js} +167 -177
- package/server/chunk-RM4B2VKS.js.map +1 -0
- package/server/{chunk-5EBXHT6C.js → chunk-SY6GDNVI.js} +24 -18
- package/server/chunk-SY6GDNVI.js.map +1 -0
- package/server/{chunk-FBNDMCT5.js → chunk-T66KH2XH.js} +13 -11
- package/server/chunk-T66KH2XH.js.map +1 -0
- package/server/{chunk-NRFXAJF7.js → chunk-W5E7FHC4.js} +24 -52
- package/server/chunk-W5E7FHC4.js.map +1 -0
- package/server/{config-6IZXEFEX.js → config-DK7KOMNL.js} +1 -1
- package/server/config-DK7KOMNL.js.map +1 -0
- package/server/daily-log-processor-B2VPU2FQ.js +10 -0
- package/server/{daily-logs-store-NU7M4L5C.js → daily-logs-store-YFNXTB7J.js} +2 -2
- package/server/http-server.js +2 -5
- package/server/index.js +2497 -2029
- package/server/index.js.map +1 -1
- package/server/iv-solver-worker.js +9 -1
- package/server/market-provider-L56RFHAC.js +17 -0
- package/server/{proto-DWRZJO4E.js → proto-YZ4TLX2N.js} +2 -2
- package/server/{sync-QFI5L7S7.js → sync-CENKDN53.js} +7 -7
- package/server/trade-processor-3PI4LHZI.js +10 -0
- package/server/{trades-store-VY26MWER.js → trades-store-RTTEYMXH.js} +2 -2
- package/src/auth/clients-store.ts +4 -4
- package/src/auth/code-store.ts +1 -1
- package/src/auth/config.ts +11 -12
- package/src/auth/login-page.ts +8 -10
- package/src/auth/provider.ts +35 -30
- package/src/auth/token.ts +17 -15
- package/src/db/backtest-schemas.ts +38 -8
- package/src/db/connection.ts +107 -40
- package/src/db/data-root.ts +22 -0
- package/src/db/index.ts +37 -5
- package/src/db/json-adapters.ts +22 -31
- package/src/db/json-migration.ts +11 -15
- package/src/db/json-store.ts +5 -3
- package/src/db/market-datasets.ts +48 -15
- package/src/db/market-schemas.ts +11 -5
- package/src/db/market-views.ts +119 -32
- package/src/db/parquet-writer.ts +7 -7
- package/src/db/profile-schemas.ts +39 -22
- package/src/db/schemas.ts +2 -2
- package/src/http-server.ts +7 -12
- package/src/index.ts +8 -9
- package/src/market/ingestor/index.ts +1 -4
- package/src/market/ingestor/market-ingestor.ts +140 -67
- package/src/market/ingestor/types.ts +1 -3
- package/src/market/stores/chain-sql.ts +2 -8
- package/src/market/stores/chain-store.ts +1 -4
- package/src/market/stores/duckdb-chain-store.ts +3 -14
- package/src/market/stores/duckdb-enriched-store.ts +3 -13
- package/src/market/stores/duckdb-quote-store.ts +10 -27
- package/src/market/stores/duckdb-spot-store.ts +4 -20
- package/src/market/stores/enriched-sql.ts +3 -9
- package/src/market/stores/enriched-store.ts +2 -2
- package/src/market/stores/parquet-chain-store.ts +5 -22
- package/src/market/stores/parquet-enriched-store.ts +4 -16
- package/src/market/stores/parquet-oi-daily-store.ts +3 -14
- package/src/market/stores/parquet-quote-store.ts +9 -29
- package/src/market/stores/parquet-spot-store.ts +14 -33
- package/src/market/stores/quote-store.ts +2 -10
- package/src/market/stores/spot-sql.ts +3 -15
- package/src/market/stores/spot-store.ts +1 -1
- package/src/market/tickers/defaults.json +6 -6
- package/src/market/tickers/loader.ts +2 -7
- package/src/market/tickers/registry.ts +5 -15
- package/src/market/tickers/schemas.ts +2 -6
- package/src/models/strategy-profile.ts +39 -39
- package/src/sync/block-sync.ts +44 -65
- package/src/sync/index.ts +1 -4
- package/src/sync/metadata.ts +14 -20
- package/src/test-exports.ts +139 -123
- package/src/tools/analysis.ts +180 -159
- package/src/tools/batch-exit-analysis.ts +100 -59
- package/src/tools/blocks/analysis.ts +116 -101
- package/src/tools/blocks/comparison.ts +281 -364
- package/src/tools/blocks/core.ts +208 -253
- package/src/tools/blocks/health.ts +571 -563
- package/src/tools/blocks/index.ts +2 -0
- package/src/tools/blocks/paired-comparison.ts +425 -0
- package/src/tools/blocks/similarity.ts +316 -283
- package/src/tools/edge-decay.ts +124 -197
- package/src/tools/exit-analysis.ts +130 -77
- package/src/tools/greeks-attribution.ts +84 -35
- package/src/tools/guides.ts +4 -6
- package/src/tools/imports.ts +11 -14
- package/src/tools/market-data.ts +1067 -757
- package/src/tools/market-enrichment.ts +3 -3
- package/src/tools/market-fetch.ts +148 -67
- package/src/tools/market-imports.ts +12 -12
- package/src/tools/middleware/sync-middleware.ts +5 -6
- package/src/tools/performance.ts +185 -302
- package/src/tools/profile-analysis.ts +52 -66
- package/src/tools/profiles.ts +106 -69
- package/src/tools/regime-advisor.ts +20 -45
- package/src/tools/replay.ts +81 -77
- package/src/tools/reports/discrepancies.ts +298 -328
- package/src/tools/reports/fields.ts +7 -25
- package/src/tools/reports/helpers.ts +18 -49
- package/src/tools/reports/predictive.ts +27 -70
- package/src/tools/reports/slippage-trends.ts +315 -345
- package/src/tools/reports/slippage.ts +1 -4
- package/src/tools/reports/strategy-matches.ts +399 -441
- package/src/tools/schema.ts +43 -40
- package/src/tools/shared/filters.ts +3 -9
- package/src/tools/snapshot.ts +9 -30
- package/src/tools/sql.ts +15 -14
- package/src/tools/tickers.ts +1 -4
- package/src/utils/batch-exit-analysis.ts +31 -42
- package/src/utils/black-scholes.ts +39 -29
- package/src/utils/block-loader.ts +69 -83
- package/src/utils/calibration-probe.ts +3 -4
- package/src/utils/chain-loader.ts +3 -3
- package/src/utils/csv-discovery.ts +16 -22
- package/src/utils/data-quality.ts +24 -36
- package/src/utils/exit-triggers.ts +91 -96
- package/src/utils/field-timing.ts +94 -79
- package/src/utils/filter-predicates.ts +13 -9
- package/src/utils/flatfile-importer.ts +94 -64
- package/src/utils/greeks-decomposition.ts +152 -100
- package/src/utils/iv-solver-pool.ts +55 -25
- package/src/utils/iv-solver-worker.ts +5 -5
- package/src/utils/market-enricher.ts +528 -497
- package/src/utils/market-importer.ts +31 -12
- package/src/utils/market-provider.ts +21 -23
- package/src/utils/massive-tier.ts +5 -7
- package/src/utils/migrate-option-data-helpers.ts +2 -8
- package/src/utils/option-quote-greeks.ts +25 -31
- package/src/utils/option-time.ts +4 -8
- package/src/utils/output-formatter.ts +1 -4
- package/src/utils/provider-capabilities.ts +1 -4
- package/src/utils/providers/massive.ts +59 -93
- package/src/utils/providers/thetadata/backfill.ts +14 -23
- package/src/utils/providers/thetadata/client.ts +12 -8
- package/src/utils/providers/thetadata/decode.ts +2 -20
- package/src/utils/providers/thetadata/endpoints.ts +32 -40
- package/src/utils/providers/thetadata/join.ts +11 -10
- package/src/utils/providers/thetadata/proto.ts +12 -10
- package/src/utils/providers/thetadata/quote-mid-greeks.ts +5 -5
- package/src/utils/providers/thetadata.ts +11 -10
- package/src/utils/quote-enricher.ts +4 -4
- package/src/utils/quote-parquet-projection.ts +3 -11
- package/src/utils/sample-date-selector.ts +3 -5
- package/src/utils/schema-metadata.ts +102 -70
- package/src/utils/ticker.ts +5 -9
- package/src/utils/trade-replay.ts +77 -68
- package/dist/chunk-4BLCXNQ6.js.map +0 -1
- package/dist/chunk-BKQ4PM4Y.js.map +0 -1
- package/dist/chunk-FGZH632F.js.map +0 -1
- package/dist/chunk-LDKTV7GW.js.map +0 -1
- package/dist/chunk-PRAYH3RT.js.map +0 -1
- package/dist/chunk-QTTR7AAW.js.map +0 -1
- package/dist/chunk-W2PP3LEH.js.map +0 -1
- package/dist/chunk-XXYOUIZY.js.map +0 -1
- package/dist/daily-log-processor-6MWJ23JK.js +0 -9
- package/dist/market-provider-VDRJUEF2.js +0 -16
- package/dist/trade-processor-NHU4VWRX.js +0 -9
- package/manifest.json +0 -141
- package/server/chunk-5EBXHT6C.js.map +0 -1
- package/server/chunk-72GKJE2U.js.map +0 -1
- package/server/chunk-CSDVJPBB.js.map +0 -1
- package/server/chunk-FBNDMCT5.js.map +0 -1
- package/server/chunk-NRFXAJF7.js.map +0 -1
- package/server/chunk-NZO6PT64.js.map +0 -1
- package/server/chunk-PNKG7RY7.js.map +0 -1
- package/server/chunk-WA5AAPCH.js.map +0 -1
- package/server/config-6IZXEFEX.js.map +0 -1
- package/server/daily-log-processor-Y3PVSVBM.js +0 -10
- package/server/market-provider-VOYYVYWT.js +0 -17
- package/server/trade-processor-JWVS37KM.js +0 -10
- /package/dist/{daily-log-processor-6MWJ23JK.js.map → daily-log-processor-MZW2XBZY.js.map} +0 -0
- /package/dist/{daily-logs-store-D3QC2XPV.js.map → daily-logs-store-3C3OTGPK.js.map} +0 -0
- /package/dist/{market-provider-VDRJUEF2.js.map → market-provider-6U33TQUT.js.map} +0 -0
- /package/dist/{proto-3HZTCWK4.js.map → proto-V23GIZG2.js.map} +0 -0
- /package/dist/{sync-PO4IPCYV.js.map → sync-3UBC37VW.js.map} +0 -0
- /package/dist/{trade-processor-NHU4VWRX.js.map → trade-processor-DKDCWGKD.js.map} +0 -0
- /package/dist/{trades-store-LOL3FQSF.js.map → trades-store-R3NL25T2.js.map} +0 -0
- /package/server/{daily-log-processor-Y3PVSVBM.js.map → daily-log-processor-B2VPU2FQ.js.map} +0 -0
- /package/server/{daily-logs-store-NU7M4L5C.js.map → daily-logs-store-YFNXTB7J.js.map} +0 -0
- /package/server/{market-provider-VOYYVYWT.js.map → market-provider-L56RFHAC.js.map} +0 -0
- /package/server/{proto-DWRZJO4E.js.map → proto-YZ4TLX2N.js.map} +0 -0
- /package/server/{sync-QFI5L7S7.js.map → sync-CENKDN53.js.map} +0 -0
- /package/server/{trade-processor-JWVS37KM.js.map → trade-processor-3PI4LHZI.js.map} +0 -0
- /package/server/{trades-store-VY26MWER.js.map → trades-store-RTTEYMXH.js.map} +0 -0
|
@@ -20,10 +20,7 @@ import { existsSync, readdirSync } from "fs";
|
|
|
20
20
|
import * as path from "path";
|
|
21
21
|
import { isParquetMode, writeParquetAtomic } from "../db/parquet-writer.ts";
|
|
22
22
|
import { resolveCanonicalMarketFile, resolveMarketDir } from "../db/market-datasets.ts";
|
|
23
|
-
import {
|
|
24
|
-
getEnrichedThrough,
|
|
25
|
-
upsertEnrichedThrough,
|
|
26
|
-
} from "../db/json-adapters.ts";
|
|
23
|
+
import { getEnrichedThrough, upsertEnrichedThrough } from "../db/json-adapters.ts";
|
|
27
24
|
import { DEFAULT_MARKET_TICKER } from "./ticker.ts";
|
|
28
25
|
import type { SpotStore } from "../market/stores/spot-store.ts";
|
|
29
26
|
|
|
@@ -121,7 +118,7 @@ export function computeATR(
|
|
|
121
118
|
highs: number[],
|
|
122
119
|
lows: number[],
|
|
123
120
|
closes: number[],
|
|
124
|
-
period = 14
|
|
121
|
+
period = 14,
|
|
125
122
|
): number[] {
|
|
126
123
|
const n = closes.length;
|
|
127
124
|
const result = new Array<number>(n).fill(NaN);
|
|
@@ -134,7 +131,7 @@ export function computeATR(
|
|
|
134
131
|
tr[i] = Math.max(
|
|
135
132
|
highs[i] - lows[i],
|
|
136
133
|
Math.abs(highs[i] - prevClose),
|
|
137
|
-
Math.abs(lows[i] - prevClose)
|
|
134
|
+
Math.abs(lows[i] - prevClose),
|
|
138
135
|
);
|
|
139
136
|
}
|
|
140
137
|
|
|
@@ -281,12 +278,7 @@ export function computeConsecutiveDays(closes: number[]): number[] {
|
|
|
281
278
|
* Gap down (open < priorClose): filled if high >= priorClose
|
|
282
279
|
* No gap (open = priorClose): returns 0
|
|
283
280
|
*/
|
|
284
|
-
export function isGapFilled(
|
|
285
|
-
open: number,
|
|
286
|
-
high: number,
|
|
287
|
-
low: number,
|
|
288
|
-
priorClose: number
|
|
289
|
-
): number {
|
|
281
|
+
export function isGapFilled(open: number, high: number, low: number, priorClose: number): number {
|
|
290
282
|
if (open > priorClose && low <= priorClose) return 1;
|
|
291
283
|
if (open < priorClose && high >= priorClose) return 1;
|
|
292
284
|
return 0;
|
|
@@ -451,7 +443,7 @@ export function classifyVolRegime(vixClose: number): number {
|
|
|
451
443
|
export function classifyTermStructure(
|
|
452
444
|
vix9dClose: number,
|
|
453
445
|
vixClose: number,
|
|
454
|
-
vix3mClose: number
|
|
446
|
+
vix3mClose: number,
|
|
455
447
|
): number {
|
|
456
448
|
// Match PineScript: vix9dClose > vixClose ? -1 : vixClose > vix3mClose ? 0 : 1
|
|
457
449
|
if (vix9dClose > vixClose) return -1;
|
|
@@ -470,7 +462,8 @@ export function computeIVR(values: number[], period = 252): number[] {
|
|
|
470
462
|
const n = values.length;
|
|
471
463
|
const result = new Array<number>(n).fill(NaN);
|
|
472
464
|
for (let i = period - 1; i < n; i++) {
|
|
473
|
-
let min = Infinity,
|
|
465
|
+
let min = Infinity,
|
|
466
|
+
max = -Infinity;
|
|
474
467
|
for (let j = i - period + 1; j <= i; j++) {
|
|
475
468
|
if (values[j] < min) min = values[j];
|
|
476
469
|
if (values[j] > max) max = values[j];
|
|
@@ -508,7 +501,7 @@ export function computeIVP(values: number[], period = 252): number[] {
|
|
|
508
501
|
|
|
509
502
|
export interface EnrichmentOptions {
|
|
510
503
|
forceFull?: boolean;
|
|
511
|
-
dataDir?: string;
|
|
504
|
+
dataDir?: string; // Required in Parquet mode for file paths
|
|
512
505
|
parquetMode?: boolean;
|
|
513
506
|
}
|
|
514
507
|
|
|
@@ -667,7 +660,9 @@ async function setupParquetWorkingTables(
|
|
|
667
660
|
// ---- Daily working table seed ---------------------------------------------
|
|
668
661
|
if (existsSync(dailyPath)) {
|
|
669
662
|
// Legacy single-file seed
|
|
670
|
-
await conn.run(
|
|
663
|
+
await conn.run(
|
|
664
|
+
`CREATE TEMP TABLE "${dailyTable}" AS SELECT * FROM read_parquet('${dailyPath}')`,
|
|
665
|
+
);
|
|
671
666
|
// Parquet files from fresh imports may lack enrichment columns — add them
|
|
672
667
|
await alignDailyWorkingTableColumns(conn, dailyTable);
|
|
673
668
|
} else if (hasEnrichedTickerFiles(enrichedDir)) {
|
|
@@ -702,9 +697,7 @@ async function setupParquetWorkingTables(
|
|
|
702
697
|
// per-ticker computed-fields view) and ALTER-ADD the OHLCV columns the
|
|
703
698
|
// Tier 1 math expects. Matches the shape used by the
|
|
704
699
|
// enriched-ticker-files branch above.
|
|
705
|
-
await conn.run(
|
|
706
|
-
`CREATE TEMP TABLE "${dailyTable}" AS SELECT * FROM market.enriched WHERE 1=0`,
|
|
707
|
-
);
|
|
700
|
+
await conn.run(`CREATE TEMP TABLE "${dailyTable}" AS SELECT * FROM market.enriched WHERE 1=0`);
|
|
708
701
|
for (const ohlcv of ["open", "high", "low", "close"]) {
|
|
709
702
|
try {
|
|
710
703
|
await conn.run(`ALTER TABLE "${dailyTable}" ADD COLUMN "${ohlcv}" DOUBLE`);
|
|
@@ -758,7 +751,9 @@ async function setupParquetWorkingTables(
|
|
|
758
751
|
// ---- Date-context working table seed -------------------------------------
|
|
759
752
|
if (existsSync(dateContextPath)) {
|
|
760
753
|
// Legacy single-file seed
|
|
761
|
-
await conn.run(
|
|
754
|
+
await conn.run(
|
|
755
|
+
`CREATE TEMP TABLE "${dateContextTable}" AS SELECT * FROM read_parquet('${dateContextPath}')`,
|
|
756
|
+
);
|
|
762
757
|
} else if (existsSync(enrichedContextPath)) {
|
|
763
758
|
// Seed from the per-ticker enriched/context/data.parquet file
|
|
764
759
|
await conn.run(
|
|
@@ -773,7 +768,9 @@ async function setupParquetWorkingTables(
|
|
|
773
768
|
// INSERT OR REPLACE in runTier2 needs a UNIQUE/PRIMARY KEY on `date`. CREATE
|
|
774
769
|
// TABLE AS SELECT does not carry over PK constraints from Parquet (Parquet has
|
|
775
770
|
// no constraints), so attach one explicitly here.
|
|
776
|
-
await conn.run(
|
|
771
|
+
await conn.run(
|
|
772
|
+
`CREATE UNIQUE INDEX "idx_${dateContextTable}_date" ON "${dateContextTable}"(date)`,
|
|
773
|
+
);
|
|
777
774
|
|
|
778
775
|
// Same rationale for the daily working table: batchUpdateDaily uses
|
|
779
776
|
// INSERT OR REPLACE so first-time enrichment of a ticker (whose seed
|
|
@@ -842,12 +839,7 @@ async function flushEnrichedToParquet(
|
|
|
842
839
|
selectQuery: `SELECT ticker, date, ${enrichedColList} FROM "${tables.dailyTable}" WHERE ticker = '${ticker}' ORDER BY date`,
|
|
843
840
|
});
|
|
844
841
|
|
|
845
|
-
const contextFile = path.join(
|
|
846
|
-
resolveMarketDir(dataDir),
|
|
847
|
-
"enriched",
|
|
848
|
-
"context",
|
|
849
|
-
"data.parquet",
|
|
850
|
-
);
|
|
842
|
+
const contextFile = path.join(resolveMarketDir(dataDir), "enriched", "context", "data.parquet");
|
|
851
843
|
await writeParquetAtomic(conn, {
|
|
852
844
|
targetPath: contextFile,
|
|
853
845
|
selectQuery: `SELECT * FROM "${tables.dateContextTable}" ORDER BY date`,
|
|
@@ -924,7 +916,10 @@ async function runTier2(
|
|
|
924
916
|
for (let start = 0; start < bars.length; start += BATCH_SIZE) {
|
|
925
917
|
const batch = bars.slice(start, start + BATCH_SIZE);
|
|
926
918
|
const placeholders = batch
|
|
927
|
-
.map(
|
|
919
|
+
.map(
|
|
920
|
+
(_, i) =>
|
|
921
|
+
`($${i * 6 + 1},$${i * 6 + 2},$${i * 6 + 3},$${i * 6 + 4},$${i * 6 + 5},$${i * 6 + 6})`,
|
|
922
|
+
)
|
|
928
923
|
.join(",");
|
|
929
924
|
const params = batch.flatMap((b) => [b.ticker, b.date, b.open, b.high, b.low, b.close]);
|
|
930
925
|
await conn.run(
|
|
@@ -937,73 +932,75 @@ async function runTier2(
|
|
|
937
932
|
}
|
|
938
933
|
|
|
939
934
|
try {
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
);
|
|
944
|
-
const vixTickers = tickerResult.getRows().map(r => r[0] as string);
|
|
945
|
-
if (vixTickers.length === 0 || !vixTickers.includes('VIX')) {
|
|
946
|
-
return { status: "skipped", reason: "no VIX data — import VIX ticker first" };
|
|
947
|
-
}
|
|
948
|
-
|
|
949
|
-
// Step 2: Compute IVR/IVP for each VIX-family ticker and write to daily table.
|
|
950
|
-
// `market.enriched` no longer carries OHLCV columns (raw bars live in spot/);
|
|
951
|
-
// read close from the OHLCV source: the spotStore-seeded TEMP table when
|
|
952
|
-
// io.spotStore is present, else `market.spot_daily` (RTH-aggregated view).
|
|
953
|
-
const closeSource = spotStore ? effectiveDailyTarget : "market.spot_daily";
|
|
954
|
-
for (const ticker of vixTickers) {
|
|
955
|
-
const closeResult = await conn.runAndReadAll(
|
|
956
|
-
`SELECT date, close FROM ${closeSource} WHERE ticker = $1 AND close IS NOT NULL ORDER BY date ASC`,
|
|
957
|
-
[ticker]
|
|
935
|
+
// Step 1: Discover VIX-family tickers dynamically
|
|
936
|
+
const tickerResult = await conn.runAndReadAll(
|
|
937
|
+
`SELECT DISTINCT ticker FROM ${effectiveDailyTarget} WHERE ticker LIKE 'VIX%' ORDER BY ticker`,
|
|
958
938
|
);
|
|
959
|
-
const
|
|
960
|
-
if (
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
const closes = rows.map(r => r[1] as number);
|
|
964
|
-
const ivrValues = computeIVR(closes, 252);
|
|
965
|
-
const ivpValues = computeIVP(closes, 252);
|
|
939
|
+
const vixTickers = tickerResult.getRows().map((r) => r[0] as string);
|
|
940
|
+
if (vixTickers.length === 0 || !vixTickers.includes("VIX")) {
|
|
941
|
+
return { status: "skipped", reason: "no VIX data — import VIX ticker first" };
|
|
942
|
+
}
|
|
966
943
|
|
|
967
|
-
//
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
944
|
+
// Step 2: Compute IVR/IVP for each VIX-family ticker and write to daily table.
|
|
945
|
+
// `market.enriched` no longer carries OHLCV columns (raw bars live in spot/);
|
|
946
|
+
// read close from the OHLCV source: the spotStore-seeded TEMP table when
|
|
947
|
+
// io.spotStore is present, else `market.spot_daily` (RTH-aggregated view).
|
|
948
|
+
const closeSource = spotStore ? effectiveDailyTarget : "market.spot_daily";
|
|
949
|
+
for (const ticker of vixTickers) {
|
|
950
|
+
const closeResult = await conn.runAndReadAll(
|
|
951
|
+
`SELECT date, close FROM ${closeSource} WHERE ticker = $1 AND close IS NOT NULL ORDER BY date ASC`,
|
|
952
|
+
[ticker],
|
|
953
|
+
);
|
|
954
|
+
const rows = closeResult.getRows();
|
|
955
|
+
if (rows.length === 0) continue;
|
|
973
956
|
|
|
974
|
-
const
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
957
|
+
const dates = rows.map((r) => r[0] as string);
|
|
958
|
+
const closes = rows.map((r) => r[1] as number);
|
|
959
|
+
const ivrValues = computeIVR(closes, 252);
|
|
960
|
+
const ivpValues = computeIVP(closes, 252);
|
|
978
961
|
|
|
979
|
-
|
|
962
|
+
// Batch UPDATE daily table SET ivr, ivp WHERE ticker = ? AND date = ?
|
|
963
|
+
const BATCH_SIZE = 500;
|
|
964
|
+
for (let start = 0; start < dates.length; start += BATCH_SIZE) {
|
|
965
|
+
const batchDates = dates.slice(start, start + BATCH_SIZE);
|
|
966
|
+
const batchIvr = ivrValues.slice(start, start + BATCH_SIZE);
|
|
967
|
+
const batchIvp = ivpValues.slice(start, start + BATCH_SIZE);
|
|
968
|
+
|
|
969
|
+
const placeholders = batchDates
|
|
970
|
+
.map((_, rowIdx) => {
|
|
971
|
+
const base = rowIdx * 3;
|
|
972
|
+
return `($${base + 1}, $${base + 2}, $${base + 3})`;
|
|
973
|
+
})
|
|
974
|
+
.join(", ");
|
|
975
|
+
|
|
976
|
+
const sql = `
|
|
980
977
|
UPDATE ${dailyTarget} AS t
|
|
981
978
|
SET ivr = v.ivr, ivp = v.ivp
|
|
982
979
|
FROM (VALUES ${placeholders}) AS v(date, ivr, ivp)
|
|
983
980
|
WHERE t.ticker = $${batchDates.length * 3 + 1} AND t.date = v.date
|
|
984
981
|
`;
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
982
|
+
const params: (string | number | null)[] = [];
|
|
983
|
+
for (let i = 0; i < batchDates.length; i++) {
|
|
984
|
+
params.push(batchDates[i]);
|
|
985
|
+
params.push(isNaN(batchIvr[i]) ? null : batchIvr[i]);
|
|
986
|
+
params.push(isNaN(batchIvp[i]) ? null : batchIvp[i]);
|
|
987
|
+
}
|
|
988
|
+
params.push(ticker);
|
|
989
|
+
await conn.run(sql, params as (string | number | boolean | null | bigint)[]);
|
|
990
990
|
}
|
|
991
|
-
params.push(ticker);
|
|
992
|
-
await conn.run(sql, params as (string | number | boolean | null | bigint)[]);
|
|
993
991
|
}
|
|
994
|
-
}
|
|
995
992
|
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
993
|
+
// Step 3: Build ContextRow objects from daily VIX tickers for derived fields
|
|
994
|
+
// Query VIX close/open/high, VIX9D close/open, VIX3M close/open, plus Return_20D for Trend_Direction.
|
|
995
|
+
//
|
|
996
|
+
// The VIX-family OHLCV source is `effectiveDailyTarget` (the
|
|
997
|
+
// spotStore-seeded TEMP when io.spotStore is present) or
|
|
998
|
+
// `market.spot_daily` when io.spotStore is absent — `market.enriched` no
|
|
999
|
+
// longer carries OHLCV columns. The SPX JOIN keeps `dailyTarget` because
|
|
1000
|
+
// Return_20D is a Tier 1 enriched column written to the working table
|
|
1001
|
+
// earlier in the runEnrichment pipeline; spot/ never holds enriched fields.
|
|
1002
|
+
const vixOhlcvSource = spotStore ? effectiveDailyTarget : "market.spot_daily";
|
|
1003
|
+
const contextQuery = `
|
|
1007
1004
|
SELECT
|
|
1008
1005
|
vix.date,
|
|
1009
1006
|
vix.open AS VIX_Open,
|
|
@@ -1022,118 +1019,133 @@ async function runTier2(
|
|
|
1022
1019
|
ORDER BY vix.date ASC
|
|
1023
1020
|
`;
|
|
1024
1021
|
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1022
|
+
const rawResult = await conn.runAndReadAll(contextQuery, [DEFAULT_MARKET_TICKER]);
|
|
1023
|
+
const rawRows = rawResult.getRows();
|
|
1024
|
+
if (rawRows.length === 0) return { status: "complete", fieldsWritten: 0 };
|
|
1025
|
+
|
|
1026
|
+
// Query VIX RTH open from intraday bars.
|
|
1027
|
+
// When spotStore is provided, route through SpotStore.readBars('VIX', ...)
|
|
1028
|
+
// and filter to the 09:30–09:32 RTH window in TypeScript. Result is
|
|
1029
|
+
// bit-exact: same ticker filter, same time window, same "first seen per
|
|
1030
|
+
// date" selection (readBars sorts by (date, time)).
|
|
1031
|
+
const rthOpenByDate = new Map<string, number>();
|
|
1032
|
+
if (spotStore) {
|
|
1033
|
+
try {
|
|
1034
|
+
const vixBars = await spotStore.readBars(
|
|
1035
|
+
"VIX",
|
|
1036
|
+
rawRows[0][0] as string,
|
|
1037
|
+
rawRows[rawRows.length - 1][0] as string,
|
|
1038
|
+
);
|
|
1039
|
+
for (const bar of vixBars) {
|
|
1040
|
+
const timeStr = bar.time;
|
|
1041
|
+
if (timeStr == null || timeStr < "09:30" || timeStr > "09:32") continue;
|
|
1042
|
+
// Defense-in-depth: skip 09:30-09:32 bars with zero/null open. A
|
|
1043
|
+
// 09:30 provider gap would otherwise cache as the day's VIX_RTH_Open.
|
|
1044
|
+
// The first non-zero bar in the window wins.
|
|
1045
|
+
if (!Number.isFinite(bar.open) || bar.open <= 0) continue;
|
|
1046
|
+
const dateStr = bar.date;
|
|
1047
|
+
if (!rthOpenByDate.has(dateStr)) {
|
|
1048
|
+
const openVal = bar.open;
|
|
1049
|
+
if (openVal != null) rthOpenByDate.set(dateStr, openVal);
|
|
1050
|
+
}
|
|
1053
1051
|
}
|
|
1052
|
+
} catch {
|
|
1053
|
+
// No intraday VIX data — continue
|
|
1054
1054
|
}
|
|
1055
|
-
}
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
// bar in 09:30-09:32 wins.
|
|
1065
|
-
const rthReader = await conn.runAndReadAll(
|
|
1066
|
-
`SELECT date, open FROM market.spot
|
|
1055
|
+
} else {
|
|
1056
|
+
try {
|
|
1057
|
+
// Canonical minute-bar view is `market.spot` — same ticker/time/open
|
|
1058
|
+
// schema as the earlier intraday view it replaced.
|
|
1059
|
+
// Defense-in-depth: skip zero/null open bars so a 09:30 provider gap
|
|
1060
|
+
// doesn't get cached as the day's VIX_RTH_Open. The first non-zero
|
|
1061
|
+
// bar in 09:30-09:32 wins.
|
|
1062
|
+
const rthReader = await conn.runAndReadAll(
|
|
1063
|
+
`SELECT date, open FROM market.spot
|
|
1067
1064
|
WHERE ticker = 'VIX' AND time >= '09:30' AND time <= '09:32'
|
|
1068
1065
|
AND open IS NOT NULL AND open > 0
|
|
1069
|
-
ORDER BY date, time ASC
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1066
|
+
ORDER BY date, time ASC`,
|
|
1067
|
+
);
|
|
1068
|
+
for (const r of rthReader.getRows()) {
|
|
1069
|
+
const dateStr = r[0] as string;
|
|
1070
|
+
if (!rthOpenByDate.has(dateStr)) {
|
|
1071
|
+
const openVal = r[1] as number | null;
|
|
1072
|
+
if (openVal != null && openVal > 0) rthOpenByDate.set(dateStr, openVal);
|
|
1073
|
+
}
|
|
1076
1074
|
}
|
|
1075
|
+
} catch {
|
|
1076
|
+
// No intraday VIX data — continue
|
|
1077
1077
|
}
|
|
1078
|
-
} catch {
|
|
1079
|
-
// No intraday VIX data — continue
|
|
1080
1078
|
}
|
|
1081
|
-
}
|
|
1082
|
-
|
|
1083
|
-
const return20dByDate = new Map<string, number | null>();
|
|
1084
|
-
const contextRows: ContextRow[] = rawRows.map((r) => {
|
|
1085
|
-
const dateStr = r[0] as string;
|
|
1086
|
-
return20dByDate.set(dateStr, r[8] as number | null);
|
|
1087
|
-
return {
|
|
1088
|
-
date: dateStr,
|
|
1089
|
-
VIX_Open: r[1] as number | null,
|
|
1090
|
-
VIX_Close: r[2] as number | null,
|
|
1091
|
-
VIX_High: r[3] as number | null,
|
|
1092
|
-
VIX_RTH_Open: rthOpenByDate.get(dateStr) ?? null,
|
|
1093
|
-
VIX9D_Open: r[4] as number | null,
|
|
1094
|
-
VIX9D_Close: r[5] as number | null,
|
|
1095
|
-
VIX3M_Open: r[6] as number | null,
|
|
1096
|
-
VIX3M_Close: r[7] as number | null,
|
|
1097
|
-
};
|
|
1098
|
-
});
|
|
1099
1079
|
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
const vc = r.VIX_Close ?? null;
|
|
1116
|
-
const v9 = r.VIX9D_Close ?? null;
|
|
1117
|
-
const v3m = r.VIX3M_Close ?? null;
|
|
1118
|
-
return [
|
|
1119
|
-
r.date,
|
|
1120
|
-
vc !== null ? classifyVolRegime(vc) : null,
|
|
1121
|
-
v9 !== null && vc !== null && v3m !== null ? classifyTermStructure(v9, vc, v3m) : null,
|
|
1122
|
-
classifyTrendDirection(return20dByDate.get(r.date) ?? null),
|
|
1123
|
-
r.VIX_Spike_Pct ?? null,
|
|
1124
|
-
r.VIX_Gap_Pct ?? null,
|
|
1125
|
-
];
|
|
1080
|
+
const return20dByDate = new Map<string, number | null>();
|
|
1081
|
+
const contextRows: ContextRow[] = rawRows.map((r) => {
|
|
1082
|
+
const dateStr = r[0] as string;
|
|
1083
|
+
return20dByDate.set(dateStr, r[8] as number | null);
|
|
1084
|
+
return {
|
|
1085
|
+
date: dateStr,
|
|
1086
|
+
VIX_Open: r[1] as number | null,
|
|
1087
|
+
VIX_Close: r[2] as number | null,
|
|
1088
|
+
VIX_High: r[3] as number | null,
|
|
1089
|
+
VIX_RTH_Open: rthOpenByDate.get(dateStr) ?? null,
|
|
1090
|
+
VIX9D_Open: r[4] as number | null,
|
|
1091
|
+
VIX9D_Close: r[5] as number | null,
|
|
1092
|
+
VIX3M_Open: r[6] as number | null,
|
|
1093
|
+
VIX3M_Close: r[7] as number | null,
|
|
1094
|
+
};
|
|
1126
1095
|
});
|
|
1127
|
-
await conn.run(sql, params as (string | number | boolean | null | bigint)[]);
|
|
1128
|
-
}
|
|
1129
1096
|
|
|
1130
|
-
|
|
1097
|
+
// Step 4: Compute derived fields (reuse existing pure functions unchanged)
|
|
1098
|
+
const enrichedContext = computeVIXDerivedFields(contextRows);
|
|
1099
|
+
|
|
1100
|
+
// Step 5: Write derived fields to market.enriched_context (INSERT OR REPLACE)
|
|
1101
|
+
const derivedCols = [
|
|
1102
|
+
"date",
|
|
1103
|
+
"Vol_Regime",
|
|
1104
|
+
"Term_Structure_State",
|
|
1105
|
+
"Trend_Direction",
|
|
1106
|
+
"VIX_Spike_Pct",
|
|
1107
|
+
"VIX_Gap_Pct",
|
|
1108
|
+
];
|
|
1109
|
+
const BATCH_SIZE = 500;
|
|
1110
|
+
for (let start = 0; start < enrichedContext.length; start += BATCH_SIZE) {
|
|
1111
|
+
const batch = enrichedContext.slice(start, start + BATCH_SIZE);
|
|
1112
|
+
const placeholders = batch
|
|
1113
|
+
.map((_, rowIdx) => {
|
|
1114
|
+
const params = derivedCols.map(
|
|
1115
|
+
(__, colIdx) => `$${rowIdx * derivedCols.length + colIdx + 1}`,
|
|
1116
|
+
);
|
|
1117
|
+
return `(${params.join(", ")})`;
|
|
1118
|
+
})
|
|
1119
|
+
.join(", ");
|
|
1120
|
+
|
|
1121
|
+
const sql = `INSERT OR REPLACE INTO ${dateContextTarget} (${derivedCols.join(", ")}) VALUES ${placeholders}`;
|
|
1122
|
+
const params = batch.flatMap((r) => {
|
|
1123
|
+
const vc = r.VIX_Close ?? null;
|
|
1124
|
+
const v9 = r.VIX9D_Close ?? null;
|
|
1125
|
+
const v3m = r.VIX3M_Close ?? null;
|
|
1126
|
+
return [
|
|
1127
|
+
r.date,
|
|
1128
|
+
vc !== null ? classifyVolRegime(vc) : null,
|
|
1129
|
+
v9 !== null && vc !== null && v3m !== null ? classifyTermStructure(v9, vc, v3m) : null,
|
|
1130
|
+
classifyTrendDirection(return20dByDate.get(r.date) ?? null),
|
|
1131
|
+
r.VIX_Spike_Pct ?? null,
|
|
1132
|
+
r.VIX_Gap_Pct ?? null,
|
|
1133
|
+
];
|
|
1134
|
+
});
|
|
1135
|
+
await conn.run(sql, params as (string | number | boolean | null | bigint)[]);
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
return { status: "complete", fieldsWritten: derivedCols.length - 1 }; // -1 for date
|
|
1131
1139
|
} finally {
|
|
1132
1140
|
// Drop the spotStore-seeded TEMP unconditionally so it cannot leak across
|
|
1133
1141
|
// runEnrichment calls (each call gets a fresh ts-suffixed table name, but
|
|
1134
1142
|
// DROP-on-finally keeps DuckDB's TEMP catalog clean).
|
|
1135
1143
|
if (vixTempTable) {
|
|
1136
|
-
try {
|
|
1144
|
+
try {
|
|
1145
|
+
await conn.run(`DROP TABLE IF EXISTS "${vixTempTable}"`);
|
|
1146
|
+
} catch {
|
|
1147
|
+
/* */
|
|
1148
|
+
}
|
|
1137
1149
|
}
|
|
1138
1150
|
}
|
|
1139
1151
|
}
|
|
@@ -1155,10 +1167,9 @@ async function hasTier3Data(
|
|
|
1155
1167
|
}
|
|
1156
1168
|
// Canonical minute-bar view is `market.spot` — same ticker-filter schema
|
|
1157
1169
|
// as the earlier intraday view it replaced.
|
|
1158
|
-
const r = await conn.runAndReadAll(
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
);
|
|
1170
|
+
const r = await conn.runAndReadAll(`SELECT COUNT(*) FROM market.spot WHERE ticker = $1 LIMIT 1`, [
|
|
1171
|
+
ticker,
|
|
1172
|
+
]);
|
|
1162
1173
|
return Number(r.getRows()[0]?.[0] ?? 0) > 0;
|
|
1163
1174
|
}
|
|
1164
1175
|
|
|
@@ -1230,329 +1241,327 @@ export async function runEnrichment(
|
|
|
1230
1241
|
// ctxDerivedTarget and ctxTarget are passed via tier2Targets object to runTier2()
|
|
1231
1242
|
|
|
1232
1243
|
try {
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
+
// 1. Get the persisted enrichment watermark.
|
|
1245
|
+
// Every watermark read goes through the JSON adapter. The legacy SQL
|
|
1246
|
+
// SELECT against the metadata sync table has been removed — when callers
|
|
1247
|
+
// don't supply `io.watermarkStore` we fall back to the same JSON adapter
|
|
1248
|
+
// the store wrappers wire (`getEnrichedThrough(ticker, dataDir)`).
|
|
1249
|
+
let watermark: string | null = null;
|
|
1250
|
+
if (!forceFull) {
|
|
1251
|
+
if (io?.watermarkStore) {
|
|
1252
|
+
watermark = await io.watermarkStore.get(ticker);
|
|
1253
|
+
} else if (opts.dataDir) {
|
|
1254
|
+
watermark = await getEnrichedThrough(ticker, opts.dataDir);
|
|
1255
|
+
} else {
|
|
1256
|
+
// No JSON adapter path available without dataDir, and the SQL fallback
|
|
1257
|
+
// is gone. Treat as "no prior watermark" (fresh enrichment); callers that
|
|
1258
|
+
// need watermark continuity must supply io or dataDir.
|
|
1259
|
+
watermark = null;
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
// 2. Compute lookback start: watermark - 200 calendar days (as string comparison)
|
|
1264
|
+
const lookbackStart = watermark ? subtractDays(watermark, 200) : null;
|
|
1265
|
+
|
|
1266
|
+
// 3. Fetch OHLCV rows.
|
|
1267
|
+
//
|
|
1268
|
+
// When `io.spotStore` is provided, read daily OHLCV via
|
|
1269
|
+
// `SpotStore.readDailyBars` (aggregated from spot/ minute bars). This path
|
|
1270
|
+
// remains functional after the legacy `daily.parquet` retirement because
|
|
1271
|
+
// readDailyBars aggregates from spot/ticker=X/date=Y/data.parquet.
|
|
1272
|
+
//
|
|
1273
|
+
// Fallback: when `io.spotStore` is absent (legacy callers), retain a SQL
|
|
1274
|
+
// path against `market.spot_daily`. The fallback may be removed once all
|
|
1275
|
+
// callers pass io.
|
|
1276
|
+
let rawRows: Array<Array<unknown>>;
|
|
1277
|
+
if (io?.spotStore) {
|
|
1278
|
+
const startDate = lookbackStart ?? "1970-01-01";
|
|
1279
|
+
const endDate = "9999-12-31"; // readDailyBars caps internally via partition discovery
|
|
1280
|
+
const dailyBars = await io.spotStore.readDailyBars(ticker, startDate, endDate);
|
|
1281
|
+
rawRows = dailyBars.map((b) => [b.ticker, b.date, b.open, b.high, b.low, b.close]);
|
|
1244
1282
|
} else {
|
|
1245
|
-
//
|
|
1246
|
-
//
|
|
1247
|
-
//
|
|
1248
|
-
|
|
1283
|
+
// The legacy daily-view SQL fallback path is gone — the view no longer
|
|
1284
|
+
// exists in the catalog. Route OHLCV reads through the canonical
|
|
1285
|
+
// `market.spot_daily` view (RTH-aggregated from `market.spot`). This
|
|
1286
|
+
// bridges callers that have not yet migrated to io.spotStore; new
|
|
1287
|
+
// callers SHOULD pass io.spotStore for parity with the Parquet-direct path.
|
|
1288
|
+
let fetchSql = `SELECT ticker, date, open, high, low, close FROM market.spot_daily WHERE ticker = $1`;
|
|
1289
|
+
const fetchParams: unknown[] = [ticker];
|
|
1290
|
+
if (lookbackStart) {
|
|
1291
|
+
fetchSql += ` AND date >= $2`;
|
|
1292
|
+
fetchParams.push(lookbackStart);
|
|
1293
|
+
}
|
|
1294
|
+
fetchSql += ` ORDER BY date ASC`;
|
|
1295
|
+
const rawReader = await conn.runAndReadAll(
|
|
1296
|
+
fetchSql,
|
|
1297
|
+
fetchParams as (string | number | boolean | null | bigint)[],
|
|
1298
|
+
);
|
|
1299
|
+
rawRows = rawReader.getRows();
|
|
1249
1300
|
}
|
|
1250
|
-
}
|
|
1251
1301
|
|
|
1252
|
-
|
|
1253
|
-
|
|
1302
|
+
if (rawRows.length === 0) {
|
|
1303
|
+
return {
|
|
1304
|
+
ticker,
|
|
1305
|
+
tier1: {
|
|
1306
|
+
status: "skipped",
|
|
1307
|
+
reason: io?.spotStore ? "no data from spotStore" : "no data in market.spot_daily",
|
|
1308
|
+
},
|
|
1309
|
+
tier2: { status: "skipped", reason: "no daily data" },
|
|
1310
|
+
tier3: { status: "skipped", reason: "no daily data" },
|
|
1311
|
+
rowsEnriched: 0,
|
|
1312
|
+
enrichedThrough: null,
|
|
1313
|
+
};
|
|
1314
|
+
}
|
|
1254
1315
|
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
const
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
// exists in the catalog. Route OHLCV reads through the canonical
|
|
1274
|
-
// `market.spot_daily` view (RTH-aggregated from `market.spot`). This
|
|
1275
|
-
// bridges callers that have not yet migrated to io.spotStore; new
|
|
1276
|
-
// callers SHOULD pass io.spotStore for parity with the Parquet-direct path.
|
|
1277
|
-
let fetchSql = `SELECT ticker, date, open, high, low, close FROM market.spot_daily WHERE ticker = $1`;
|
|
1278
|
-
const fetchParams: unknown[] = [ticker];
|
|
1279
|
-
if (lookbackStart) {
|
|
1280
|
-
fetchSql += ` AND date >= $2`;
|
|
1281
|
-
fetchParams.push(lookbackStart);
|
|
1316
|
+
// 3b. Defensive zero-OHLC filter. Partitions should already be clean after
|
|
1317
|
+
// the ParquetSpotStore.writeBars guard, but this second line of defense
|
|
1318
|
+
// catches any future provider-outage bleed and prevents
|
|
1319
|
+
// RSI/ATR/EMA/SMA/RealizedVol from being poisoned by zero closes. Filter
|
|
1320
|
+
// at the rawRows level so date/OHLC alignment is preserved across all
|
|
1321
|
+
// five arrays (dates/opens/highs/lows/closes) constructed below.
|
|
1322
|
+
const filteredRawRows = rawRows.filter((r) => {
|
|
1323
|
+
const o = Number(r[2]);
|
|
1324
|
+
const h = Number(r[3]);
|
|
1325
|
+
const l = Number(r[4]);
|
|
1326
|
+
const c = Number(r[5]);
|
|
1327
|
+
return !(o === 0 && h === 0 && l === 0 && c === 0);
|
|
1328
|
+
});
|
|
1329
|
+
const zeroRowsDropped = rawRows.length - filteredRawRows.length;
|
|
1330
|
+
if (zeroRowsDropped > 0) {
|
|
1331
|
+
console.warn(
|
|
1332
|
+
`[market-enricher] ticker=${ticker} dropped ${zeroRowsDropped} all-zero-OHLC rows before indicator math`,
|
|
1333
|
+
);
|
|
1282
1334
|
}
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1335
|
+
rawRows = filteredRawRows;
|
|
1336
|
+
|
|
1337
|
+
// 4. Extract typed arrays from raw rows
|
|
1338
|
+
// Columns: ticker(0), date(1), open(2), high(3), low(4), close(5)
|
|
1339
|
+
const dates = rawRows.map((r) => r[1] as string);
|
|
1340
|
+
const opens = rawRows.map((r) => Number(r[2]));
|
|
1341
|
+
const highs = rawRows.map((r) => Number(r[3]));
|
|
1342
|
+
const lows = rawRows.map((r) => Number(r[4]));
|
|
1343
|
+
const closes = rawRows.map((r) => Number(r[5]));
|
|
1344
|
+
|
|
1345
|
+
// 5. Compute Tier 1 indicators
|
|
1346
|
+
const rsi14 = computeRSI(closes, 14);
|
|
1347
|
+
const atrArr = computeATR(highs, lows, closes, 14);
|
|
1348
|
+
const ema21 = computeEMA(closes, 21);
|
|
1349
|
+
const sma50 = computeSMA(closes, 50);
|
|
1350
|
+
const rvol5 = computeRealizedVol(closes, 5);
|
|
1351
|
+
const rvol20 = computeRealizedVol(closes, 20);
|
|
1352
|
+
const consecutiveDays = computeConsecutiveDays(closes);
|
|
1353
|
+
|
|
1354
|
+
// 6. Determine which rows to write back (only rows after watermark)
|
|
1355
|
+
const writeRows =
|
|
1356
|
+
watermark && !forceFull
|
|
1357
|
+
? rawRows.map((_, i) => i).filter((i) => dates[i] > watermark)
|
|
1358
|
+
: rawRows.map((_, i) => i);
|
|
1359
|
+
|
|
1360
|
+
if (writeRows.length === 0) {
|
|
1361
|
+
const tier2Targets = workingTables
|
|
1362
|
+
? {
|
|
1363
|
+
daily: workingTables.dailyTable,
|
|
1364
|
+
dateContext: workingTables.dateContextTable,
|
|
1365
|
+
}
|
|
1366
|
+
: undefined;
|
|
1367
|
+
const tier2Result = await runTier2(conn, tier2Targets, io?.spotStore);
|
|
1368
|
+
|
|
1369
|
+
// Flush even if no Tier 1 rows — Tier 2 may have written to working tables
|
|
1370
|
+
if (parquetMode && workingTables && opts.dataDir) {
|
|
1371
|
+
await flushEnrichedToParquet(conn, opts.dataDir, ticker, workingTables);
|
|
1372
|
+
}
|
|
1287
1373
|
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
}
|
|
1300
|
-
}
|
|
1374
|
+
return {
|
|
1375
|
+
ticker,
|
|
1376
|
+
tier1: { status: "complete", fieldsWritten: 0, reason: "already up to date" },
|
|
1377
|
+
tier2: tier2Result,
|
|
1378
|
+
tier3: {
|
|
1379
|
+
status: "skipped",
|
|
1380
|
+
reason: "no intraday data in market.spot",
|
|
1381
|
+
},
|
|
1382
|
+
rowsEnriched: 0,
|
|
1383
|
+
enrichedThrough: watermark,
|
|
1384
|
+
};
|
|
1385
|
+
}
|
|
1301
1386
|
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1387
|
+
// 7. Build enriched rows for batch UPDATE
|
|
1388
|
+
const enrichedRows = writeRows.map((i) => {
|
|
1389
|
+
const atrVal = atrArr[i];
|
|
1390
|
+
const atrPct = !isNaN(atrVal) && closes[i] > 0 ? (atrVal / closes[i]) * 100 : null;
|
|
1391
|
+
const priorClose = i > 0 ? closes[i - 1] : null;
|
|
1392
|
+
const priorReturn = i > 1 ? ((closes[i - 1] - closes[i - 2]) / closes[i - 2]) * 100 : null;
|
|
1393
|
+
const gapPct =
|
|
1394
|
+
priorClose !== null && priorClose > 0 ? ((opens[i] - priorClose) / priorClose) * 100 : null;
|
|
1395
|
+
// Intraday_Range_Pct: high-low range as % of close.
|
|
1396
|
+
// Use close (not open) for consistency with every other "_Pct" column in
|
|
1397
|
+
// this file (ATR_Pct, Price_vs_EMA21_Pct, Return_5D, etc. all divide by
|
|
1398
|
+
// close). Also guards against zero-low contamination: if the day's low
|
|
1399
|
+
// came in as 0 from a bad minute bar, (high - 0) inflates to ~100% of
|
|
1400
|
+
// close — meaningless. Requiring lows[i] > 0 forces such rows to null.
|
|
1401
|
+
const intradayRangePct =
|
|
1402
|
+
closes[i] > 0 && highs[i] > 0 && lows[i] > 0
|
|
1403
|
+
? ((highs[i] - lows[i]) / closes[i]) * 100
|
|
1404
|
+
: null;
|
|
1405
|
+
const intradayReturnPct = opens[i] > 0 ? ((closes[i] - opens[i]) / opens[i]) * 100 : null;
|
|
1406
|
+
const hiLoRange = highs[i] - lows[i];
|
|
1407
|
+
const closePosInRange = hiLoRange > 0 ? (closes[i] - lows[i]) / hiLoRange : null;
|
|
1408
|
+
const ret5d =
|
|
1409
|
+
i >= 5 && closes[i - 5] > 0 ? ((closes[i] - closes[i - 5]) / closes[i - 5]) * 100 : null;
|
|
1410
|
+
const ret20d =
|
|
1411
|
+
i >= 20 && closes[i - 20] > 0
|
|
1412
|
+
? ((closes[i] - closes[i - 20]) / closes[i - 20]) * 100
|
|
1413
|
+
: null;
|
|
1414
|
+
const gapFilled =
|
|
1415
|
+
priorClose !== null ? isGapFilled(opens[i], highs[i], lows[i], priorClose) : null;
|
|
1416
|
+
const dateObj = parseDateStr(dates[i]);
|
|
1417
|
+
const dayOfWeek = dateObj ? dateObj.getDay() : null; // 0=Sun..6=Sat
|
|
1418
|
+
const monthVal = dateObj ? dateObj.getMonth() + 1 : null;
|
|
1419
|
+
const opex = isOpex(dates[i]);
|
|
1420
|
+
const ema21val = ema21[i];
|
|
1421
|
+
const sma50val = sma50[i];
|
|
1422
|
+
const priceVsEma21 =
|
|
1423
|
+
!isNaN(ema21val) && ema21val > 0 ? ((closes[i] - ema21val) / ema21val) * 100 : null;
|
|
1424
|
+
const priceVsSma50 =
|
|
1425
|
+
!isNaN(sma50val) && sma50val > 0 ? ((closes[i] - sma50val) / sma50val) * 100 : null;
|
|
1426
|
+
const rsi14val = rsi14[i];
|
|
1427
|
+
|
|
1428
|
+
// Prior_Range_vs_ATR: ratio of prior day's intraday range (% of close) to
|
|
1429
|
+
// prior day's ATR (% of close). Known at market open — prior day range
|
|
1430
|
+
// and ATR are both available before today's trading begins.
|
|
1431
|
+
//
|
|
1432
|
+
// Algebraically (range_pct / atr_pct) = (range / atr) since the close
|
|
1433
|
+
// cancels, but writing it as a ratio of percents makes the intent
|
|
1434
|
+
// explicit and matches how downstream analysis reads the column.
|
|
1435
|
+
//
|
|
1436
|
+
// Sanity guards: prior close > 0 (otherwise the percent denominators
|
|
1437
|
+
// explode), prior high/low > 0 (catches zero-bar contamination from the
|
|
1438
|
+
// upstream spot ingester), and priorATR > 0 (avoid div-by-zero).
|
|
1439
|
+
// First bar (i=0) has no prior day → null.
|
|
1440
|
+
let priorRangeVsATR: number | null = null;
|
|
1441
|
+
if (i > 0) {
|
|
1442
|
+
const priorClose = closes[i - 1];
|
|
1443
|
+
const priorHigh = highs[i - 1];
|
|
1444
|
+
const priorLow = lows[i - 1];
|
|
1445
|
+
const priorATR = atrArr[i - 1];
|
|
1446
|
+
if (priorClose > 0 && priorHigh > 0 && priorLow > 0 && !isNaN(priorATR) && priorATR > 0) {
|
|
1447
|
+
const priorRangePct = ((priorHigh - priorLow) / priorClose) * 100;
|
|
1448
|
+
const priorAtrPct = (priorATR / priorClose) * 100;
|
|
1449
|
+
priorRangeVsATR = priorRangePct / priorAtrPct;
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1352
1452
|
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1453
|
+
return {
|
|
1454
|
+
ticker,
|
|
1455
|
+
date: dates[i],
|
|
1456
|
+
Prior_Close: priorClose,
|
|
1457
|
+
Gap_Pct: gapPct,
|
|
1458
|
+
RSI_14: isNaN(rsi14val) ? null : rsi14val,
|
|
1459
|
+
ATR_Pct: atrPct,
|
|
1460
|
+
Price_vs_EMA21_Pct: priceVsEma21,
|
|
1461
|
+
Price_vs_SMA50_Pct: priceVsSma50,
|
|
1462
|
+
Realized_Vol_5D: isNaN(rvol5[i]) ? null : rvol5[i],
|
|
1463
|
+
Realized_Vol_20D: isNaN(rvol20[i]) ? null : rvol20[i],
|
|
1464
|
+
Return_5D: ret5d,
|
|
1465
|
+
Return_20D: ret20d,
|
|
1466
|
+
Intraday_Range_Pct: intradayRangePct,
|
|
1467
|
+
Intraday_Return_Pct: intradayReturnPct,
|
|
1468
|
+
Close_Position_In_Range: closePosInRange,
|
|
1469
|
+
Gap_Filled: gapFilled,
|
|
1470
|
+
Consecutive_Days: consecutiveDays[i],
|
|
1471
|
+
Prev_Return_Pct: priorReturn,
|
|
1472
|
+
Day_of_Week: dayOfWeek,
|
|
1473
|
+
Month: monthVal,
|
|
1474
|
+
Is_Opex: opex,
|
|
1475
|
+
Prior_Range_vs_ATR: priorRangeVsATR,
|
|
1476
|
+
};
|
|
1477
|
+
});
|
|
1478
|
+
|
|
1479
|
+
// 8. Batch UPDATE via DuckDB VALUES CTE, batches of 500
|
|
1480
|
+
const BATCH_SIZE = 500;
|
|
1481
|
+
const columns = [
|
|
1482
|
+
"Prior_Close",
|
|
1483
|
+
"Gap_Pct",
|
|
1484
|
+
"RSI_14",
|
|
1485
|
+
"ATR_Pct",
|
|
1486
|
+
"Price_vs_EMA21_Pct",
|
|
1487
|
+
"Price_vs_SMA50_Pct",
|
|
1488
|
+
"Realized_Vol_5D",
|
|
1489
|
+
"Realized_Vol_20D",
|
|
1490
|
+
"Return_5D",
|
|
1491
|
+
"Return_20D",
|
|
1492
|
+
"Intraday_Range_Pct",
|
|
1493
|
+
"Intraday_Return_Pct",
|
|
1494
|
+
"Close_Position_In_Range",
|
|
1495
|
+
"Gap_Filled",
|
|
1496
|
+
"Consecutive_Days",
|
|
1497
|
+
"Prev_Return_Pct",
|
|
1498
|
+
"Day_of_Week",
|
|
1499
|
+
"Month",
|
|
1500
|
+
"Is_Opex",
|
|
1501
|
+
"Prior_Range_vs_ATR",
|
|
1502
|
+
];
|
|
1503
|
+
for (let start = 0; start < enrichedRows.length; start += BATCH_SIZE) {
|
|
1504
|
+
const batch = enrichedRows.slice(start, start + BATCH_SIZE);
|
|
1505
|
+
await batchUpdateDaily(conn, batch, columns, dailyTarget);
|
|
1356
1506
|
}
|
|
1357
1507
|
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
rowsEnriched: 0,
|
|
1367
|
-
enrichedThrough: watermark,
|
|
1368
|
-
};
|
|
1369
|
-
}
|
|
1508
|
+
// 9. Run Tier 2 (VIX context enrichment) with parameterized targets
|
|
1509
|
+
const tier2Targets = workingTables
|
|
1510
|
+
? {
|
|
1511
|
+
daily: workingTables.dailyTable,
|
|
1512
|
+
dateContext: workingTables.dateContextTable,
|
|
1513
|
+
}
|
|
1514
|
+
: undefined;
|
|
1515
|
+
const tier2Result = await runTier2(conn, tier2Targets, io?.spotStore);
|
|
1370
1516
|
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
const intradayRangePct =
|
|
1389
|
-
closes[i] > 0 && highs[i] > 0 && lows[i] > 0
|
|
1390
|
-
? ((highs[i] - lows[i]) / closes[i]) * 100
|
|
1391
|
-
: null;
|
|
1392
|
-
const intradayReturnPct =
|
|
1393
|
-
opens[i] > 0 ? ((closes[i] - opens[i]) / opens[i]) * 100 : null;
|
|
1394
|
-
const hiLoRange = highs[i] - lows[i];
|
|
1395
|
-
const closePosInRange =
|
|
1396
|
-
hiLoRange > 0 ? (closes[i] - lows[i]) / hiLoRange : null;
|
|
1397
|
-
const ret5d =
|
|
1398
|
-
i >= 5 && closes[i - 5] > 0
|
|
1399
|
-
? ((closes[i] - closes[i - 5]) / closes[i - 5]) * 100
|
|
1400
|
-
: null;
|
|
1401
|
-
const ret20d =
|
|
1402
|
-
i >= 20 && closes[i - 20] > 0
|
|
1403
|
-
? ((closes[i] - closes[i - 20]) / closes[i - 20]) * 100
|
|
1404
|
-
: null;
|
|
1405
|
-
const gapFilled =
|
|
1406
|
-
priorClose !== null ? isGapFilled(opens[i], highs[i], lows[i], priorClose) : null;
|
|
1407
|
-
const dateObj = parseDateStr(dates[i]);
|
|
1408
|
-
const dayOfWeek = dateObj ? dateObj.getDay() : null; // 0=Sun..6=Sat
|
|
1409
|
-
const monthVal = dateObj ? dateObj.getMonth() + 1 : null;
|
|
1410
|
-
const opex = isOpex(dates[i]);
|
|
1411
|
-
const ema21val = ema21[i];
|
|
1412
|
-
const sma50val = sma50[i];
|
|
1413
|
-
const priceVsEma21 =
|
|
1414
|
-
!isNaN(ema21val) && ema21val > 0
|
|
1415
|
-
? ((closes[i] - ema21val) / ema21val) * 100
|
|
1416
|
-
: null;
|
|
1417
|
-
const priceVsSma50 =
|
|
1418
|
-
!isNaN(sma50val) && sma50val > 0
|
|
1419
|
-
? ((closes[i] - sma50val) / sma50val) * 100
|
|
1420
|
-
: null;
|
|
1421
|
-
const rsi14val = rsi14[i];
|
|
1517
|
+
// 10. Tier 3 — intraday timing fields (routes through io.spotStore when provided)
|
|
1518
|
+
const tier3Result = await runTier3(conn, ticker, dates, dailyTarget, io?.spotStore);
|
|
1519
|
+
|
|
1520
|
+
// 11. Persist the new watermark.
|
|
1521
|
+
// Every watermark write goes through the JSON adapter. The legacy SQL
|
|
1522
|
+
// UPSERT against the metadata sync table has been removed — when callers
|
|
1523
|
+
// don't supply `io.watermarkStore` we fall back to
|
|
1524
|
+
// `upsertEnrichedThrough(ticker, val, dataDir)` directly. If neither io
|
|
1525
|
+
// nor dataDir is supplied the watermark simply isn't persisted (math
|
|
1526
|
+
// still runs); callers that need watermark continuity must supply one of
|
|
1527
|
+
// the two.
|
|
1528
|
+
const newWatermark = dates[dates.length - 1];
|
|
1529
|
+
if (io?.watermarkStore) {
|
|
1530
|
+
await io.watermarkStore.upsert(ticker, newWatermark);
|
|
1531
|
+
} else if (opts.dataDir) {
|
|
1532
|
+
await upsertEnrichedThrough(ticker, newWatermark, opts.dataDir);
|
|
1533
|
+
}
|
|
1422
1534
|
|
|
1423
|
-
//
|
|
1424
|
-
//
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
// Algebraically (range_pct / atr_pct) = (range / atr) since the close
|
|
1428
|
-
// cancels, but writing it as a ratio of percents makes the intent
|
|
1429
|
-
// explicit and matches how downstream analysis reads the column.
|
|
1430
|
-
//
|
|
1431
|
-
// Sanity guards: prior close > 0 (otherwise the percent denominators
|
|
1432
|
-
// explode), prior high/low > 0 (catches zero-bar contamination from the
|
|
1433
|
-
// upstream spot ingester), and priorATR > 0 (avoid div-by-zero).
|
|
1434
|
-
// First bar (i=0) has no prior day → null.
|
|
1435
|
-
let priorRangeVsATR: number | null = null;
|
|
1436
|
-
if (i > 0) {
|
|
1437
|
-
const priorClose = closes[i - 1];
|
|
1438
|
-
const priorHigh = highs[i - 1];
|
|
1439
|
-
const priorLow = lows[i - 1];
|
|
1440
|
-
const priorATR = atrArr[i - 1];
|
|
1441
|
-
if (
|
|
1442
|
-
priorClose > 0 &&
|
|
1443
|
-
priorHigh > 0 &&
|
|
1444
|
-
priorLow > 0 &&
|
|
1445
|
-
!isNaN(priorATR) && priorATR > 0
|
|
1446
|
-
) {
|
|
1447
|
-
const priorRangePct = ((priorHigh - priorLow) / priorClose) * 100;
|
|
1448
|
-
const priorAtrPct = (priorATR / priorClose) * 100;
|
|
1449
|
-
priorRangeVsATR = priorRangePct / priorAtrPct;
|
|
1450
|
-
}
|
|
1535
|
+
// 12. Parquet mode: write enrichment to the enriched/ partition layout
|
|
1536
|
+
// (legacy daily.parquet output retired)
|
|
1537
|
+
if (parquetMode && workingTables && opts.dataDir) {
|
|
1538
|
+
await flushEnrichedToParquet(conn, opts.dataDir, ticker, workingTables);
|
|
1451
1539
|
}
|
|
1452
1540
|
|
|
1453
1541
|
return {
|
|
1454
1542
|
ticker,
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
Price_vs_EMA21_Pct: priceVsEma21,
|
|
1461
|
-
Price_vs_SMA50_Pct: priceVsSma50,
|
|
1462
|
-
Realized_Vol_5D: isNaN(rvol5[i]) ? null : rvol5[i],
|
|
1463
|
-
Realized_Vol_20D: isNaN(rvol20[i]) ? null : rvol20[i],
|
|
1464
|
-
Return_5D: ret5d,
|
|
1465
|
-
Return_20D: ret20d,
|
|
1466
|
-
Intraday_Range_Pct: intradayRangePct,
|
|
1467
|
-
Intraday_Return_Pct: intradayReturnPct,
|
|
1468
|
-
Close_Position_In_Range: closePosInRange,
|
|
1469
|
-
Gap_Filled: gapFilled,
|
|
1470
|
-
Consecutive_Days: consecutiveDays[i],
|
|
1471
|
-
Prev_Return_Pct: priorReturn,
|
|
1472
|
-
Day_of_Week: dayOfWeek,
|
|
1473
|
-
Month: monthVal,
|
|
1474
|
-
Is_Opex: opex,
|
|
1475
|
-
Prior_Range_vs_ATR: priorRangeVsATR,
|
|
1543
|
+
tier1: { status: "complete", fieldsWritten: columns.length },
|
|
1544
|
+
tier2: tier2Result,
|
|
1545
|
+
tier3: tier3Result,
|
|
1546
|
+
rowsEnriched: enrichedRows.length,
|
|
1547
|
+
enrichedThrough: newWatermark,
|
|
1476
1548
|
};
|
|
1477
|
-
});
|
|
1478
|
-
|
|
1479
|
-
// 8. Batch UPDATE via DuckDB VALUES CTE, batches of 500
|
|
1480
|
-
const BATCH_SIZE = 500;
|
|
1481
|
-
const columns = [
|
|
1482
|
-
"Prior_Close",
|
|
1483
|
-
"Gap_Pct",
|
|
1484
|
-
"RSI_14",
|
|
1485
|
-
"ATR_Pct",
|
|
1486
|
-
"Price_vs_EMA21_Pct",
|
|
1487
|
-
"Price_vs_SMA50_Pct",
|
|
1488
|
-
"Realized_Vol_5D",
|
|
1489
|
-
"Realized_Vol_20D",
|
|
1490
|
-
"Return_5D",
|
|
1491
|
-
"Return_20D",
|
|
1492
|
-
"Intraday_Range_Pct",
|
|
1493
|
-
"Intraday_Return_Pct",
|
|
1494
|
-
"Close_Position_In_Range",
|
|
1495
|
-
"Gap_Filled",
|
|
1496
|
-
"Consecutive_Days",
|
|
1497
|
-
"Prev_Return_Pct",
|
|
1498
|
-
"Day_of_Week",
|
|
1499
|
-
"Month",
|
|
1500
|
-
"Is_Opex",
|
|
1501
|
-
"Prior_Range_vs_ATR",
|
|
1502
|
-
];
|
|
1503
|
-
for (let start = 0; start < enrichedRows.length; start += BATCH_SIZE) {
|
|
1504
|
-
const batch = enrichedRows.slice(start, start + BATCH_SIZE);
|
|
1505
|
-
await batchUpdateDaily(conn, batch, columns, dailyTarget);
|
|
1506
|
-
}
|
|
1507
|
-
|
|
1508
|
-
// 9. Run Tier 2 (VIX context enrichment) with parameterized targets
|
|
1509
|
-
const tier2Targets = workingTables ? {
|
|
1510
|
-
daily: workingTables.dailyTable,
|
|
1511
|
-
dateContext: workingTables.dateContextTable,
|
|
1512
|
-
} : undefined;
|
|
1513
|
-
const tier2Result = await runTier2(conn, tier2Targets, io?.spotStore);
|
|
1514
|
-
|
|
1515
|
-
// 10. Tier 3 — intraday timing fields (routes through io.spotStore when provided)
|
|
1516
|
-
const tier3Result = await runTier3(conn, ticker, dates, dailyTarget, io?.spotStore);
|
|
1517
|
-
|
|
1518
|
-
// 11. Persist the new watermark.
|
|
1519
|
-
// Every watermark write goes through the JSON adapter. The legacy SQL
|
|
1520
|
-
// UPSERT against the metadata sync table has been removed — when callers
|
|
1521
|
-
// don't supply `io.watermarkStore` we fall back to
|
|
1522
|
-
// `upsertEnrichedThrough(ticker, val, dataDir)` directly. If neither io
|
|
1523
|
-
// nor dataDir is supplied the watermark simply isn't persisted (math
|
|
1524
|
-
// still runs); callers that need watermark continuity must supply one of
|
|
1525
|
-
// the two.
|
|
1526
|
-
const newWatermark = dates[dates.length - 1];
|
|
1527
|
-
if (io?.watermarkStore) {
|
|
1528
|
-
await io.watermarkStore.upsert(ticker, newWatermark);
|
|
1529
|
-
} else if (opts.dataDir) {
|
|
1530
|
-
await upsertEnrichedThrough(ticker, newWatermark, opts.dataDir);
|
|
1531
|
-
}
|
|
1532
|
-
|
|
1533
|
-
// 12. Parquet mode: write enrichment to the enriched/ partition layout
|
|
1534
|
-
// (legacy daily.parquet output retired)
|
|
1535
|
-
if (parquetMode && workingTables && opts.dataDir) {
|
|
1536
|
-
await flushEnrichedToParquet(conn, opts.dataDir, ticker, workingTables);
|
|
1537
|
-
}
|
|
1538
|
-
|
|
1539
|
-
return {
|
|
1540
|
-
ticker,
|
|
1541
|
-
tier1: { status: "complete", fieldsWritten: columns.length },
|
|
1542
|
-
tier2: tier2Result,
|
|
1543
|
-
tier3: tier3Result,
|
|
1544
|
-
rowsEnriched: enrichedRows.length,
|
|
1545
|
-
enrichedThrough: newWatermark,
|
|
1546
|
-
};
|
|
1547
|
-
|
|
1548
1549
|
} finally {
|
|
1549
1550
|
// Sole owner of working table cleanup — always runs on success or error.
|
|
1550
1551
|
// On success: tables still exist (flushParquetWorkingTables does not drop them).
|
|
1551
1552
|
// On error: tables may contain partial results useful for debugging, but we
|
|
1552
1553
|
// clean up to avoid leaking temp tables across calls.
|
|
1553
1554
|
if (workingTables) {
|
|
1554
|
-
try {
|
|
1555
|
-
|
|
1555
|
+
try {
|
|
1556
|
+
await conn.run(`DROP TABLE IF EXISTS "${workingTables.dailyTable}"`);
|
|
1557
|
+
} catch {
|
|
1558
|
+
/* */
|
|
1559
|
+
}
|
|
1560
|
+
try {
|
|
1561
|
+
await conn.run(`DROP TABLE IF EXISTS "${workingTables.dateContextTable}"`);
|
|
1562
|
+
} catch {
|
|
1563
|
+
/* */
|
|
1564
|
+
}
|
|
1556
1565
|
}
|
|
1557
1566
|
}
|
|
1558
1567
|
}
|
|
@@ -1586,7 +1595,7 @@ function hhmmToDecimalHours(time: string): number {
|
|
|
1586
1595
|
* @returns Computed fields or null if bars is empty
|
|
1587
1596
|
*/
|
|
1588
1597
|
export function computeIntradayTimingFields(
|
|
1589
|
-
bars: Array<{ time: string; open: number; high: number; low: number; close: number }
|
|
1598
|
+
bars: Array<{ time: string; open: number; high: number; low: number; close: number }>,
|
|
1590
1599
|
): {
|
|
1591
1600
|
highTime: number;
|
|
1592
1601
|
lowTime: number;
|
|
@@ -1603,10 +1612,14 @@ export function computeIntradayTimingFields(
|
|
|
1603
1612
|
// makes the pure function safe regardless of caller.
|
|
1604
1613
|
bars = bars.filter(
|
|
1605
1614
|
(b) =>
|
|
1606
|
-
Number.isFinite(b.open) &&
|
|
1607
|
-
|
|
1608
|
-
Number.isFinite(b.
|
|
1609
|
-
|
|
1615
|
+
Number.isFinite(b.open) &&
|
|
1616
|
+
b.open > 0 &&
|
|
1617
|
+
Number.isFinite(b.high) &&
|
|
1618
|
+
b.high > 0 &&
|
|
1619
|
+
Number.isFinite(b.low) &&
|
|
1620
|
+
b.low > 0 &&
|
|
1621
|
+
Number.isFinite(b.close) &&
|
|
1622
|
+
b.close > 0,
|
|
1610
1623
|
);
|
|
1611
1624
|
if (bars.length === 0) return null;
|
|
1612
1625
|
|
|
@@ -1637,17 +1650,18 @@ export function computeIntradayTimingFields(
|
|
|
1637
1650
|
const lowInAfternoon = lowTime >= 12;
|
|
1638
1651
|
|
|
1639
1652
|
let reversalType = 0;
|
|
1640
|
-
if (highInMorning && lowInAfternoon)
|
|
1653
|
+
if (highInMorning && lowInAfternoon)
|
|
1654
|
+
reversalType = 1; // High morning, low afternoon
|
|
1641
1655
|
else if (lowInMorning && highInAfternoon) reversalType = -1; // Low morning, high afternoon
|
|
1642
1656
|
|
|
1643
1657
|
// Opening Drive Strength: ratio of first-30-min range to full-day range
|
|
1644
1658
|
// First 30 min = bars with time < 10:00 (market opens 09:30)
|
|
1645
|
-
const openingBars = bars.filter(b => hhmmToDecimalHours(b.time) < 10);
|
|
1659
|
+
const openingBars = bars.filter((b) => hhmmToDecimalHours(b.time) < 10);
|
|
1646
1660
|
let openingDriveStrength = 0;
|
|
1647
1661
|
const fullDayRange = maxHigh - minLow;
|
|
1648
1662
|
if (openingBars.length > 0 && fullDayRange > 0) {
|
|
1649
|
-
const openHigh = Math.max(...openingBars.map(b => b.high));
|
|
1650
|
-
const openLow = Math.min(...openingBars.map(b => b.low));
|
|
1663
|
+
const openHigh = Math.max(...openingBars.map((b) => b.high));
|
|
1664
|
+
const openLow = Math.min(...openingBars.map((b) => b.low));
|
|
1651
1665
|
openingDriveStrength = (openHigh - openLow) / fullDayRange;
|
|
1652
1666
|
}
|
|
1653
1667
|
|
|
@@ -1671,7 +1685,14 @@ export function computeIntradayTimingFields(
|
|
|
1671
1685
|
}
|
|
1672
1686
|
}
|
|
1673
1687
|
|
|
1674
|
-
return {
|
|
1688
|
+
return {
|
|
1689
|
+
highTime,
|
|
1690
|
+
lowTime,
|
|
1691
|
+
highBeforeLow,
|
|
1692
|
+
reversalType,
|
|
1693
|
+
openingDriveStrength,
|
|
1694
|
+
intradayRealizedVol,
|
|
1695
|
+
};
|
|
1675
1696
|
}
|
|
1676
1697
|
|
|
1677
1698
|
/** Run Tier 3: compute intraday timing fields from market.spot and write to the daily write-target table */
|
|
@@ -1730,7 +1751,7 @@ async function runTier3(
|
|
|
1730
1751
|
AND low IS NOT NULL AND low > 0
|
|
1731
1752
|
AND close IS NOT NULL AND close > 0
|
|
1732
1753
|
ORDER BY date, time`,
|
|
1733
|
-
[ticker, dates[0], dates[dates.length - 1]]
|
|
1754
|
+
[ticker, dates[0], dates[dates.length - 1]],
|
|
1734
1755
|
);
|
|
1735
1756
|
|
|
1736
1757
|
rows = result.getRows();
|
|
@@ -1744,7 +1765,10 @@ async function runTier3(
|
|
|
1744
1765
|
}
|
|
1745
1766
|
|
|
1746
1767
|
// Group bars by date
|
|
1747
|
-
const barsByDate = new Map<
|
|
1768
|
+
const barsByDate = new Map<
|
|
1769
|
+
string,
|
|
1770
|
+
Array<{ time: string; open: number; high: number; low: number; close: number }>
|
|
1771
|
+
>();
|
|
1748
1772
|
for (const row of rows) {
|
|
1749
1773
|
const dateStr = String(row[dateIdx]);
|
|
1750
1774
|
const bar = {
|
|
@@ -1766,7 +1790,14 @@ async function runTier3(
|
|
|
1766
1790
|
}
|
|
1767
1791
|
|
|
1768
1792
|
// Compute timing fields for each date and batch update the enriched table
|
|
1769
|
-
const tier3Cols = [
|
|
1793
|
+
const tier3Cols = [
|
|
1794
|
+
"High_Time",
|
|
1795
|
+
"Low_Time",
|
|
1796
|
+
"High_Before_Low",
|
|
1797
|
+
"Reversal_Type",
|
|
1798
|
+
"Opening_Drive_Strength",
|
|
1799
|
+
"Intraday_Realized_Vol",
|
|
1800
|
+
];
|
|
1770
1801
|
const enrichedRows: Array<Record<string, unknown>> = [];
|
|
1771
1802
|
|
|
1772
1803
|
for (const [dateStr, bars] of barsByDate) {
|