cotdata 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cotdata/__init__.py +11 -0
- cotdata/config.py +35 -0
- cotdata/cot.py +62 -0
- cotdata/prices.py +43 -0
- cotdata/providers/__init__.py +2 -0
- cotdata/providers/base.py +14 -0
- cotdata/providers/cftc.py +129 -0
- cotdata/providers/cftc_disagg.py +146 -0
- cotdata/providers/cftc_tff.py +146 -0
- cotdata/providers/databento.py +280 -0
- cotdata/providers/norgate.py +78 -0
- cotdata/registry.py +127 -0
- cotdata/registry.yaml +105 -0
- cotdata/store.py +96 -0
- cotdata/update.py +42 -0
- cotdata-0.1.0.dist-info/METADATA +246 -0
- cotdata-0.1.0.dist-info/RECORD +21 -0
- cotdata-0.1.0.dist-info/WHEEL +5 -0
- cotdata-0.1.0.dist-info/entry_points.txt +2 -0
- cotdata-0.1.0.dist-info/licenses/LICENSE +21 -0
- cotdata-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""Databento provider — DORMANT. NOT in the live EOD path (Norgate replaced it).
|
|
2
|
+
|
|
3
|
+
Retained deliberately for:
|
|
4
|
+
1. the intraday news-failure work (Norgate has no intraday; Databento is the
|
|
5
|
+
source for the release-window reaction refinement of the CMR trigger), and
|
|
6
|
+
2. cross-checking Norgate's settlement close against Databento statistics.
|
|
7
|
+
|
|
8
|
+
Ported from cot-analyzer/src/core/market_data.py (GLBX.MDP3 ohlcv-1d + statistics).
|
|
9
|
+
The hard-won part preserved here is the STATISTICS extraction — Open Interest
|
|
10
|
+
(stat_type 9) and the settlement fix: use StatType.SETTLEMENT_PRICE == 3 (NOT 7 =
|
|
11
|
+
LOWEST_OFFER, which overwrote Close with the day's lowest offer), dated by ts_ref
|
|
12
|
+
(the session it applies to) not ts_event (final settle is disseminated next morning).
|
|
13
|
+
|
|
14
|
+
Standalone-adapted for cotdata: lazy `import databento` (behind the [databento]
|
|
15
|
+
extra), symbols from the registry, cache under $COTDATA_STORE/_cache/databento.
|
|
16
|
+
Requires DATABENTO_API_KEY. The daily path below is superseded by Norgate; the
|
|
17
|
+
intraday work will reuse this statistics logic against ohlcv-1h / trades schemas.
|
|
18
|
+
"""
|
|
19
|
+
import logging
|
|
20
|
+
import os
|
|
21
|
+
import sys
|
|
22
|
+
import time
|
|
23
|
+
import warnings
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
import pandas as pd
|
|
27
|
+
|
|
28
|
+
from .. import config
|
|
29
|
+
from ..registry import all_symbols
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
# Symbols Databento GLBX.MDP3 doesn't carry as .n.0 continuous (fall back to yfinance).
|
|
34
|
+
_DATABENTO_UNSUPPORTED = {"CC", "OJ", "SB", "KC", "LBR", "CT"}
|
|
35
|
+
_API_LAST_CHECKED = {} # in-memory throttle to avoid hammering the API
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _cache_dir() -> Path:
|
|
39
|
+
d = config.store_root() / "_cache" / "databento"
|
|
40
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
41
|
+
return d
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def run_batch_backfill(symbols: list) -> None:
|
|
45
|
+
"""Submit a Databento batch job to download massive historical daily data."""
|
|
46
|
+
import databento as db
|
|
47
|
+
db_key = os.environ.get("DATABENTO_API_KEY")
|
|
48
|
+
if not db_key:
|
|
49
|
+
raise ValueError("DATABENTO_API_KEY is missing. Cannot run batch backfill.")
|
|
50
|
+
client = db.Historical(key=db_key)
|
|
51
|
+
|
|
52
|
+
db_syms, sym_map = [], {}
|
|
53
|
+
for sym in symbols:
|
|
54
|
+
db_sym = f"{sym}.n.0"
|
|
55
|
+
db_syms.append(db_sym)
|
|
56
|
+
sym_map[db_sym] = sym
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
res = client.symbology.resolve(
|
|
60
|
+
dataset="GLBX.MDP3", symbols=db_syms, stype_in="continuous",
|
|
61
|
+
stype_out="instrument_id", start_date="2010-06-06",
|
|
62
|
+
end_date=pd.Timestamp.now().strftime("%Y-%m-%d"),
|
|
63
|
+
)
|
|
64
|
+
not_found = res.get("not_found", [])
|
|
65
|
+
if not_found:
|
|
66
|
+
db_syms = [s for s in db_syms if s not in not_found]
|
|
67
|
+
except Exception as e: # noqa: BLE001
|
|
68
|
+
logger.warning("Symbol resolution failed, submitting anyway: %s", e)
|
|
69
|
+
if not db_syms:
|
|
70
|
+
raise ValueError("No valid Databento symbols remain after validation.")
|
|
71
|
+
|
|
72
|
+
job = client.batch.submit_job(
|
|
73
|
+
dataset="GLBX.MDP3", symbols=db_syms, stype_in="continuous", schema="ohlcv-1d",
|
|
74
|
+
start="2010-06-06", end=pd.Timestamp.now().strftime("%Y-%m-%d"),
|
|
75
|
+
encoding="csv", split_symbols=False, delivery="download",
|
|
76
|
+
)
|
|
77
|
+
job_id = job.get("id")
|
|
78
|
+
if not job_id:
|
|
79
|
+
raise RuntimeError("Failed to submit batch job: no job ID returned.")
|
|
80
|
+
|
|
81
|
+
while True:
|
|
82
|
+
time.sleep(30)
|
|
83
|
+
my_job = next((j for j in client.batch.list_jobs() if j.get("id") == job_id), None)
|
|
84
|
+
if not my_job:
|
|
85
|
+
continue
|
|
86
|
+
state = my_job.get("state", "unknown")
|
|
87
|
+
if state == "done":
|
|
88
|
+
break
|
|
89
|
+
if state == "expired" or "fail" in state.lower():
|
|
90
|
+
raise RuntimeError(f"Batch job failed or expired: {my_job}")
|
|
91
|
+
|
|
92
|
+
import tempfile
|
|
93
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
94
|
+
paths = client.batch.download(job_id=job_id, output_dir=tmpdir)
|
|
95
|
+
for path in paths:
|
|
96
|
+
if not str(path).endswith(".csv"):
|
|
97
|
+
continue
|
|
98
|
+
df_raw = pd.read_csv(path)
|
|
99
|
+
if "symbol" not in df_raw.columns:
|
|
100
|
+
continue
|
|
101
|
+
for db_sym, df_sym in df_raw.groupby("symbol"):
|
|
102
|
+
internal_sym = sym_map.get(db_sym)
|
|
103
|
+
if not internal_sym:
|
|
104
|
+
continue
|
|
105
|
+
df_sym = df_sym.rename(columns={
|
|
106
|
+
"ts_event": "Date", "open": "Open", "high": "High",
|
|
107
|
+
"low": "Low", "close": "Close", "volume": "Volume"})
|
|
108
|
+
df_sym["Date"] = pd.to_datetime(df_sym["Date"]).dt.tz_localize(None).dt.normalize()
|
|
109
|
+
df_sym = df_sym.set_index("Date")
|
|
110
|
+
keep = [c for c in ("Open", "High", "Low", "Close", "Volume") if c in df_sym.columns]
|
|
111
|
+
df_clean = df_sym[keep].copy()
|
|
112
|
+
df_clean["Open Interest"] = float("nan")
|
|
113
|
+
df_clean = df_clean.dropna(subset=["Close"]).sort_index()
|
|
114
|
+
df_clean.to_parquet(_cache_dir() / f"{internal_sym}_daily.parquet")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def fetch_daily_ohlc(symbol: str, start_date: str = "2000-01-01",
|
|
118
|
+
force_refresh: bool = False, price_type: str = "close") -> pd.DataFrame:
|
|
119
|
+
"""Daily OHLC + Open Interest via Databento (GLBX.MDP3 ohlcv-1d + statistics),
|
|
120
|
+
append-only cache; yfinance fallback for _DATABENTO_UNSUPPORTED. price_type
|
|
121
|
+
'settlement' pulls stat_type 3 dated by ts_ref."""
|
|
122
|
+
import databento as db
|
|
123
|
+
display_sym = symbol
|
|
124
|
+
_pt_suffix = "" if price_type == "close" else f"_{price_type}"
|
|
125
|
+
cache_path = _cache_dir() / f"{symbol}_daily{_pt_suffix}.parquet"
|
|
126
|
+
|
|
127
|
+
local_df = pd.DataFrame()
|
|
128
|
+
if cache_path.exists() and not force_refresh:
|
|
129
|
+
try:
|
|
130
|
+
cached = pd.read_parquet(cache_path)
|
|
131
|
+
if not cached.empty:
|
|
132
|
+
has_oi = "Open Interest" in cached.columns and not cached["Open Interest"].isna().all()
|
|
133
|
+
if has_oi or symbol in _DATABENTO_UNSUPPORTED:
|
|
134
|
+
local_df = cached
|
|
135
|
+
except Exception as e: # noqa: BLE001
|
|
136
|
+
logger.warning("Failed to read cache for %s: %s", display_sym, e)
|
|
137
|
+
|
|
138
|
+
if local_df.empty:
|
|
139
|
+
fetch_start = "2000-01-01"
|
|
140
|
+
else:
|
|
141
|
+
last_date = local_df.index.max()
|
|
142
|
+
today = pd.Timestamp.now().normalize()
|
|
143
|
+
if last_date >= today - pd.Timedelta(days=1):
|
|
144
|
+
return local_df
|
|
145
|
+
if "--fast" in sys.argv:
|
|
146
|
+
return local_df
|
|
147
|
+
now = time.time()
|
|
148
|
+
if not force_refresh and (now - _API_LAST_CHECKED.get(symbol, 0) < 3600):
|
|
149
|
+
return local_df
|
|
150
|
+
_API_LAST_CHECKED[symbol] = now
|
|
151
|
+
fetch_start = (last_date + pd.Timedelta(days=1)).strftime("%Y-%m-%d")
|
|
152
|
+
|
|
153
|
+
fetch_end = (pd.Timestamp.now() - pd.Timedelta(days=1)).strftime("%Y-%m-%d")
|
|
154
|
+
if pd.Timestamp(fetch_start) >= pd.Timestamp(fetch_end):
|
|
155
|
+
return local_df
|
|
156
|
+
|
|
157
|
+
new_df = pd.DataFrame()
|
|
158
|
+
db_key = os.environ.get("DATABENTO_API_KEY")
|
|
159
|
+
if pd.Timestamp(fetch_start) < pd.Timestamp("2010-06-06"):
|
|
160
|
+
fetch_start = "2010-06-06" # GLBX.MDP3 history floor
|
|
161
|
+
db_success = False
|
|
162
|
+
|
|
163
|
+
if db_key and symbol not in _DATABENTO_UNSUPPORTED:
|
|
164
|
+
for attempt in range(1, 4):
|
|
165
|
+
try:
|
|
166
|
+
client = db.Historical(key=db_key)
|
|
167
|
+
db_sym = f"{symbol}.n.0"
|
|
168
|
+
with warnings.catch_warnings():
|
|
169
|
+
warnings.filterwarnings("ignore", message=".*No data found.*")
|
|
170
|
+
warnings.filterwarnings("ignore", message=".*did not resolve.*")
|
|
171
|
+
data = client.timeseries.get_range(
|
|
172
|
+
dataset="GLBX.MDP3", symbols=[db_sym], stype_in="continuous",
|
|
173
|
+
schema="ohlcv-1d", start=fetch_start, end=fetch_end)
|
|
174
|
+
raw = data.to_df()
|
|
175
|
+
if not raw.empty:
|
|
176
|
+
raw = raw.reset_index().rename(columns={
|
|
177
|
+
"ts_event": "Date", "open": "Open", "high": "High",
|
|
178
|
+
"low": "Low", "close": "Close", "volume": "Volume"})
|
|
179
|
+
raw["Date"] = pd.to_datetime(raw["Date"]).dt.tz_localize(None).dt.normalize()
|
|
180
|
+
raw = raw.set_index("Date")
|
|
181
|
+
keep = [c for c in ("Open", "High", "Low", "Close", "Volume") if c in raw.columns]
|
|
182
|
+
new_df = raw[keep].copy()
|
|
183
|
+
try:
|
|
184
|
+
with warnings.catch_warnings():
|
|
185
|
+
warnings.filterwarnings("ignore", message=".*No data found.*")
|
|
186
|
+
warnings.filterwarnings("ignore", message=".*did not resolve.*")
|
|
187
|
+
stat_raw = client.timeseries.get_range(
|
|
188
|
+
dataset="GLBX.MDP3", symbols=[db_sym], stype_in="continuous",
|
|
189
|
+
schema="statistics", start=fetch_start, end=fetch_end).to_df()
|
|
190
|
+
if not stat_raw.empty:
|
|
191
|
+
stat_raw = stat_raw.reset_index()
|
|
192
|
+
if "stat_type" in stat_raw.columns:
|
|
193
|
+
new_df = new_df.reset_index()
|
|
194
|
+
date_col = "ts_event"
|
|
195
|
+
# Open Interest = stat_type 9
|
|
196
|
+
oi_df = stat_raw[stat_raw["stat_type"] == 9].copy()
|
|
197
|
+
if not oi_df.empty:
|
|
198
|
+
oi_df["Date"] = pd.to_datetime(oi_df[date_col]).dt.tz_localize(None).dt.normalize()
|
|
199
|
+
oi_df["Open Interest"] = oi_df["quantity"] if "quantity" in oi_df.columns else oi_df["price"]
|
|
200
|
+
oi_df = oi_df.groupby("Date")["Open Interest"].last().reset_index()
|
|
201
|
+
new_df = pd.merge(new_df, oi_df, on="Date", how="left")
|
|
202
|
+
# Settlement = stat_type 3 (NOT 7=LOWEST_OFFER), dated by ts_ref
|
|
203
|
+
if price_type == "settlement":
|
|
204
|
+
stl_df = stat_raw[stat_raw["stat_type"] == 3].copy()
|
|
205
|
+
if not stl_df.empty:
|
|
206
|
+
settle_dt = "ts_ref" if "ts_ref" in stl_df.columns else date_col
|
|
207
|
+
stl_df["Date"] = pd.to_datetime(stl_df[settle_dt]).dt.tz_localize(None).dt.normalize()
|
|
208
|
+
stl_df["Settlement"] = stl_df["price"]
|
|
209
|
+
stl_df = stl_df.groupby("Date")["Settlement"].last().reset_index()
|
|
210
|
+
new_df = pd.merge(new_df, stl_df, on="Date", how="left")
|
|
211
|
+
if "Settlement" in new_df.columns:
|
|
212
|
+
new_df["Close"] = new_df["Settlement"].combine_first(new_df["Close"])
|
|
213
|
+
new_df = new_df.drop(columns=["Settlement"])
|
|
214
|
+
new_df = new_df.set_index("Date")
|
|
215
|
+
except Exception as e: # noqa: BLE001
|
|
216
|
+
logger.warning("Could not fetch Open Interest for %s: %s", display_sym, e)
|
|
217
|
+
if "Open Interest" not in new_df.columns:
|
|
218
|
+
new_df["Open Interest"] = float("nan")
|
|
219
|
+
db_success = True
|
|
220
|
+
break
|
|
221
|
+
db_success = True # empty (holiday) — don't trigger fallback
|
|
222
|
+
break
|
|
223
|
+
except Exception as exc: # noqa: BLE001
|
|
224
|
+
logger.warning("Databento download failed for %s (%d/3): %s", display_sym, attempt, exc)
|
|
225
|
+
if attempt < 3:
|
|
226
|
+
time.sleep(5)
|
|
227
|
+
|
|
228
|
+
if not db_success and symbol in _DATABENTO_UNSUPPORTED:
|
|
229
|
+
try:
|
|
230
|
+
import yfinance as yf
|
|
231
|
+
yf_df = yf.download(f"{symbol}=F", start=fetch_start,
|
|
232
|
+
end=pd.Timestamp.now().strftime("%Y-%m-%d"), progress=False)
|
|
233
|
+
if not yf_df.empty:
|
|
234
|
+
yf_df = yf_df.reset_index()
|
|
235
|
+
if isinstance(yf_df.columns, pd.MultiIndex):
|
|
236
|
+
yf_df.columns = [c[0] for c in yf_df.columns]
|
|
237
|
+
yf_df["Date"] = pd.to_datetime(yf_df["Date"]).dt.tz_localize(None).dt.normalize()
|
|
238
|
+
yf_df = yf_df.set_index("Date")
|
|
239
|
+
keep = [c for c in ("Open", "High", "Low", "Close", "Volume") if c in yf_df.columns]
|
|
240
|
+
new_df = yf_df[keep].copy()
|
|
241
|
+
new_df["Open Interest"] = float("nan")
|
|
242
|
+
except Exception as yf_exc: # noqa: BLE001
|
|
243
|
+
logger.error("yfinance fallback failed for %s: %s", display_sym, yf_exc)
|
|
244
|
+
|
|
245
|
+
if not local_df.empty and not new_df.empty:
|
|
246
|
+
new_df = new_df[new_df.index > local_df.index.max()]
|
|
247
|
+
if new_df.empty:
|
|
248
|
+
return local_df
|
|
249
|
+
new_df = new_df.dropna(subset=["Close"])
|
|
250
|
+
combined = pd.concat([local_df, new_df]) if not local_df.empty else new_df
|
|
251
|
+
combined = combined[~combined.index.duplicated(keep="last")].sort_index()
|
|
252
|
+
combined.to_parquet(cache_path)
|
|
253
|
+
return combined
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def update_all_daily_prices(force_refresh: bool = False) -> None:
|
|
257
|
+
"""Refresh the Databento price cache for all registry symbols (dormant path)."""
|
|
258
|
+
to_batch = []
|
|
259
|
+
for s in all_symbols():
|
|
260
|
+
symbol = s.internal
|
|
261
|
+
raw_cache_path = _cache_dir() / f"{symbol}_daily.parquet"
|
|
262
|
+
needs = force_refresh or not raw_cache_path.exists()
|
|
263
|
+
if not needs:
|
|
264
|
+
try:
|
|
265
|
+
if pd.read_parquet(raw_cache_path).empty:
|
|
266
|
+
needs = True
|
|
267
|
+
except Exception: # noqa: BLE001
|
|
268
|
+
needs = True
|
|
269
|
+
if needs:
|
|
270
|
+
to_batch.append(symbol)
|
|
271
|
+
else:
|
|
272
|
+
try:
|
|
273
|
+
fetch_daily_ohlc(symbol)
|
|
274
|
+
except Exception as e: # noqa: BLE001
|
|
275
|
+
logger.error("Failed daily gap fetch for %s: %s", symbol, e)
|
|
276
|
+
if to_batch:
|
|
277
|
+
try:
|
|
278
|
+
run_batch_backfill(to_batch)
|
|
279
|
+
except Exception as e: # noqa: BLE001
|
|
280
|
+
logger.error("Batch backfill failed: %s", e)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Norgate price producer — RUNS ON WINDOWS (Norgate Data Updater running +
|
|
2
|
+
`norgatedata`). The active EOD source: exchange settlement close, deep history,
|
|
3
|
+
back-adjusted (gap-free, shape-preserving) continuous contracts.
|
|
4
|
+
|
|
5
|
+
ADJUSTMENT (verified 2026-07 via test_adjustment.py): Norgate selects continuous
|
|
6
|
+
adjustment by SYMBOL SUFFIX, not the stock_price_adjustment_setting kwarg. The
|
|
7
|
+
BASE symbol '&ES' is UNADJUSTED (shows real calendar-spread gaps at each roll,
|
|
8
|
+
e.g. +146 pts at the 2026-06 Jun→Sep roll). '&ES_CCB' is BACK-ADJUSTED (gaps
|
|
9
|
+
stitched out). A close-based stop needs the gap-free series → we fetch _CCB.
|
|
10
|
+
"""
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
from ..registry import REGISTRY, all_symbols
|
|
14
|
+
from .. import store
|
|
15
|
+
|
|
16
|
+
CCB_SUFFIX = "_CCB" # Norgate "Continuous Contract Back-adjusted"
|
|
17
|
+
# If roll-day overnight moves exceed this multiple of the normal-day median, the
|
|
18
|
+
# series looks UNADJUSTED (calendar-spread gaps not stitched). Self-calibrating
|
|
19
|
+
# per symbol, so it works across products with different spread magnitudes.
|
|
20
|
+
ROLL_GAP_RATIO_WARN = 1.5
|
|
21
|
+
|
|
22
|
+
_COLMAP = {
|
|
23
|
+
"Open": "Open", "High": "High", "Low": "Low", "Close": "Close",
|
|
24
|
+
"Volume": "Volume", "Open Interest": "Open Interest",
|
|
25
|
+
"Delivery Month": "Delivery Month", # kept → exact roll detection downstream
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def fetch(internal_symbol: str, start: str = "1970-01-01") -> pd.DataFrame:
|
|
30
|
+
"""Fetch Norgate back-adjusted continuous bars: settlement close, gap-free."""
|
|
31
|
+
import norgatedata # imported lazily; only present on the Windows producer
|
|
32
|
+
ng_sym = REGISTRY[internal_symbol].norgate + CCB_SUFFIX # "&ES" → "&ES_CCB"
|
|
33
|
+
df = norgatedata.price_timeseries(
|
|
34
|
+
ng_sym,
|
|
35
|
+
padding_setting=norgatedata.PaddingType.NONE,
|
|
36
|
+
timeseriesformat="pandas-dataframe",
|
|
37
|
+
start_date=start,
|
|
38
|
+
)
|
|
39
|
+
df = df.rename(columns=_COLMAP)
|
|
40
|
+
keep = [c for c in _COLMAP.values() if c in df.columns]
|
|
41
|
+
out = df[keep].copy()
|
|
42
|
+
out.index = pd.to_datetime(out.index).tz_localize(None).normalize()
|
|
43
|
+
out.index.name = "Date"
|
|
44
|
+
return out.sort_index()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _check_roll_gaps(internal_symbol: str, df: pd.DataFrame) -> bool:
|
|
48
|
+
"""Warn if roll-day overnight moves ≫ normal-day moves — the signature of an
|
|
49
|
+
UNADJUSTED continuous. Returns True if it looks unadjusted. Self-calibrating:
|
|
50
|
+
compares each symbol's roll-day moves to its own non-roll baseline."""
|
|
51
|
+
if "Delivery Month" not in df.columns or len(df) < 60:
|
|
52
|
+
return False
|
|
53
|
+
dm = df["Delivery Month"]
|
|
54
|
+
roll = dm.ne(dm.shift()) & dm.shift().notna()
|
|
55
|
+
if int(roll.sum()) < 8:
|
|
56
|
+
return False
|
|
57
|
+
overnight = (df["Close"] - df["Close"].shift(1)).abs()
|
|
58
|
+
roll_med = overnight[roll].median()
|
|
59
|
+
nonroll_med = overnight[~roll].median()
|
|
60
|
+
if nonroll_med and roll_med > ROLL_GAP_RATIO_WARN * nonroll_med:
|
|
61
|
+
print(f" ⚠️ {internal_symbol}: roll-day moves {roll_med:.1f} vs normal {nonroll_med:.1f} "
|
|
62
|
+
f"({roll_med / nonroll_med:.1f}x) — series looks UNADJUSTED. Expected the _CCB "
|
|
63
|
+
f"back-adjusted symbol; a close-based stop would false-trigger on roll gaps.")
|
|
64
|
+
return True
|
|
65
|
+
return False
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def update(symbols=None) -> None:
|
|
69
|
+
"""Fetch + write to the store for the given internal symbols (backadj only)."""
|
|
70
|
+
syms = symbols or [s.internal for s in all_symbols()]
|
|
71
|
+
for sym in syms:
|
|
72
|
+
try:
|
|
73
|
+
out = fetch(sym)
|
|
74
|
+
_check_roll_gaps(sym, out) # sanity: warn (don't block) if it looks unadjusted
|
|
75
|
+
store.write_prices(sym, "backadj", out, source="norgate")
|
|
76
|
+
print(f"{sym:5s}: {len(out):6d} bars -> store")
|
|
77
|
+
except Exception as e: # noqa: BLE001
|
|
78
|
+
print(f"{sym:5s}: FAILED — {e}")
|
cotdata/registry.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""The single symbol registry: internal ↔ Norgate ↔ CFTC code ↔ asset class.
|
|
2
|
+
Replaces the scattered maps (CotSymbolCodeMap, the databento_mapping).
|
|
3
|
+
|
|
4
|
+
Scope = FIXED IDENTITY facts only (never change). TUNABLE strategy parameters
|
|
5
|
+
(positioning-index CustomLookbackWeeks, thresholds, TV chart symbols) stay in
|
|
6
|
+
cot-analyzer/config/params.yaml — the data layer must not carry strategy knobs.
|
|
7
|
+
|
|
8
|
+
The table lives in registry.yaml (asset_class -> symbol -> attrs), loaded at
|
|
9
|
+
import. Point $COTDATA_REGISTRY at an alternate file to override it.
|
|
10
|
+
|
|
11
|
+
Sources:
|
|
12
|
+
• cftc_code — cot-analyzer CotSymbolCodeMap
|
|
13
|
+
• asset_class — CotIndexer asset classes
|
|
14
|
+
• norgate — '&' + CME root (e.g., "&ES"); required by norgatedata.price_timeseries().
|
|
15
|
+
is_equity is True only for the four equity indices (a fixed classification the
|
|
16
|
+
equity-vs-commodity rules key off); it is derived from asset_class == "Equities"
|
|
17
|
+
unless a symbol overrides it explicitly.
|
|
18
|
+
"""
|
|
19
|
+
import os
|
|
20
|
+
import yaml
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Dict, List, Optional
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class Symbol:
|
|
28
|
+
internal: str # pipeline root, e.g. "ES"
|
|
29
|
+
norgate: str # Norgate continuous symbol, e.g. "&ES"
|
|
30
|
+
asset_class: str
|
|
31
|
+
is_equity: bool
|
|
32
|
+
report_type: str = "disagg" # "tff" for financials, "disagg" for commodities
|
|
33
|
+
cftc_code: Optional[str] = None
|
|
34
|
+
# Predecessor CFTC codes from earlier exchange/contract listings of the SAME
|
|
35
|
+
# instrument, stitched in chronologically behind cftc_code by get_cot. Each
|
|
36
|
+
# entry is a bare code string (scale 1.0) or a (code, scale) tuple. Kept as a
|
|
37
|
+
# tuple of hashable entries so Symbol (frozen) stays hashable.
|
|
38
|
+
hist_codes: tuple = ()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def hist_code_scales(hist_codes) -> List[tuple]:
|
|
42
|
+
"""Normalize hist_codes entries (str or (code, scale)) → list of (code, scale)."""
|
|
43
|
+
out = []
|
|
44
|
+
for h in hist_codes:
|
|
45
|
+
out.append((h[0], float(h[1])) if isinstance(h, (list, tuple)) else (h, 1.0))
|
|
46
|
+
return out
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _coerce_hist_codes(raw) -> tuple:
|
|
50
|
+
"""YAML encodes scaled entries as lists ([code, scale]); store them as tuples
|
|
51
|
+
so Symbol stays hashable (a list nested in the tuple would break hashing)."""
|
|
52
|
+
return tuple(tuple(h) if isinstance(h, list) else h for h in (raw or []))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def load_registry(yaml_path=None) -> Dict[str, Symbol]:
|
|
56
|
+
"""Build the symbol registry from YAML.
|
|
57
|
+
|
|
58
|
+
Path resolution: explicit ``yaml_path`` arg, else $COTDATA_REGISTRY, else the
|
|
59
|
+
packaged registry.yaml beside this module. Raises with a clear message on a
|
|
60
|
+
missing/malformed file or an invalid entry, since the whole package depends
|
|
61
|
+
on this at import.
|
|
62
|
+
"""
|
|
63
|
+
yaml_path = yaml_path or os.environ.get(
|
|
64
|
+
"COTDATA_REGISTRY", Path(__file__).parent / "registry.yaml")
|
|
65
|
+
try:
|
|
66
|
+
with open(yaml_path, "r") as f:
|
|
67
|
+
data = yaml.safe_load(f)
|
|
68
|
+
except FileNotFoundError as e:
|
|
69
|
+
raise FileNotFoundError(
|
|
70
|
+
f"cotdata registry file not found: {yaml_path}. Set $COTDATA_REGISTRY "
|
|
71
|
+
f"to a valid registry YAML, or restore the packaged registry.yaml.") from e
|
|
72
|
+
except yaml.YAMLError as e:
|
|
73
|
+
raise ValueError(f"cotdata registry YAML is malformed ({yaml_path}): {e}") from e
|
|
74
|
+
|
|
75
|
+
if not isinstance(data, dict):
|
|
76
|
+
raise ValueError(
|
|
77
|
+
f"cotdata registry YAML must be a mapping of asset_class -> symbols "
|
|
78
|
+
f"({yaml_path}); got {type(data).__name__}.")
|
|
79
|
+
|
|
80
|
+
registry: Dict[str, Symbol] = {}
|
|
81
|
+
for asset_class, symbols in data.items():
|
|
82
|
+
if not isinstance(symbols, dict):
|
|
83
|
+
raise ValueError(
|
|
84
|
+
f"cotdata registry: asset class '{asset_class}' must map to a dict of "
|
|
85
|
+
f"symbols, got {type(symbols).__name__}.")
|
|
86
|
+
for internal, attrs in symbols.items():
|
|
87
|
+
attrs = attrs or {}
|
|
88
|
+
if not isinstance(attrs, dict):
|
|
89
|
+
raise ValueError(
|
|
90
|
+
f"cotdata registry: symbol '{internal}' must map to a dict of "
|
|
91
|
+
f"attrs (e.g. 'cftc_code: ...'), got {type(attrs).__name__}.")
|
|
92
|
+
if internal in registry:
|
|
93
|
+
raise ValueError(
|
|
94
|
+
f"cotdata registry: duplicate symbol '{internal}' "
|
|
95
|
+
f"(re-declared under asset class '{asset_class}').")
|
|
96
|
+
if not attrs.get("cftc_code"):
|
|
97
|
+
raise ValueError(
|
|
98
|
+
f"cotdata registry: symbol '{internal}' is missing cftc_code.")
|
|
99
|
+
registry[internal] = Symbol(
|
|
100
|
+
internal=internal,
|
|
101
|
+
norgate=attrs.get("norgate", f"&{internal}"),
|
|
102
|
+
asset_class=asset_class,
|
|
103
|
+
# Derived from the class so the two can't drift; an explicit
|
|
104
|
+
# is_equity in the YAML still wins if a symbol ever needs it.
|
|
105
|
+
is_equity=attrs.get("is_equity", asset_class == "Equities"),
|
|
106
|
+
# Financials use TFF (Traders in Financial Futures), Commodities use Disaggregated
|
|
107
|
+
report_type=attrs.get("report_type", "tff" if asset_class in ("Equities", "FX", "Rates") else "disagg"),
|
|
108
|
+
cftc_code=attrs["cftc_code"],
|
|
109
|
+
hist_codes=_coerce_hist_codes(attrs.get("hist_codes")),
|
|
110
|
+
)
|
|
111
|
+
return registry
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# Built once at import from the default location.
|
|
115
|
+
REGISTRY: Dict[str, Symbol] = load_registry()
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def symbol(internal: str) -> Symbol:
|
|
119
|
+
return REGISTRY[internal]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def all_symbols() -> List[Symbol]:
|
|
123
|
+
return list(REGISTRY.values())
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def by_asset_class(asset_class: str) -> List[Symbol]:
|
|
127
|
+
return [s for s in REGISTRY.values() if s.asset_class == asset_class]
|
cotdata/registry.yaml
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Equities:
|
|
2
|
+
ES:
|
|
3
|
+
cftc_code: "13874A"
|
|
4
|
+
NQ:
|
|
5
|
+
cftc_code: "209742"
|
|
6
|
+
YM:
|
|
7
|
+
cftc_code: "124603"
|
|
8
|
+
RTY:
|
|
9
|
+
cftc_code: "239742"
|
|
10
|
+
hist_codes:
|
|
11
|
+
- "23977A"
|
|
12
|
+
|
|
13
|
+
Metals:
|
|
14
|
+
GC:
|
|
15
|
+
cftc_code: "088691"
|
|
16
|
+
SI:
|
|
17
|
+
cftc_code: "084691"
|
|
18
|
+
HG:
|
|
19
|
+
cftc_code: "085692"
|
|
20
|
+
PL:
|
|
21
|
+
cftc_code: "076651"
|
|
22
|
+
PA:
|
|
23
|
+
cftc_code: "075651"
|
|
24
|
+
|
|
25
|
+
Energies:
|
|
26
|
+
CL:
|
|
27
|
+
cftc_code: "067651"
|
|
28
|
+
RB:
|
|
29
|
+
cftc_code: "111659"
|
|
30
|
+
HO:
|
|
31
|
+
cftc_code: "022651"
|
|
32
|
+
NG:
|
|
33
|
+
cftc_code: "023651"
|
|
34
|
+
|
|
35
|
+
Grains:
|
|
36
|
+
ZC:
|
|
37
|
+
cftc_code: "002602"
|
|
38
|
+
ZS:
|
|
39
|
+
cftc_code: "005602"
|
|
40
|
+
ZM:
|
|
41
|
+
cftc_code: "026603"
|
|
42
|
+
ZL:
|
|
43
|
+
cftc_code: "007601"
|
|
44
|
+
ZW:
|
|
45
|
+
cftc_code: "001602"
|
|
46
|
+
|
|
47
|
+
Currencies:
|
|
48
|
+
6E:
|
|
49
|
+
cftc_code: "099741"
|
|
50
|
+
6A:
|
|
51
|
+
cftc_code: "232741"
|
|
52
|
+
6B:
|
|
53
|
+
cftc_code: "096742"
|
|
54
|
+
6C:
|
|
55
|
+
cftc_code: "090741"
|
|
56
|
+
6J:
|
|
57
|
+
cftc_code: "097741"
|
|
58
|
+
6S:
|
|
59
|
+
cftc_code: "092741"
|
|
60
|
+
6M:
|
|
61
|
+
cftc_code: "095741"
|
|
62
|
+
6N:
|
|
63
|
+
cftc_code: "112741"
|
|
64
|
+
DX:
|
|
65
|
+
cftc_code: "098662"
|
|
66
|
+
|
|
67
|
+
Fixed Income:
|
|
68
|
+
ZN:
|
|
69
|
+
cftc_code: "043602"
|
|
70
|
+
ZT:
|
|
71
|
+
cftc_code: "042601"
|
|
72
|
+
ZF:
|
|
73
|
+
cftc_code: "044601"
|
|
74
|
+
ZB:
|
|
75
|
+
cftc_code: "020601"
|
|
76
|
+
|
|
77
|
+
Softs:
|
|
78
|
+
SB:
|
|
79
|
+
cftc_code: "080732"
|
|
80
|
+
CT:
|
|
81
|
+
cftc_code: "033661"
|
|
82
|
+
CC:
|
|
83
|
+
cftc_code: "073732"
|
|
84
|
+
KC:
|
|
85
|
+
cftc_code: "083731"
|
|
86
|
+
OJ:
|
|
87
|
+
cftc_code: "040701"
|
|
88
|
+
LBR:
|
|
89
|
+
cftc_code: "058644"
|
|
90
|
+
hist_codes:
|
|
91
|
+
- ["058643", 4.0]
|
|
92
|
+
|
|
93
|
+
Live Stock:
|
|
94
|
+
LE:
|
|
95
|
+
cftc_code: "057642"
|
|
96
|
+
HE:
|
|
97
|
+
cftc_code: "054642"
|
|
98
|
+
GF:
|
|
99
|
+
cftc_code: "061641"
|
|
100
|
+
|
|
101
|
+
Crypto:
|
|
102
|
+
BTC:
|
|
103
|
+
cftc_code: "133741"
|
|
104
|
+
ETH:
|
|
105
|
+
cftc_code: "146021"
|