cotdata 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,280 @@
1
+ """Databento provider — DORMANT. NOT in the live EOD path (Norgate replaced it).
2
+
3
+ Retained deliberately for:
4
+ 1. the intraday news-failure work (Norgate has no intraday; Databento is the
5
+ source for the release-window reaction refinement of the CMR trigger), and
6
+ 2. cross-checking Norgate's settlement close against Databento statistics.
7
+
8
+ Ported from cot-analyzer/src/core/market_data.py (GLBX.MDP3 ohlcv-1d + statistics).
9
+ The hard-won part preserved here is the STATISTICS extraction — Open Interest
10
+ (stat_type 9) and the settlement fix: use StatType.SETTLEMENT_PRICE == 3 (NOT 7 =
11
+ LOWEST_OFFER, which overwrote Close with the day's lowest offer), dated by ts_ref
12
+ (the session it applies to) not ts_event (final settle is disseminated next morning).
13
+
14
+ Standalone-adapted for cotdata: lazy `import databento` (behind the [databento]
15
+ extra), symbols from the registry, cache under $COTDATA_STORE/_cache/databento.
16
+ Requires DATABENTO_API_KEY. The daily path below is superseded by Norgate; the
17
+ intraday work will reuse this statistics logic against ohlcv-1h / trades schemas.
18
+ """
19
+ import logging
20
+ import os
21
+ import sys
22
+ import time
23
+ import warnings
24
+ from pathlib import Path
25
+
26
+ import pandas as pd
27
+
28
+ from .. import config
29
+ from ..registry import all_symbols
30
+
31
+ logger = logging.getLogger(__name__)
32
+
33
+ # Symbols Databento GLBX.MDP3 doesn't carry as .n.0 continuous (fall back to yfinance).
34
+ _DATABENTO_UNSUPPORTED = {"CC", "OJ", "SB", "KC", "LBR", "CT"}
35
+ _API_LAST_CHECKED = {} # in-memory throttle to avoid hammering the API
36
+
37
+
38
+ def _cache_dir() -> Path:
39
+ d = config.store_root() / "_cache" / "databento"
40
+ d.mkdir(parents=True, exist_ok=True)
41
+ return d
42
+
43
+
44
+ def run_batch_backfill(symbols: list) -> None:
45
+ """Submit a Databento batch job to download massive historical daily data."""
46
+ import databento as db
47
+ db_key = os.environ.get("DATABENTO_API_KEY")
48
+ if not db_key:
49
+ raise ValueError("DATABENTO_API_KEY is missing. Cannot run batch backfill.")
50
+ client = db.Historical(key=db_key)
51
+
52
+ db_syms, sym_map = [], {}
53
+ for sym in symbols:
54
+ db_sym = f"{sym}.n.0"
55
+ db_syms.append(db_sym)
56
+ sym_map[db_sym] = sym
57
+
58
+ try:
59
+ res = client.symbology.resolve(
60
+ dataset="GLBX.MDP3", symbols=db_syms, stype_in="continuous",
61
+ stype_out="instrument_id", start_date="2010-06-06",
62
+ end_date=pd.Timestamp.now().strftime("%Y-%m-%d"),
63
+ )
64
+ not_found = res.get("not_found", [])
65
+ if not_found:
66
+ db_syms = [s for s in db_syms if s not in not_found]
67
+ except Exception as e: # noqa: BLE001
68
+ logger.warning("Symbol resolution failed, submitting anyway: %s", e)
69
+ if not db_syms:
70
+ raise ValueError("No valid Databento symbols remain after validation.")
71
+
72
+ job = client.batch.submit_job(
73
+ dataset="GLBX.MDP3", symbols=db_syms, stype_in="continuous", schema="ohlcv-1d",
74
+ start="2010-06-06", end=pd.Timestamp.now().strftime("%Y-%m-%d"),
75
+ encoding="csv", split_symbols=False, delivery="download",
76
+ )
77
+ job_id = job.get("id")
78
+ if not job_id:
79
+ raise RuntimeError("Failed to submit batch job: no job ID returned.")
80
+
81
+ while True:
82
+ time.sleep(30)
83
+ my_job = next((j for j in client.batch.list_jobs() if j.get("id") == job_id), None)
84
+ if not my_job:
85
+ continue
86
+ state = my_job.get("state", "unknown")
87
+ if state == "done":
88
+ break
89
+ if state == "expired" or "fail" in state.lower():
90
+ raise RuntimeError(f"Batch job failed or expired: {my_job}")
91
+
92
+ import tempfile
93
+ with tempfile.TemporaryDirectory() as tmpdir:
94
+ paths = client.batch.download(job_id=job_id, output_dir=tmpdir)
95
+ for path in paths:
96
+ if not str(path).endswith(".csv"):
97
+ continue
98
+ df_raw = pd.read_csv(path)
99
+ if "symbol" not in df_raw.columns:
100
+ continue
101
+ for db_sym, df_sym in df_raw.groupby("symbol"):
102
+ internal_sym = sym_map.get(db_sym)
103
+ if not internal_sym:
104
+ continue
105
+ df_sym = df_sym.rename(columns={
106
+ "ts_event": "Date", "open": "Open", "high": "High",
107
+ "low": "Low", "close": "Close", "volume": "Volume"})
108
+ df_sym["Date"] = pd.to_datetime(df_sym["Date"]).dt.tz_localize(None).dt.normalize()
109
+ df_sym = df_sym.set_index("Date")
110
+ keep = [c for c in ("Open", "High", "Low", "Close", "Volume") if c in df_sym.columns]
111
+ df_clean = df_sym[keep].copy()
112
+ df_clean["Open Interest"] = float("nan")
113
+ df_clean = df_clean.dropna(subset=["Close"]).sort_index()
114
+ df_clean.to_parquet(_cache_dir() / f"{internal_sym}_daily.parquet")
115
+
116
+
117
+ def fetch_daily_ohlc(symbol: str, start_date: str = "2000-01-01",
118
+ force_refresh: bool = False, price_type: str = "close") -> pd.DataFrame:
119
+ """Daily OHLC + Open Interest via Databento (GLBX.MDP3 ohlcv-1d + statistics),
120
+ append-only cache; yfinance fallback for _DATABENTO_UNSUPPORTED. price_type
121
+ 'settlement' pulls stat_type 3 dated by ts_ref."""
122
+ import databento as db
123
+ display_sym = symbol
124
+ _pt_suffix = "" if price_type == "close" else f"_{price_type}"
125
+ cache_path = _cache_dir() / f"{symbol}_daily{_pt_suffix}.parquet"
126
+
127
+ local_df = pd.DataFrame()
128
+ if cache_path.exists() and not force_refresh:
129
+ try:
130
+ cached = pd.read_parquet(cache_path)
131
+ if not cached.empty:
132
+ has_oi = "Open Interest" in cached.columns and not cached["Open Interest"].isna().all()
133
+ if has_oi or symbol in _DATABENTO_UNSUPPORTED:
134
+ local_df = cached
135
+ except Exception as e: # noqa: BLE001
136
+ logger.warning("Failed to read cache for %s: %s", display_sym, e)
137
+
138
+ if local_df.empty:
139
+ fetch_start = "2000-01-01"
140
+ else:
141
+ last_date = local_df.index.max()
142
+ today = pd.Timestamp.now().normalize()
143
+ if last_date >= today - pd.Timedelta(days=1):
144
+ return local_df
145
+ if "--fast" in sys.argv:
146
+ return local_df
147
+ now = time.time()
148
+ if not force_refresh and (now - _API_LAST_CHECKED.get(symbol, 0) < 3600):
149
+ return local_df
150
+ _API_LAST_CHECKED[symbol] = now
151
+ fetch_start = (last_date + pd.Timedelta(days=1)).strftime("%Y-%m-%d")
152
+
153
+ fetch_end = (pd.Timestamp.now() - pd.Timedelta(days=1)).strftime("%Y-%m-%d")
154
+ if pd.Timestamp(fetch_start) >= pd.Timestamp(fetch_end):
155
+ return local_df
156
+
157
+ new_df = pd.DataFrame()
158
+ db_key = os.environ.get("DATABENTO_API_KEY")
159
+ if pd.Timestamp(fetch_start) < pd.Timestamp("2010-06-06"):
160
+ fetch_start = "2010-06-06" # GLBX.MDP3 history floor
161
+ db_success = False
162
+
163
+ if db_key and symbol not in _DATABENTO_UNSUPPORTED:
164
+ for attempt in range(1, 4):
165
+ try:
166
+ client = db.Historical(key=db_key)
167
+ db_sym = f"{symbol}.n.0"
168
+ with warnings.catch_warnings():
169
+ warnings.filterwarnings("ignore", message=".*No data found.*")
170
+ warnings.filterwarnings("ignore", message=".*did not resolve.*")
171
+ data = client.timeseries.get_range(
172
+ dataset="GLBX.MDP3", symbols=[db_sym], stype_in="continuous",
173
+ schema="ohlcv-1d", start=fetch_start, end=fetch_end)
174
+ raw = data.to_df()
175
+ if not raw.empty:
176
+ raw = raw.reset_index().rename(columns={
177
+ "ts_event": "Date", "open": "Open", "high": "High",
178
+ "low": "Low", "close": "Close", "volume": "Volume"})
179
+ raw["Date"] = pd.to_datetime(raw["Date"]).dt.tz_localize(None).dt.normalize()
180
+ raw = raw.set_index("Date")
181
+ keep = [c for c in ("Open", "High", "Low", "Close", "Volume") if c in raw.columns]
182
+ new_df = raw[keep].copy()
183
+ try:
184
+ with warnings.catch_warnings():
185
+ warnings.filterwarnings("ignore", message=".*No data found.*")
186
+ warnings.filterwarnings("ignore", message=".*did not resolve.*")
187
+ stat_raw = client.timeseries.get_range(
188
+ dataset="GLBX.MDP3", symbols=[db_sym], stype_in="continuous",
189
+ schema="statistics", start=fetch_start, end=fetch_end).to_df()
190
+ if not stat_raw.empty:
191
+ stat_raw = stat_raw.reset_index()
192
+ if "stat_type" in stat_raw.columns:
193
+ new_df = new_df.reset_index()
194
+ date_col = "ts_event"
195
+ # Open Interest = stat_type 9
196
+ oi_df = stat_raw[stat_raw["stat_type"] == 9].copy()
197
+ if not oi_df.empty:
198
+ oi_df["Date"] = pd.to_datetime(oi_df[date_col]).dt.tz_localize(None).dt.normalize()
199
+ oi_df["Open Interest"] = oi_df["quantity"] if "quantity" in oi_df.columns else oi_df["price"]
200
+ oi_df = oi_df.groupby("Date")["Open Interest"].last().reset_index()
201
+ new_df = pd.merge(new_df, oi_df, on="Date", how="left")
202
+ # Settlement = stat_type 3 (NOT 7=LOWEST_OFFER), dated by ts_ref
203
+ if price_type == "settlement":
204
+ stl_df = stat_raw[stat_raw["stat_type"] == 3].copy()
205
+ if not stl_df.empty:
206
+ settle_dt = "ts_ref" if "ts_ref" in stl_df.columns else date_col
207
+ stl_df["Date"] = pd.to_datetime(stl_df[settle_dt]).dt.tz_localize(None).dt.normalize()
208
+ stl_df["Settlement"] = stl_df["price"]
209
+ stl_df = stl_df.groupby("Date")["Settlement"].last().reset_index()
210
+ new_df = pd.merge(new_df, stl_df, on="Date", how="left")
211
+ if "Settlement" in new_df.columns:
212
+ new_df["Close"] = new_df["Settlement"].combine_first(new_df["Close"])
213
+ new_df = new_df.drop(columns=["Settlement"])
214
+ new_df = new_df.set_index("Date")
215
+ except Exception as e: # noqa: BLE001
216
+ logger.warning("Could not fetch Open Interest for %s: %s", display_sym, e)
217
+ if "Open Interest" not in new_df.columns:
218
+ new_df["Open Interest"] = float("nan")
219
+ db_success = True
220
+ break
221
+ db_success = True # empty (holiday) — don't trigger fallback
222
+ break
223
+ except Exception as exc: # noqa: BLE001
224
+ logger.warning("Databento download failed for %s (%d/3): %s", display_sym, attempt, exc)
225
+ if attempt < 3:
226
+ time.sleep(5)
227
+
228
+ if not db_success and symbol in _DATABENTO_UNSUPPORTED:
229
+ try:
230
+ import yfinance as yf
231
+ yf_df = yf.download(f"{symbol}=F", start=fetch_start,
232
+ end=pd.Timestamp.now().strftime("%Y-%m-%d"), progress=False)
233
+ if not yf_df.empty:
234
+ yf_df = yf_df.reset_index()
235
+ if isinstance(yf_df.columns, pd.MultiIndex):
236
+ yf_df.columns = [c[0] for c in yf_df.columns]
237
+ yf_df["Date"] = pd.to_datetime(yf_df["Date"]).dt.tz_localize(None).dt.normalize()
238
+ yf_df = yf_df.set_index("Date")
239
+ keep = [c for c in ("Open", "High", "Low", "Close", "Volume") if c in yf_df.columns]
240
+ new_df = yf_df[keep].copy()
241
+ new_df["Open Interest"] = float("nan")
242
+ except Exception as yf_exc: # noqa: BLE001
243
+ logger.error("yfinance fallback failed for %s: %s", display_sym, yf_exc)
244
+
245
+ if not local_df.empty and not new_df.empty:
246
+ new_df = new_df[new_df.index > local_df.index.max()]
247
+ if new_df.empty:
248
+ return local_df
249
+ new_df = new_df.dropna(subset=["Close"])
250
+ combined = pd.concat([local_df, new_df]) if not local_df.empty else new_df
251
+ combined = combined[~combined.index.duplicated(keep="last")].sort_index()
252
+ combined.to_parquet(cache_path)
253
+ return combined
254
+
255
+
256
+ def update_all_daily_prices(force_refresh: bool = False) -> None:
257
+ """Refresh the Databento price cache for all registry symbols (dormant path)."""
258
+ to_batch = []
259
+ for s in all_symbols():
260
+ symbol = s.internal
261
+ raw_cache_path = _cache_dir() / f"{symbol}_daily.parquet"
262
+ needs = force_refresh or not raw_cache_path.exists()
263
+ if not needs:
264
+ try:
265
+ if pd.read_parquet(raw_cache_path).empty:
266
+ needs = True
267
+ except Exception: # noqa: BLE001
268
+ needs = True
269
+ if needs:
270
+ to_batch.append(symbol)
271
+ else:
272
+ try:
273
+ fetch_daily_ohlc(symbol)
274
+ except Exception as e: # noqa: BLE001
275
+ logger.error("Failed daily gap fetch for %s: %s", symbol, e)
276
+ if to_batch:
277
+ try:
278
+ run_batch_backfill(to_batch)
279
+ except Exception as e: # noqa: BLE001
280
+ logger.error("Batch backfill failed: %s", e)
@@ -0,0 +1,78 @@
1
+ """Norgate price producer — RUNS ON WINDOWS (Norgate Data Updater running +
2
+ `norgatedata`). The active EOD source: exchange settlement close, deep history,
3
+ back-adjusted (gap-free, shape-preserving) continuous contracts.
4
+
5
+ ADJUSTMENT (verified 2026-07 via test_adjustment.py): Norgate selects continuous
6
+ adjustment by SYMBOL SUFFIX, not the stock_price_adjustment_setting kwarg. The
7
+ BASE symbol '&ES' is UNADJUSTED (shows real calendar-spread gaps at each roll,
8
+ e.g. +146 pts at the 2026-06 Jun→Sep roll). '&ES_CCB' is BACK-ADJUSTED (gaps
9
+ stitched out). A close-based stop needs the gap-free series → we fetch _CCB.
10
+ """
11
+ import pandas as pd
12
+
13
+ from ..registry import REGISTRY, all_symbols
14
+ from .. import store
15
+
16
+ CCB_SUFFIX = "_CCB" # Norgate "Continuous Contract Back-adjusted"
17
+ # If roll-day overnight moves exceed this multiple of the normal-day median, the
18
+ # series looks UNADJUSTED (calendar-spread gaps not stitched). Self-calibrating
19
+ # per symbol, so it works across products with different spread magnitudes.
20
+ ROLL_GAP_RATIO_WARN = 1.5
21
+
22
+ _COLMAP = {
23
+ "Open": "Open", "High": "High", "Low": "Low", "Close": "Close",
24
+ "Volume": "Volume", "Open Interest": "Open Interest",
25
+ "Delivery Month": "Delivery Month", # kept → exact roll detection downstream
26
+ }
27
+
28
+
29
+ def fetch(internal_symbol: str, start: str = "1970-01-01") -> pd.DataFrame:
30
+ """Fetch Norgate back-adjusted continuous bars: settlement close, gap-free."""
31
+ import norgatedata # imported lazily; only present on the Windows producer
32
+ ng_sym = REGISTRY[internal_symbol].norgate + CCB_SUFFIX # "&ES" → "&ES_CCB"
33
+ df = norgatedata.price_timeseries(
34
+ ng_sym,
35
+ padding_setting=norgatedata.PaddingType.NONE,
36
+ timeseriesformat="pandas-dataframe",
37
+ start_date=start,
38
+ )
39
+ df = df.rename(columns=_COLMAP)
40
+ keep = [c for c in _COLMAP.values() if c in df.columns]
41
+ out = df[keep].copy()
42
+ out.index = pd.to_datetime(out.index).tz_localize(None).normalize()
43
+ out.index.name = "Date"
44
+ return out.sort_index()
45
+
46
+
47
+ def _check_roll_gaps(internal_symbol: str, df: pd.DataFrame) -> bool:
48
+ """Warn if roll-day overnight moves ≫ normal-day moves — the signature of an
49
+ UNADJUSTED continuous. Returns True if it looks unadjusted. Self-calibrating:
50
+ compares each symbol's roll-day moves to its own non-roll baseline."""
51
+ if "Delivery Month" not in df.columns or len(df) < 60:
52
+ return False
53
+ dm = df["Delivery Month"]
54
+ roll = dm.ne(dm.shift()) & dm.shift().notna()
55
+ if int(roll.sum()) < 8:
56
+ return False
57
+ overnight = (df["Close"] - df["Close"].shift(1)).abs()
58
+ roll_med = overnight[roll].median()
59
+ nonroll_med = overnight[~roll].median()
60
+ if nonroll_med and roll_med > ROLL_GAP_RATIO_WARN * nonroll_med:
61
+ print(f" ⚠️ {internal_symbol}: roll-day moves {roll_med:.1f} vs normal {nonroll_med:.1f} "
62
+ f"({roll_med / nonroll_med:.1f}x) — series looks UNADJUSTED. Expected the _CCB "
63
+ f"back-adjusted symbol; a close-based stop would false-trigger on roll gaps.")
64
+ return True
65
+ return False
66
+
67
+
68
+ def update(symbols=None) -> None:
69
+ """Fetch + write to the store for the given internal symbols (backadj only)."""
70
+ syms = symbols or [s.internal for s in all_symbols()]
71
+ for sym in syms:
72
+ try:
73
+ out = fetch(sym)
74
+ _check_roll_gaps(sym, out) # sanity: warn (don't block) if it looks unadjusted
75
+ store.write_prices(sym, "backadj", out, source="norgate")
76
+ print(f"{sym:5s}: {len(out):6d} bars -> store")
77
+ except Exception as e: # noqa: BLE001
78
+ print(f"{sym:5s}: FAILED — {e}")
cotdata/registry.py ADDED
@@ -0,0 +1,127 @@
1
+ """The single symbol registry: internal ↔ Norgate ↔ CFTC code ↔ asset class.
2
+ Replaces the scattered maps (CotSymbolCodeMap, the databento_mapping).
3
+
4
+ Scope = FIXED IDENTITY facts only (never change). TUNABLE strategy parameters
5
+ (positioning-index CustomLookbackWeeks, thresholds, TV chart symbols) stay in
6
+ cot-analyzer/config/params.yaml — the data layer must not carry strategy knobs.
7
+
8
+ The table lives in registry.yaml (asset_class -> symbol -> attrs), loaded at
9
+ import. Point $COTDATA_REGISTRY at an alternate file to override it.
10
+
11
+ Sources:
12
+ • cftc_code — cot-analyzer CotSymbolCodeMap
13
+ • asset_class — CotIndexer asset classes
14
+ • norgate — '&' + CME root (e.g., "&ES"); required by norgatedata.price_timeseries().
15
+ is_equity is True only for the four equity indices (a fixed classification the
16
+ equity-vs-commodity rules key off); it is derived from asset_class == "Equities"
17
+ unless a symbol overrides it explicitly.
18
+ """
19
+ import os
20
+ import yaml
21
+ from dataclasses import dataclass
22
+ from pathlib import Path
23
+ from typing import Dict, List, Optional
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class Symbol:
28
+ internal: str # pipeline root, e.g. "ES"
29
+ norgate: str # Norgate continuous symbol, e.g. "&ES"
30
+ asset_class: str
31
+ is_equity: bool
32
+ report_type: str = "disagg" # "tff" for financials, "disagg" for commodities
33
+ cftc_code: Optional[str] = None
34
+ # Predecessor CFTC codes from earlier exchange/contract listings of the SAME
35
+ # instrument, stitched in chronologically behind cftc_code by get_cot. Each
36
+ # entry is a bare code string (scale 1.0) or a (code, scale) tuple. Kept as a
37
+ # tuple of hashable entries so Symbol (frozen) stays hashable.
38
+ hist_codes: tuple = ()
39
+
40
+
41
+ def hist_code_scales(hist_codes) -> List[tuple]:
42
+ """Normalize hist_codes entries (str or (code, scale)) → list of (code, scale)."""
43
+ out = []
44
+ for h in hist_codes:
45
+ out.append((h[0], float(h[1])) if isinstance(h, (list, tuple)) else (h, 1.0))
46
+ return out
47
+
48
+
49
+ def _coerce_hist_codes(raw) -> tuple:
50
+ """YAML encodes scaled entries as lists ([code, scale]); store them as tuples
51
+ so Symbol stays hashable (a list nested in the tuple would break hashing)."""
52
+ return tuple(tuple(h) if isinstance(h, list) else h for h in (raw or []))
53
+
54
+
55
+ def load_registry(yaml_path=None) -> Dict[str, Symbol]:
56
+ """Build the symbol registry from YAML.
57
+
58
+ Path resolution: explicit ``yaml_path`` arg, else $COTDATA_REGISTRY, else the
59
+ packaged registry.yaml beside this module. Raises with a clear message on a
60
+ missing/malformed file or an invalid entry, since the whole package depends
61
+ on this at import.
62
+ """
63
+ yaml_path = yaml_path or os.environ.get(
64
+ "COTDATA_REGISTRY", Path(__file__).parent / "registry.yaml")
65
+ try:
66
+ with open(yaml_path, "r") as f:
67
+ data = yaml.safe_load(f)
68
+ except FileNotFoundError as e:
69
+ raise FileNotFoundError(
70
+ f"cotdata registry file not found: {yaml_path}. Set $COTDATA_REGISTRY "
71
+ f"to a valid registry YAML, or restore the packaged registry.yaml.") from e
72
+ except yaml.YAMLError as e:
73
+ raise ValueError(f"cotdata registry YAML is malformed ({yaml_path}): {e}") from e
74
+
75
+ if not isinstance(data, dict):
76
+ raise ValueError(
77
+ f"cotdata registry YAML must be a mapping of asset_class -> symbols "
78
+ f"({yaml_path}); got {type(data).__name__}.")
79
+
80
+ registry: Dict[str, Symbol] = {}
81
+ for asset_class, symbols in data.items():
82
+ if not isinstance(symbols, dict):
83
+ raise ValueError(
84
+ f"cotdata registry: asset class '{asset_class}' must map to a dict of "
85
+ f"symbols, got {type(symbols).__name__}.")
86
+ for internal, attrs in symbols.items():
87
+ attrs = attrs or {}
88
+ if not isinstance(attrs, dict):
89
+ raise ValueError(
90
+ f"cotdata registry: symbol '{internal}' must map to a dict of "
91
+ f"attrs (e.g. 'cftc_code: ...'), got {type(attrs).__name__}.")
92
+ if internal in registry:
93
+ raise ValueError(
94
+ f"cotdata registry: duplicate symbol '{internal}' "
95
+ f"(re-declared under asset class '{asset_class}').")
96
+ if not attrs.get("cftc_code"):
97
+ raise ValueError(
98
+ f"cotdata registry: symbol '{internal}' is missing cftc_code.")
99
+ registry[internal] = Symbol(
100
+ internal=internal,
101
+ norgate=attrs.get("norgate", f"&{internal}"),
102
+ asset_class=asset_class,
103
+ # Derived from the class so the two can't drift; an explicit
104
+ # is_equity in the YAML still wins if a symbol ever needs it.
105
+ is_equity=attrs.get("is_equity", asset_class == "Equities"),
106
+ # Financials use TFF (Traders in Financial Futures), Commodities use Disaggregated
107
+ report_type=attrs.get("report_type", "tff" if asset_class in ("Equities", "FX", "Rates") else "disagg"),
108
+ cftc_code=attrs["cftc_code"],
109
+ hist_codes=_coerce_hist_codes(attrs.get("hist_codes")),
110
+ )
111
+ return registry
112
+
113
+
114
+ # Built once at import from the default location.
115
+ REGISTRY: Dict[str, Symbol] = load_registry()
116
+
117
+
118
+ def symbol(internal: str) -> Symbol:
119
+ return REGISTRY[internal]
120
+
121
+
122
+ def all_symbols() -> List[Symbol]:
123
+ return list(REGISTRY.values())
124
+
125
+
126
+ def by_asset_class(asset_class: str) -> List[Symbol]:
127
+ return [s for s in REGISTRY.values() if s.asset_class == asset_class]
cotdata/registry.yaml ADDED
@@ -0,0 +1,105 @@
1
+ Equities:
2
+ ES:
3
+ cftc_code: "13874A"
4
+ NQ:
5
+ cftc_code: "209742"
6
+ YM:
7
+ cftc_code: "124603"
8
+ RTY:
9
+ cftc_code: "239742"
10
+ hist_codes:
11
+ - "23977A"
12
+
13
+ Metals:
14
+ GC:
15
+ cftc_code: "088691"
16
+ SI:
17
+ cftc_code: "084691"
18
+ HG:
19
+ cftc_code: "085692"
20
+ PL:
21
+ cftc_code: "076651"
22
+ PA:
23
+ cftc_code: "075651"
24
+
25
+ Energies:
26
+ CL:
27
+ cftc_code: "067651"
28
+ RB:
29
+ cftc_code: "111659"
30
+ HO:
31
+ cftc_code: "022651"
32
+ NG:
33
+ cftc_code: "023651"
34
+
35
+ Grains:
36
+ ZC:
37
+ cftc_code: "002602"
38
+ ZS:
39
+ cftc_code: "005602"
40
+ ZM:
41
+ cftc_code: "026603"
42
+ ZL:
43
+ cftc_code: "007601"
44
+ ZW:
45
+ cftc_code: "001602"
46
+
47
+ Currencies:
48
+ 6E:
49
+ cftc_code: "099741"
50
+ 6A:
51
+ cftc_code: "232741"
52
+ 6B:
53
+ cftc_code: "096742"
54
+ 6C:
55
+ cftc_code: "090741"
56
+ 6J:
57
+ cftc_code: "097741"
58
+ 6S:
59
+ cftc_code: "092741"
60
+ 6M:
61
+ cftc_code: "095741"
62
+ 6N:
63
+ cftc_code: "112741"
64
+ DX:
65
+ cftc_code: "098662"
66
+
67
+ Fixed Income:
68
+ ZN:
69
+ cftc_code: "043602"
70
+ ZT:
71
+ cftc_code: "042601"
72
+ ZF:
73
+ cftc_code: "044601"
74
+ ZB:
75
+ cftc_code: "020601"
76
+
77
+ Softs:
78
+ SB:
79
+ cftc_code: "080732"
80
+ CT:
81
+ cftc_code: "033661"
82
+ CC:
83
+ cftc_code: "073732"
84
+ KC:
85
+ cftc_code: "083731"
86
+ OJ:
87
+ cftc_code: "040701"
88
+ LBR:
89
+ cftc_code: "058644"
90
+ hist_codes:
91
+ - ["058643", 4.0]
92
+
93
+ Live Stock:
94
+ LE:
95
+ cftc_code: "057642"
96
+ HE:
97
+ cftc_code: "054642"
98
+ GF:
99
+ cftc_code: "061641"
100
+
101
+ Crypto:
102
+ BTC:
103
+ cftc_code: "133741"
104
+ ETH:
105
+ cftc_code: "146021"