cotdata 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cotdata/__init__.py ADDED
@@ -0,0 +1,11 @@
1
+ """cotdata — canonical data layer (see README). Public consumer API."""
2
+ from .prices import get_prices, roll_dates
3
+ from .cot import get_cot
4
+ from .registry import symbol, all_symbols, REGISTRY, Symbol
5
+ from .store import load_manifest
6
+
7
+ __version__ = "0.1.0"
8
+ __all__ = [
9
+ "get_prices", "roll_dates", "get_cot",
10
+ "symbol", "all_symbols", "REGISTRY", "Symbol", "load_manifest",
11
+ ]
cotdata/config.py ADDED
@@ -0,0 +1,35 @@
1
+ """Store location + schema version. The store is set via COTDATA_STORE."""
2
+ import os
3
+ from pathlib import Path
4
+
5
+ SCHEMA_VERSION = 1
6
+
7
+
8
+ def store_root() -> Path:
9
+ root = os.environ.get("COTDATA_STORE", "").strip()
10
+ if not root:
11
+ raise RuntimeError(
12
+ "COTDATA_STORE is not set. Point it at the shared data store "
13
+ "(the synced folder holding prices/, cot/, manifest.json)."
14
+ )
15
+ return Path(root)
16
+
17
+
18
+ def prices_dir() -> Path:
19
+ return store_root() / "prices"
20
+
21
+
22
+ def cot_legacy_dir() -> Path:
23
+ return store_root() / "cot_legacy"
24
+
25
+
26
+ def cot_disagg_dir() -> Path:
27
+ return store_root() / "cot_disagg"
28
+
29
+
30
+ def cot_tff_dir() -> Path:
31
+ return store_root() / "cot_tff"
32
+
33
+
34
+ def manifest_path() -> Path:
35
+ return store_root() / "manifest.json"
cotdata/cot.py ADDED
@@ -0,0 +1,62 @@
1
+ """Consumer COT API. Reads weekly CFTC positioning the producer wrote.
2
+
3
+ NOTE: the positioning-INDEX computation and the trading signals stay in
4
+ cot-analyzer (CotIndexer / metrics) — cotdata owns only the raw data. This
5
+ returns the cleaned weekly COT table; the indexer builds indices on top.
6
+ """
7
+ import pandas as pd
8
+
9
+ from . import store
10
+ from .registry import REGISTRY, hist_code_scales
11
+
12
+ _CODE_COL = "CFTC_Contract_Market_Code" # Legacy schema contract-code column
13
+
14
+
15
+ def get_cot(name: str, report: str = "legacy") -> pd.DataFrame:
16
+ """Read a symbol's weekly COT history with predecessor code stitching.
17
+
18
+ name: the internal pipeline symbol, e.g. 'ES' or 'GC'.
19
+ report: 'legacy' (default), 'disagg' (commodity futures only), or 'tff' (financial futures only).
20
+
21
+ Returns:
22
+ DataFrame indexed by date. The columns depend on the `report` requested:
23
+ - legacy: Open_Interest_All, NonComm_Positions_Long_All, Comm_Positions_Long_All, etc.
24
+ - disagg: Open_Interest_All, Prod_Merc_Positions_Long_All, M_Money_Positions_Long_All, etc.
25
+ - tff: Open_Interest_All, Dealer_Positions_Long_All, Lev_Money_Positions_Long_All, etc.
26
+ If no data exists, returns an empty DataFrame.
27
+ """
28
+ sym = REGISTRY.get(name)
29
+ if sym is None: # allow lookup by primary CFTC code, not just symbol
30
+ sym = next((s for s in REGISTRY.values() if s.cftc_code == name), None)
31
+
32
+ read_fn = {
33
+ "legacy": store.read_cot_legacy,
34
+ "disagg": store.read_cot_disagg,
35
+ "tff": store.read_cot_tff,
36
+ }.get(report)
37
+ if not read_fn:
38
+ raise ValueError(f"Unknown report type: {report}. Expected 'legacy', 'disagg', or 'tff'")
39
+
40
+ if sym is None or not sym.cftc_code:
41
+ return read_fn(name)
42
+ primary = read_fn(sym.cftc_code)
43
+ if not sym.hist_codes:
44
+ return primary
45
+ frames = [primary] # primary first → wins de-duplication on overlaps
46
+ for hc, scale in hist_code_scales(sym.hist_codes):
47
+ h = read_fn(hc)
48
+ if h.empty:
49
+ continue
50
+ h = h.copy()
51
+ if scale != 1.0: # bridge a contract-size change (position/OI counts)
52
+ num = h.select_dtypes("number").columns
53
+ h[num] = h[num] * scale
54
+ if _CODE_COL in h.columns: # present predecessor rows under the primary code
55
+ h[_CODE_COL] = sym.cftc_code
56
+ frames.append(h)
57
+ frames = [f for f in frames if not f.empty]
58
+ if not frames:
59
+ return primary
60
+ combined = pd.concat(frames)
61
+ combined = combined[~combined.index.duplicated(keep="first")].sort_index()
62
+ return combined
cotdata/prices.py ADDED
@@ -0,0 +1,43 @@
1
+ """Consumer price API. Reads the store and returns the shape downstream code
2
+ expects (the old fetch_daily_ohlc contract). No network, cross-platform."""
3
+ from typing import Optional
4
+
5
+ import pandas as pd
6
+
7
+ from . import store
8
+
9
+ _COLS = ["Open", "High", "Low", "Close", "Volume", "Open Interest"]
10
+
11
+
12
+ def get_prices(symbol: str, adjustment: str = "backadj",
13
+ start: Optional[str] = None) -> pd.DataFrame:
14
+ """Daily bars for `symbol`.
15
+
16
+ adjustment: 'backadj' (signals + stops — settlement close, gap-free rolls,
17
+ shape-preserving) or 'unadj' (absolute price / point-value sizing).
18
+ Returns Open/High/Low/Close/Volume/Open Interest indexed by tz-naive Date
19
+ (plus 'Delivery Month' if the producer carried it), or empty if absent.
20
+ """
21
+ df = store.read_prices(symbol, adjustment)
22
+ if df.empty:
23
+ return df
24
+ df.index = pd.to_datetime(df.index).tz_localize(None).normalize()
25
+ df.index.name = "Date"
26
+ df = df.sort_index()
27
+ if start:
28
+ df = df[df.index >= pd.Timestamp(start)]
29
+ for c in _COLS:
30
+ if c not in df.columns:
31
+ df[c] = float("nan")
32
+ keep = _COLS + (["Delivery Month"] if "Delivery Month" in df.columns else [])
33
+ return df[keep].copy()
34
+
35
+
36
+ def roll_dates(symbol: str, adjustment: str = "backadj") -> pd.DatetimeIndex:
37
+ """Exact roll dates = days where the underlying Delivery Month changes.
38
+ Empty if the producer didn't carry 'Delivery Month'."""
39
+ df = get_prices(symbol, adjustment)
40
+ if df.empty or "Delivery Month" not in df.columns:
41
+ return pd.DatetimeIndex([])
42
+ changed = df["Delivery Month"].ne(df["Delivery Month"].shift())
43
+ return df.index[changed.fillna(False)]
@@ -0,0 +1,2 @@
1
+ """Producers: fetch from a vendor and write the canonical store. Run on the
2
+ machine that can reach the source (Norgate → Windows; CFTC → anywhere)."""
@@ -0,0 +1,14 @@
1
+ """A thin seam for price producers — enough to swap sources, not a plugin
2
+ framework (single active source = Norgate)."""
3
+ from typing import Protocol
4
+
5
+ import pandas as pd
6
+
7
+
8
+ class PriceProvider(Protocol):
9
+ name: str
10
+
11
+ def fetch(self, internal_symbol: str, adjustment: str) -> pd.DataFrame:
12
+ """Return daily bars (Open/High/Low/Close/Volume/Open Interest, tz-naive
13
+ Date index; optionally 'Delivery Month') for one symbol/adjustment."""
14
+ ...
@@ -0,0 +1,129 @@
1
+ """CFTC COT producer — cross-platform (just HTTPS zips from cftc.gov).
2
+
3
+ Downloads dea_fut_xls_{year}.zip (Legacy futures report), parses, and writes a
4
+ per-code weekly positioning table to the store via store.write_cot(). The
5
+ positioning-INDEX + trading signals stay in cot-analyzer (CotIndexer/metrics);
6
+ cotdata owns only the raw weekly data.
7
+
8
+ Self-contained — no cot-analyzer imports. Ported from cot-analyzer/src/core/etl.py.
9
+ """
10
+ import datetime as dt
11
+ import io
12
+ import zipfile
13
+ from email.utils import parsedate_to_datetime
14
+ from pathlib import Path
15
+
16
+ import pandas as pd
17
+ import requests
18
+
19
+ from .. import config, store
20
+ from ..registry import all_symbols, hist_code_scales
21
+
22
+ URL_PREFIX = "https://www.cftc.gov/files/dea/history/dea_fut_xls_"
23
+ FIRST_YEAR = 1986 # CFTC Legacy futures history start
24
+
25
+ # CFTC XLS column names (Legacy futures report)
26
+ MARKET_NAME = "Market_and_Exchange_Names"
27
+ REPORT_DATE = "Report_Date_as_MM_DD_YYYY"
28
+ CONTRACT_CODE = "CFTC_Contract_Market_Code"
29
+ OPEN_INTEREST = "Open_Interest_All"
30
+ COMM_LONG = "Comm_Positions_Long_All"
31
+ COMM_SHORT = "Comm_Positions_Short_All"
32
+ NONCOMM_LONG = "NonComm_Positions_Long_All"
33
+ NONCOMM_SHORT = "NonComm_Positions_Short_All"
34
+ NONREPT_LONG = "NonRept_Positions_Long_All"
35
+ NONREPT_SHORT = "NonRept_Positions_Short_All"
36
+
37
+ TARGET_COLS = [MARKET_NAME, REPORT_DATE, CONTRACT_CODE, OPEN_INTEREST,
38
+ COMM_LONG, COMM_SHORT, NONCOMM_LONG, NONCOMM_SHORT,
39
+ NONREPT_LONG, NONREPT_SHORT]
40
+
41
+
42
+ def _cache_dir() -> Path:
43
+ d = config.store_root() / "_cache" / "cot_legacy"
44
+ d.mkdir(parents=True, exist_ok=True)
45
+ return d
46
+
47
+
48
+ def _standardize_code(val) -> str:
49
+ """CFTC contract codes → 6-digit zero-padded string (matches CotSymbolCodeMap)."""
50
+ s = str(val).strip()
51
+ return s.zfill(6) if s.isdigit() else s
52
+
53
+
54
+ def _download_year(year: int) -> Path | None:
55
+ # CFTC changed their naming convention in 2004.
56
+ # 1986 - 2003: deafut_xls_{year}.zip
57
+ # 2004 - Present: dea_fut_xls_{year}.zip
58
+ if year < 2004:
59
+ url = f"https://www.cftc.gov/files/dea/history/deafut_xls_{year}.zip"
60
+ filename = f"deafut_xls_{year}.zip"
61
+ else:
62
+ url = f"{URL_PREFIX}{year}.zip"
63
+ filename = f"dea_fut_xls_{year}.zip"
64
+
65
+ zip_path = _cache_dir() / filename
66
+ try:
67
+ if zip_path.exists():
68
+ head = requests.head(url, timeout=30)
69
+ server_mtime = head.headers.get("Last-Modified")
70
+ if server_mtime and zip_path.stat().st_mtime >= parsedate_to_datetime(server_mtime).timestamp():
71
+ return zip_path # up to date
72
+ r = requests.get(url, timeout=180)
73
+ r.raise_for_status()
74
+ zip_path.write_bytes(r.content)
75
+ return zip_path
76
+ except Exception as e: # noqa: BLE001
77
+ print(f" {year}: download failed — {e}")
78
+ return zip_path if zip_path.exists() else None
79
+
80
+
81
+ def _parse_zip(zip_path: Path) -> pd.DataFrame:
82
+ """Extract the .xls from a year zip → cleaned target columns."""
83
+ with zipfile.ZipFile(zip_path) as zf:
84
+ with zf.open(zf.namelist()[0]) as fh:
85
+ data = fh.read()
86
+ df = pd.read_excel(io.BytesIO(data), usecols=TARGET_COLS) # .xls → needs xlrd
87
+ df[CONTRACT_CODE] = df[CONTRACT_CODE].apply(_standardize_code)
88
+ df[REPORT_DATE] = pd.to_datetime(df[REPORT_DATE]).dt.tz_localize(None)
89
+ return df
90
+
91
+
92
+ def update(codes=None, first_year: int = FIRST_YEAR, last_year=None) -> None:
93
+ """Download + parse CFTC Legacy futures COT; write full per-code history.
94
+
95
+ codes: iterable of CFTC codes; default = all registry codes.
96
+ Rebuilds the complete per-code table each run (parse is cheap; downloads are
97
+ cached and skip when unchanged). Incremental append is a future optimization.
98
+ """
99
+ last_year = last_year or dt.date.today().year
100
+ if codes:
101
+ want = set(codes)
102
+ else:
103
+ want = {s.cftc_code for s in all_symbols() if s.cftc_code}
104
+ for s in all_symbols(): # predecessor codes (migrated-contract history)
105
+ want.update(code for code, _ in hist_code_scales(s.hist_codes))
106
+
107
+ frames = []
108
+ for year in range(first_year, last_year + 1):
109
+ zp = _download_year(year)
110
+ if zp is None:
111
+ continue
112
+ try:
113
+ frames.append(_parse_zip(zp))
114
+ except Exception as e: # noqa: BLE001
115
+ print(f" {year}: parse failed — {e}")
116
+
117
+ if not frames:
118
+ print("cftc: no data parsed")
119
+ return
120
+
121
+ allrows = pd.concat(frames, ignore_index=True)
122
+ for code in sorted(want):
123
+ sub = allrows[allrows[CONTRACT_CODE] == code].copy()
124
+ if sub.empty:
125
+ print(f"{code}: no rows")
126
+ continue
127
+ sub = sub.sort_values(REPORT_DATE).set_index(REPORT_DATE)
128
+ store.write_cot_legacy(code, sub, source="cftc")
129
+ print(f"{code}: {len(sub):5d} weeks (legacy) -> store")
@@ -0,0 +1,146 @@
1
+ """CFTC COT Disaggregated Futures-Only producer — cross-platform.
2
+
3
+ Downloads fut_disagg_txt_{year}.zip, parses losslessly (preserving all Traders_*
4
+ and detailed entity columns), and writes per-code weekly positioning tables to
5
+ the store via store.write_cot_disagg().
6
+ """
7
+ import datetime as dt
8
+ import io
9
+ import zipfile
10
+ from email.utils import parsedate_to_datetime
11
+ from pathlib import Path
12
+
13
+ import pandas as pd
14
+ import requests
15
+
16
+ from .. import config, store
17
+ from ..registry import all_symbols, hist_code_scales
18
+
19
+ URL_PREFIX = "https://www.cftc.gov/files/dea/history/fut_disagg_txt_"
20
+ FIRST_YEAR = 2006 # Disaggregated futures history start
21
+
22
+ # CFTC TXT column names we must coerce/standardize (others pass through losslessly)
23
+ REPORT_DATE = "Report_Date_as_MM_DD_YYYY"
24
+ CONTRACT_CODE = "CFTC_Contract_Market_Code"
25
+
26
+
27
+ def _cache_dir() -> Path:
28
+ d = config.store_root() / "_cache" / "cot_disagg"
29
+ d.mkdir(parents=True, exist_ok=True)
30
+ return d
31
+
32
+
33
+ def _standardize_code(val) -> str:
34
+ """CFTC contract codes → 6-digit zero-padded string (matches CotSymbolCodeMap)."""
35
+ s = str(val).strip()
36
+ return s.zfill(6) if s.isdigit() else s
37
+
38
+
39
+ def _download_url(url: str, filename: str):
40
+ """Download a zip to the cache; skip if the server copy isn't newer."""
41
+ zip_path = _cache_dir() / filename
42
+ try:
43
+ if zip_path.exists():
44
+ head = requests.head(url, timeout=30)
45
+ server_mtime = head.headers.get("Last-Modified")
46
+ if server_mtime and zip_path.stat().st_mtime >= parsedate_to_datetime(server_mtime).timestamp():
47
+ return zip_path # up to date
48
+ r = requests.get(url, timeout=180)
49
+ r.raise_for_status()
50
+ zip_path.write_bytes(r.content)
51
+ return zip_path
52
+ except Exception as e: # noqa: BLE001
53
+ print(f" {filename} (disagg): download failed — {e}")
54
+ return zip_path if zip_path.exists() else None
55
+
56
+
57
+ def _parse_zip(zip_path: Path) -> pd.DataFrame:
58
+ """Extract the .txt (CSV) from a year zip → full lossless DataFrame."""
59
+ with zipfile.ZipFile(zip_path) as zf:
60
+ with zf.open(zf.namelist()[0]) as fh:
61
+ # The .txt files are actually CSVs. low_memory=False prevents dtype warnings.
62
+ df = pd.read_csv(fh, low_memory=False)
63
+
64
+ # Strip any trailing whitespace from column names BEFORE accessing them
65
+ df.columns = df.columns.str.strip()
66
+
67
+ # CFTC sometimes changes the date column name in Disaggregated reports
68
+ if REPORT_DATE not in df.columns:
69
+ date_cols = [c for c in df.columns if "Report_Date" in c]
70
+ if date_cols:
71
+ df.rename(columns={date_cols[0]: REPORT_DATE}, inplace=True)
72
+ else:
73
+ print(f"AVAILABLE COLUMNS: {list(df.columns)}")
74
+
75
+ # Coerce the key schema columns to match the Legacy schema format
76
+ df[CONTRACT_CODE] = df[CONTRACT_CODE].apply(_standardize_code)
77
+ df[REPORT_DATE] = pd.to_datetime(df[REPORT_DATE]).dt.tz_localize(None)
78
+
79
+ # Parquet cannot serialize mixed-type object columns (e.g. Traders_Tot_Old contains ints and strings)
80
+ for col in df.select_dtypes(include=['object']).columns:
81
+ if col not in [CONTRACT_CODE, REPORT_DATE]:
82
+ df[col] = df[col].astype(str)
83
+
84
+ return df
85
+
86
+
87
+ def update(codes=None, first_year: int = FIRST_YEAR, last_year=None) -> None:
88
+ """Download + parse CFTC Disaggregated futures COT; write full per-code history.
89
+
90
+ codes: iterable of CFTC codes; default = all registry codes.
91
+ Rebuilds the complete per-code table each run.
92
+ """
93
+ last_year = last_year or dt.date.today().year
94
+ if codes:
95
+ want = set(codes)
96
+ else:
97
+ want = {s.cftc_code for s in all_symbols() if s.cftc_code}
98
+ for s in all_symbols(): # predecessor codes (migrated-contract history)
99
+ want.update(code for code, _ in hist_code_scales(s.hist_codes))
100
+
101
+ frames = []
102
+
103
+ # The CFTC bundles 2006-2016 in a single historical file
104
+ if first_year <= 2016:
105
+ url = "https://www.cftc.gov/files/dea/history/fut_disagg_txt_hist_2006_2016.zip"
106
+ zp = _download_url(url, "fut_disagg_txt_hist_2006_2016.zip")
107
+ if zp:
108
+ try:
109
+ df = _parse_zip(zp)
110
+ # Filter down to the requested year range so we don't accidentally pull earlier
111
+ # than first_year if the user explicitly wanted a shorter window.
112
+ df = df[(df[REPORT_DATE].dt.year >= first_year) & (df[REPORT_DATE].dt.year <= 2016)]
113
+ frames.append(df)
114
+ except Exception as e:
115
+ print(f" hist_2006_2016 (disagg): parse failed — {e}")
116
+
117
+ # Fetch individual years for 2017+ (or first_year if it's > 2016)
118
+ for year in range(max(2017, first_year), last_year + 1):
119
+ zp = _download_url(f"{URL_PREFIX}{year}.zip", f"fut_disagg_txt_{year}.zip")
120
+ if not zp:
121
+ continue
122
+ try:
123
+ frames.append(_parse_zip(zp))
124
+ except Exception as e: # noqa: BLE001
125
+ print(f" {year} (disagg): parse failed — {e}")
126
+
127
+ if not frames:
128
+ print("cftc_disagg: no data parsed")
129
+ return
130
+
131
+ allrows = pd.concat(frames, ignore_index=True)
132
+
133
+ # Parquet cannot serialize mixed-type object columns after concat (due to NaNs)
134
+ for col in allrows.select_dtypes(include=['object']).columns:
135
+ if col not in [CONTRACT_CODE, REPORT_DATE]:
136
+ allrows[col] = allrows[col].astype(str)
137
+
138
+ for code in sorted(want):
139
+ sub = allrows[allrows[CONTRACT_CODE] == code].copy()
140
+ if sub.empty:
141
+ continue
142
+
143
+ # Index by report date (DatetimeIndex → manifest last_date)
144
+ sub = sub.sort_values(REPORT_DATE).set_index(REPORT_DATE)
145
+ store.write_cot_disagg(code, sub, source="cftc_disagg")
146
+ print(f"{code}: {len(sub):5d} weeks (disagg) -> store")
@@ -0,0 +1,146 @@
1
+ """CFTC COT Traders in Financial Futures (TFF) producer — cross-platform.
2
+
3
+ Downloads fut_fin_txt_{year}.zip, parses losslessly (preserving all Traders_*
4
+ and detailed entity columns), and writes per-code weekly positioning tables to
5
+ the store via store.write_cot_tff().
6
+ """
7
+ import datetime as dt
8
+ import io
9
+ import zipfile
10
+ from email.utils import parsedate_to_datetime
11
+ from pathlib import Path
12
+
13
+ import pandas as pd
14
+ import requests
15
+
16
+ from .. import config, store
17
+ from ..registry import all_symbols, hist_code_scales
18
+
19
+ URL_PREFIX = "https://www.cftc.gov/files/dea/history/fut_fin_txt_"
20
+ FIRST_YEAR = 2006 # TFF futures history start
21
+
22
+ # CFTC TXT column names we must coerce/standardize (others pass through losslessly)
23
+ REPORT_DATE = "Report_Date_as_MM_DD_YYYY"
24
+ CONTRACT_CODE = "CFTC_Contract_Market_Code"
25
+
26
+
27
+ def _cache_dir() -> Path:
28
+ d = config.store_root() / "_cache" / "cot_tff"
29
+ d.mkdir(parents=True, exist_ok=True)
30
+ return d
31
+
32
+
33
+ def _standardize_code(val) -> str:
34
+ """CFTC contract codes → 6-digit zero-padded string (matches CotSymbolCodeMap)."""
35
+ s = str(val).strip()
36
+ return s.zfill(6) if s.isdigit() else s
37
+
38
+
39
+ def _download_url(url: str, filename: str):
40
+ """Download a zip to the cache; skip if the server copy isn't newer."""
41
+ zip_path = _cache_dir() / filename
42
+ try:
43
+ if zip_path.exists():
44
+ head = requests.head(url, timeout=30)
45
+ server_mtime = head.headers.get("Last-Modified")
46
+ if server_mtime and zip_path.stat().st_mtime >= parsedate_to_datetime(server_mtime).timestamp():
47
+ return zip_path # up to date
48
+ r = requests.get(url, timeout=180)
49
+ r.raise_for_status()
50
+ zip_path.write_bytes(r.content)
51
+ return zip_path
52
+ except Exception as e: # noqa: BLE001
53
+ print(f" {filename} (tff): download failed — {e}")
54
+ return zip_path if zip_path.exists() else None
55
+
56
+
57
+ def _parse_zip(zip_path: Path) -> pd.DataFrame:
58
+ """Extract the .txt (CSV) from a year zip → full lossless DataFrame."""
59
+ with zipfile.ZipFile(zip_path) as zf:
60
+ with zf.open(zf.namelist()[0]) as fh:
61
+ # The .txt files are actually CSVs. low_memory=False prevents dtype warnings.
62
+ df = pd.read_csv(fh, low_memory=False)
63
+
64
+ # Strip any trailing whitespace from column names BEFORE accessing them
65
+ df.columns = df.columns.str.strip()
66
+
67
+ # CFTC sometimes changes the date column name in TFF reports
68
+ if REPORT_DATE not in df.columns:
69
+ date_cols = [c for c in df.columns if "Report_Date" in c]
70
+ if date_cols:
71
+ df.rename(columns={date_cols[0]: REPORT_DATE}, inplace=True)
72
+ else:
73
+ print(f"AVAILABLE COLUMNS: {list(df.columns)}")
74
+
75
+ # Coerce the key schema columns to match the Legacy schema format
76
+ df[CONTRACT_CODE] = df[CONTRACT_CODE].apply(_standardize_code)
77
+ df[REPORT_DATE] = pd.to_datetime(df[REPORT_DATE]).dt.tz_localize(None)
78
+
79
+ # Parquet cannot serialize mixed-type object columns (e.g. Traders_Tot_Old contains ints and strings)
80
+ for col in df.select_dtypes(include=['object']).columns:
81
+ if col not in [CONTRACT_CODE, REPORT_DATE]:
82
+ df[col] = df[col].astype(str)
83
+
84
+ return df
85
+
86
+
87
+ def update(codes=None, first_year: int = FIRST_YEAR, last_year=None) -> None:
88
+ """Download + parse CFTC TFF futures COT; write full per-code history.
89
+
90
+ codes: iterable of CFTC codes; default = all registry codes.
91
+ Rebuilds the complete per-code table each run.
92
+ """
93
+ last_year = last_year or dt.date.today().year
94
+ if codes:
95
+ want = set(codes)
96
+ else:
97
+ want = {s.cftc_code for s in all_symbols() if s.cftc_code}
98
+ for s in all_symbols(): # predecessor codes (migrated-contract history)
99
+ want.update(code for code, _ in hist_code_scales(s.hist_codes))
100
+
101
+ frames = []
102
+
103
+ # The CFTC bundles 2006-2016 in a single historical file
104
+ if first_year <= 2016:
105
+ url = "https://www.cftc.gov/files/dea/history/fin_fut_txt_2006_2016.zip"
106
+ zp = _download_url(url, "fin_fut_txt_2006_2016.zip")
107
+ if zp:
108
+ try:
109
+ df = _parse_zip(zp)
110
+ # Filter down to the requested year range so we don't accidentally pull earlier
111
+ # than first_year if the user explicitly wanted a shorter window.
112
+ df = df[(df[REPORT_DATE].dt.year >= first_year) & (df[REPORT_DATE].dt.year <= 2016)]
113
+ frames.append(df)
114
+ except Exception as e:
115
+ print(f" hist_2006_2016 (tff): parse failed — {e}")
116
+
117
+ # Fetch individual years for 2017+ (or first_year if it's > 2016)
118
+ for year in range(max(2017, first_year), last_year + 1):
119
+ zp = _download_url(f"{URL_PREFIX}{year}.zip", f"fut_fin_txt_{year}.zip")
120
+ if not zp:
121
+ continue
122
+ try:
123
+ frames.append(_parse_zip(zp))
124
+ except Exception as e: # noqa: BLE001
125
+ print(f" {year} (tff): parse failed — {e}")
126
+
127
+ if not frames:
128
+ print("cftc_tff: no data parsed")
129
+ return
130
+
131
+ allrows = pd.concat(frames, ignore_index=True)
132
+
133
+ # Parquet cannot serialize mixed-type object columns after concat (due to NaNs)
134
+ for col in allrows.select_dtypes(include=['object']).columns:
135
+ if col not in [CONTRACT_CODE, REPORT_DATE]:
136
+ allrows[col] = allrows[col].astype(str)
137
+
138
+ for code in sorted(want):
139
+ sub = allrows[allrows[CONTRACT_CODE] == code].copy()
140
+ if sub.empty:
141
+ continue
142
+
143
+ # Index by report date (DatetimeIndex → manifest last_date)
144
+ sub = sub.sort_values(REPORT_DATE).set_index(REPORT_DATE)
145
+ store.write_cot_tff(code, sub, source="cftc_tff")
146
+ print(f"{code}: {len(sub):5d} weeks (tff) -> store")