cotdata 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cotdata/__init__.py +11 -0
- cotdata/config.py +35 -0
- cotdata/cot.py +62 -0
- cotdata/prices.py +43 -0
- cotdata/providers/__init__.py +2 -0
- cotdata/providers/base.py +14 -0
- cotdata/providers/cftc.py +129 -0
- cotdata/providers/cftc_disagg.py +146 -0
- cotdata/providers/cftc_tff.py +146 -0
- cotdata/providers/databento.py +280 -0
- cotdata/providers/norgate.py +78 -0
- cotdata/registry.py +127 -0
- cotdata/registry.yaml +105 -0
- cotdata/store.py +96 -0
- cotdata/update.py +42 -0
- cotdata-0.1.0.dist-info/METADATA +246 -0
- cotdata-0.1.0.dist-info/RECORD +21 -0
- cotdata-0.1.0.dist-info/WHEEL +5 -0
- cotdata-0.1.0.dist-info/entry_points.txt +2 -0
- cotdata-0.1.0.dist-info/licenses/LICENSE +21 -0
- cotdata-0.1.0.dist-info/top_level.txt +1 -0
cotdata/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""cotdata — canonical data layer (see README). Public consumer API."""
|
|
2
|
+
from .prices import get_prices, roll_dates
|
|
3
|
+
from .cot import get_cot
|
|
4
|
+
from .registry import symbol, all_symbols, REGISTRY, Symbol
|
|
5
|
+
from .store import load_manifest
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
__all__ = [
|
|
9
|
+
"get_prices", "roll_dates", "get_cot",
|
|
10
|
+
"symbol", "all_symbols", "REGISTRY", "Symbol", "load_manifest",
|
|
11
|
+
]
|
cotdata/config.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Store location + schema version. The store is set via COTDATA_STORE."""
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
SCHEMA_VERSION = 1
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def store_root() -> Path:
|
|
9
|
+
root = os.environ.get("COTDATA_STORE", "").strip()
|
|
10
|
+
if not root:
|
|
11
|
+
raise RuntimeError(
|
|
12
|
+
"COTDATA_STORE is not set. Point it at the shared data store "
|
|
13
|
+
"(the synced folder holding prices/, cot/, manifest.json)."
|
|
14
|
+
)
|
|
15
|
+
return Path(root)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def prices_dir() -> Path:
|
|
19
|
+
return store_root() / "prices"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def cot_legacy_dir() -> Path:
|
|
23
|
+
return store_root() / "cot_legacy"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def cot_disagg_dir() -> Path:
|
|
27
|
+
return store_root() / "cot_disagg"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def cot_tff_dir() -> Path:
|
|
31
|
+
return store_root() / "cot_tff"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def manifest_path() -> Path:
|
|
35
|
+
return store_root() / "manifest.json"
|
cotdata/cot.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Consumer COT API. Reads weekly CFTC positioning the producer wrote.
|
|
2
|
+
|
|
3
|
+
NOTE: the positioning-INDEX computation and the trading signals stay in
|
|
4
|
+
cot-analyzer (CotIndexer / metrics) — cotdata owns only the raw data. This
|
|
5
|
+
returns the cleaned weekly COT table; the indexer builds indices on top.
|
|
6
|
+
"""
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
from . import store
|
|
10
|
+
from .registry import REGISTRY, hist_code_scales
|
|
11
|
+
|
|
12
|
+
_CODE_COL = "CFTC_Contract_Market_Code" # Legacy schema contract-code column
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_cot(name: str, report: str = "legacy") -> pd.DataFrame:
|
|
16
|
+
"""Read a symbol's weekly COT history with predecessor code stitching.
|
|
17
|
+
|
|
18
|
+
name: the internal pipeline symbol, e.g. 'ES' or 'GC'.
|
|
19
|
+
report: 'legacy' (default), 'disagg' (commodity futures only), or 'tff' (financial futures only).
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
DataFrame indexed by date. The columns depend on the `report` requested:
|
|
23
|
+
- legacy: Open_Interest_All, NonComm_Positions_Long_All, Comm_Positions_Long_All, etc.
|
|
24
|
+
- disagg: Open_Interest_All, Prod_Merc_Positions_Long_All, M_Money_Positions_Long_All, etc.
|
|
25
|
+
- tff: Open_Interest_All, Dealer_Positions_Long_All, Lev_Money_Positions_Long_All, etc.
|
|
26
|
+
If no data exists, returns an empty DataFrame.
|
|
27
|
+
"""
|
|
28
|
+
sym = REGISTRY.get(name)
|
|
29
|
+
if sym is None: # allow lookup by primary CFTC code, not just symbol
|
|
30
|
+
sym = next((s for s in REGISTRY.values() if s.cftc_code == name), None)
|
|
31
|
+
|
|
32
|
+
read_fn = {
|
|
33
|
+
"legacy": store.read_cot_legacy,
|
|
34
|
+
"disagg": store.read_cot_disagg,
|
|
35
|
+
"tff": store.read_cot_tff,
|
|
36
|
+
}.get(report)
|
|
37
|
+
if not read_fn:
|
|
38
|
+
raise ValueError(f"Unknown report type: {report}. Expected 'legacy', 'disagg', or 'tff'")
|
|
39
|
+
|
|
40
|
+
if sym is None or not sym.cftc_code:
|
|
41
|
+
return read_fn(name)
|
|
42
|
+
primary = read_fn(sym.cftc_code)
|
|
43
|
+
if not sym.hist_codes:
|
|
44
|
+
return primary
|
|
45
|
+
frames = [primary] # primary first → wins de-duplication on overlaps
|
|
46
|
+
for hc, scale in hist_code_scales(sym.hist_codes):
|
|
47
|
+
h = read_fn(hc)
|
|
48
|
+
if h.empty:
|
|
49
|
+
continue
|
|
50
|
+
h = h.copy()
|
|
51
|
+
if scale != 1.0: # bridge a contract-size change (position/OI counts)
|
|
52
|
+
num = h.select_dtypes("number").columns
|
|
53
|
+
h[num] = h[num] * scale
|
|
54
|
+
if _CODE_COL in h.columns: # present predecessor rows under the primary code
|
|
55
|
+
h[_CODE_COL] = sym.cftc_code
|
|
56
|
+
frames.append(h)
|
|
57
|
+
frames = [f for f in frames if not f.empty]
|
|
58
|
+
if not frames:
|
|
59
|
+
return primary
|
|
60
|
+
combined = pd.concat(frames)
|
|
61
|
+
combined = combined[~combined.index.duplicated(keep="first")].sort_index()
|
|
62
|
+
return combined
|
cotdata/prices.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Consumer price API. Reads the store and returns the shape downstream code
|
|
2
|
+
expects (the old fetch_daily_ohlc contract). No network, cross-platform."""
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from . import store
|
|
8
|
+
|
|
9
|
+
_COLS = ["Open", "High", "Low", "Close", "Volume", "Open Interest"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def get_prices(symbol: str, adjustment: str = "backadj",
|
|
13
|
+
start: Optional[str] = None) -> pd.DataFrame:
|
|
14
|
+
"""Daily bars for `symbol`.
|
|
15
|
+
|
|
16
|
+
adjustment: 'backadj' (signals + stops — settlement close, gap-free rolls,
|
|
17
|
+
shape-preserving) or 'unadj' (absolute price / point-value sizing).
|
|
18
|
+
Returns Open/High/Low/Close/Volume/Open Interest indexed by tz-naive Date
|
|
19
|
+
(plus 'Delivery Month' if the producer carried it), or empty if absent.
|
|
20
|
+
"""
|
|
21
|
+
df = store.read_prices(symbol, adjustment)
|
|
22
|
+
if df.empty:
|
|
23
|
+
return df
|
|
24
|
+
df.index = pd.to_datetime(df.index).tz_localize(None).normalize()
|
|
25
|
+
df.index.name = "Date"
|
|
26
|
+
df = df.sort_index()
|
|
27
|
+
if start:
|
|
28
|
+
df = df[df.index >= pd.Timestamp(start)]
|
|
29
|
+
for c in _COLS:
|
|
30
|
+
if c not in df.columns:
|
|
31
|
+
df[c] = float("nan")
|
|
32
|
+
keep = _COLS + (["Delivery Month"] if "Delivery Month" in df.columns else [])
|
|
33
|
+
return df[keep].copy()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def roll_dates(symbol: str, adjustment: str = "backadj") -> pd.DatetimeIndex:
|
|
37
|
+
"""Exact roll dates = days where the underlying Delivery Month changes.
|
|
38
|
+
Empty if the producer didn't carry 'Delivery Month'."""
|
|
39
|
+
df = get_prices(symbol, adjustment)
|
|
40
|
+
if df.empty or "Delivery Month" not in df.columns:
|
|
41
|
+
return pd.DatetimeIndex([])
|
|
42
|
+
changed = df["Delivery Month"].ne(df["Delivery Month"].shift())
|
|
43
|
+
return df.index[changed.fillna(False)]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""A thin seam for price producers — enough to swap sources, not a plugin
|
|
2
|
+
framework (single active source = Norgate)."""
|
|
3
|
+
from typing import Protocol
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class PriceProvider(Protocol):
|
|
9
|
+
name: str
|
|
10
|
+
|
|
11
|
+
def fetch(self, internal_symbol: str, adjustment: str) -> pd.DataFrame:
|
|
12
|
+
"""Return daily bars (Open/High/Low/Close/Volume/Open Interest, tz-naive
|
|
13
|
+
Date index; optionally 'Delivery Month') for one symbol/adjustment."""
|
|
14
|
+
...
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""CFTC COT producer — cross-platform (just HTTPS zips from cftc.gov).
|
|
2
|
+
|
|
3
|
+
Downloads dea_fut_xls_{year}.zip (Legacy futures report), parses, and writes a
|
|
4
|
+
per-code weekly positioning table to the store via store.write_cot(). The
|
|
5
|
+
positioning-INDEX + trading signals stay in cot-analyzer (CotIndexer/metrics);
|
|
6
|
+
cotdata owns only the raw weekly data.
|
|
7
|
+
|
|
8
|
+
Self-contained — no cot-analyzer imports. Ported from cot-analyzer/src/core/etl.py.
|
|
9
|
+
"""
|
|
10
|
+
import datetime as dt
|
|
11
|
+
import io
|
|
12
|
+
import zipfile
|
|
13
|
+
from email.utils import parsedate_to_datetime
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import pandas as pd
|
|
17
|
+
import requests
|
|
18
|
+
|
|
19
|
+
from .. import config, store
|
|
20
|
+
from ..registry import all_symbols, hist_code_scales
|
|
21
|
+
|
|
22
|
+
URL_PREFIX = "https://www.cftc.gov/files/dea/history/dea_fut_xls_"
|
|
23
|
+
FIRST_YEAR = 1986 # CFTC Legacy futures history start
|
|
24
|
+
|
|
25
|
+
# CFTC XLS column names (Legacy futures report)
|
|
26
|
+
MARKET_NAME = "Market_and_Exchange_Names"
|
|
27
|
+
REPORT_DATE = "Report_Date_as_MM_DD_YYYY"
|
|
28
|
+
CONTRACT_CODE = "CFTC_Contract_Market_Code"
|
|
29
|
+
OPEN_INTEREST = "Open_Interest_All"
|
|
30
|
+
COMM_LONG = "Comm_Positions_Long_All"
|
|
31
|
+
COMM_SHORT = "Comm_Positions_Short_All"
|
|
32
|
+
NONCOMM_LONG = "NonComm_Positions_Long_All"
|
|
33
|
+
NONCOMM_SHORT = "NonComm_Positions_Short_All"
|
|
34
|
+
NONREPT_LONG = "NonRept_Positions_Long_All"
|
|
35
|
+
NONREPT_SHORT = "NonRept_Positions_Short_All"
|
|
36
|
+
|
|
37
|
+
TARGET_COLS = [MARKET_NAME, REPORT_DATE, CONTRACT_CODE, OPEN_INTEREST,
|
|
38
|
+
COMM_LONG, COMM_SHORT, NONCOMM_LONG, NONCOMM_SHORT,
|
|
39
|
+
NONREPT_LONG, NONREPT_SHORT]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _cache_dir() -> Path:
|
|
43
|
+
d = config.store_root() / "_cache" / "cot_legacy"
|
|
44
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
return d
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _standardize_code(val) -> str:
|
|
49
|
+
"""CFTC contract codes → 6-digit zero-padded string (matches CotSymbolCodeMap)."""
|
|
50
|
+
s = str(val).strip()
|
|
51
|
+
return s.zfill(6) if s.isdigit() else s
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _download_year(year: int) -> Path | None:
|
|
55
|
+
# CFTC changed their naming convention in 2004.
|
|
56
|
+
# 1986 - 2003: deafut_xls_{year}.zip
|
|
57
|
+
# 2004 - Present: dea_fut_xls_{year}.zip
|
|
58
|
+
if year < 2004:
|
|
59
|
+
url = f"https://www.cftc.gov/files/dea/history/deafut_xls_{year}.zip"
|
|
60
|
+
filename = f"deafut_xls_{year}.zip"
|
|
61
|
+
else:
|
|
62
|
+
url = f"{URL_PREFIX}{year}.zip"
|
|
63
|
+
filename = f"dea_fut_xls_{year}.zip"
|
|
64
|
+
|
|
65
|
+
zip_path = _cache_dir() / filename
|
|
66
|
+
try:
|
|
67
|
+
if zip_path.exists():
|
|
68
|
+
head = requests.head(url, timeout=30)
|
|
69
|
+
server_mtime = head.headers.get("Last-Modified")
|
|
70
|
+
if server_mtime and zip_path.stat().st_mtime >= parsedate_to_datetime(server_mtime).timestamp():
|
|
71
|
+
return zip_path # up to date
|
|
72
|
+
r = requests.get(url, timeout=180)
|
|
73
|
+
r.raise_for_status()
|
|
74
|
+
zip_path.write_bytes(r.content)
|
|
75
|
+
return zip_path
|
|
76
|
+
except Exception as e: # noqa: BLE001
|
|
77
|
+
print(f" {year}: download failed — {e}")
|
|
78
|
+
return zip_path if zip_path.exists() else None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _parse_zip(zip_path: Path) -> pd.DataFrame:
|
|
82
|
+
"""Extract the .xls from a year zip → cleaned target columns."""
|
|
83
|
+
with zipfile.ZipFile(zip_path) as zf:
|
|
84
|
+
with zf.open(zf.namelist()[0]) as fh:
|
|
85
|
+
data = fh.read()
|
|
86
|
+
df = pd.read_excel(io.BytesIO(data), usecols=TARGET_COLS) # .xls → needs xlrd
|
|
87
|
+
df[CONTRACT_CODE] = df[CONTRACT_CODE].apply(_standardize_code)
|
|
88
|
+
df[REPORT_DATE] = pd.to_datetime(df[REPORT_DATE]).dt.tz_localize(None)
|
|
89
|
+
return df
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def update(codes=None, first_year: int = FIRST_YEAR, last_year=None) -> None:
|
|
93
|
+
"""Download + parse CFTC Legacy futures COT; write full per-code history.
|
|
94
|
+
|
|
95
|
+
codes: iterable of CFTC codes; default = all registry codes.
|
|
96
|
+
Rebuilds the complete per-code table each run (parse is cheap; downloads are
|
|
97
|
+
cached and skip when unchanged). Incremental append is a future optimization.
|
|
98
|
+
"""
|
|
99
|
+
last_year = last_year or dt.date.today().year
|
|
100
|
+
if codes:
|
|
101
|
+
want = set(codes)
|
|
102
|
+
else:
|
|
103
|
+
want = {s.cftc_code for s in all_symbols() if s.cftc_code}
|
|
104
|
+
for s in all_symbols(): # predecessor codes (migrated-contract history)
|
|
105
|
+
want.update(code for code, _ in hist_code_scales(s.hist_codes))
|
|
106
|
+
|
|
107
|
+
frames = []
|
|
108
|
+
for year in range(first_year, last_year + 1):
|
|
109
|
+
zp = _download_year(year)
|
|
110
|
+
if zp is None:
|
|
111
|
+
continue
|
|
112
|
+
try:
|
|
113
|
+
frames.append(_parse_zip(zp))
|
|
114
|
+
except Exception as e: # noqa: BLE001
|
|
115
|
+
print(f" {year}: parse failed — {e}")
|
|
116
|
+
|
|
117
|
+
if not frames:
|
|
118
|
+
print("cftc: no data parsed")
|
|
119
|
+
return
|
|
120
|
+
|
|
121
|
+
allrows = pd.concat(frames, ignore_index=True)
|
|
122
|
+
for code in sorted(want):
|
|
123
|
+
sub = allrows[allrows[CONTRACT_CODE] == code].copy()
|
|
124
|
+
if sub.empty:
|
|
125
|
+
print(f"{code}: no rows")
|
|
126
|
+
continue
|
|
127
|
+
sub = sub.sort_values(REPORT_DATE).set_index(REPORT_DATE)
|
|
128
|
+
store.write_cot_legacy(code, sub, source="cftc")
|
|
129
|
+
print(f"{code}: {len(sub):5d} weeks (legacy) -> store")
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""CFTC COT Disaggregated Futures-Only producer — cross-platform.
|
|
2
|
+
|
|
3
|
+
Downloads fut_disagg_txt_{year}.zip, parses losslessly (preserving all Traders_*
|
|
4
|
+
and detailed entity columns), and writes per-code weekly positioning tables to
|
|
5
|
+
the store via store.write_cot_disagg().
|
|
6
|
+
"""
|
|
7
|
+
import datetime as dt
|
|
8
|
+
import io
|
|
9
|
+
import zipfile
|
|
10
|
+
from email.utils import parsedate_to_datetime
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import pandas as pd
|
|
14
|
+
import requests
|
|
15
|
+
|
|
16
|
+
from .. import config, store
|
|
17
|
+
from ..registry import all_symbols, hist_code_scales
|
|
18
|
+
|
|
19
|
+
URL_PREFIX = "https://www.cftc.gov/files/dea/history/fut_disagg_txt_"
|
|
20
|
+
FIRST_YEAR = 2006 # Disaggregated futures history start
|
|
21
|
+
|
|
22
|
+
# CFTC TXT column names we must coerce/standardize (others pass through losslessly)
|
|
23
|
+
REPORT_DATE = "Report_Date_as_MM_DD_YYYY"
|
|
24
|
+
CONTRACT_CODE = "CFTC_Contract_Market_Code"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _cache_dir() -> Path:
|
|
28
|
+
d = config.store_root() / "_cache" / "cot_disagg"
|
|
29
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
30
|
+
return d
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _standardize_code(val) -> str:
|
|
34
|
+
"""CFTC contract codes → 6-digit zero-padded string (matches CotSymbolCodeMap)."""
|
|
35
|
+
s = str(val).strip()
|
|
36
|
+
return s.zfill(6) if s.isdigit() else s
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _download_url(url: str, filename: str):
|
|
40
|
+
"""Download a zip to the cache; skip if the server copy isn't newer."""
|
|
41
|
+
zip_path = _cache_dir() / filename
|
|
42
|
+
try:
|
|
43
|
+
if zip_path.exists():
|
|
44
|
+
head = requests.head(url, timeout=30)
|
|
45
|
+
server_mtime = head.headers.get("Last-Modified")
|
|
46
|
+
if server_mtime and zip_path.stat().st_mtime >= parsedate_to_datetime(server_mtime).timestamp():
|
|
47
|
+
return zip_path # up to date
|
|
48
|
+
r = requests.get(url, timeout=180)
|
|
49
|
+
r.raise_for_status()
|
|
50
|
+
zip_path.write_bytes(r.content)
|
|
51
|
+
return zip_path
|
|
52
|
+
except Exception as e: # noqa: BLE001
|
|
53
|
+
print(f" {filename} (disagg): download failed — {e}")
|
|
54
|
+
return zip_path if zip_path.exists() else None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _parse_zip(zip_path: Path) -> pd.DataFrame:
|
|
58
|
+
"""Extract the .txt (CSV) from a year zip → full lossless DataFrame."""
|
|
59
|
+
with zipfile.ZipFile(zip_path) as zf:
|
|
60
|
+
with zf.open(zf.namelist()[0]) as fh:
|
|
61
|
+
# The .txt files are actually CSVs. low_memory=False prevents dtype warnings.
|
|
62
|
+
df = pd.read_csv(fh, low_memory=False)
|
|
63
|
+
|
|
64
|
+
# Strip any trailing whitespace from column names BEFORE accessing them
|
|
65
|
+
df.columns = df.columns.str.strip()
|
|
66
|
+
|
|
67
|
+
# CFTC sometimes changes the date column name in Disaggregated reports
|
|
68
|
+
if REPORT_DATE not in df.columns:
|
|
69
|
+
date_cols = [c for c in df.columns if "Report_Date" in c]
|
|
70
|
+
if date_cols:
|
|
71
|
+
df.rename(columns={date_cols[0]: REPORT_DATE}, inplace=True)
|
|
72
|
+
else:
|
|
73
|
+
print(f"AVAILABLE COLUMNS: {list(df.columns)}")
|
|
74
|
+
|
|
75
|
+
# Coerce the key schema columns to match the Legacy schema format
|
|
76
|
+
df[CONTRACT_CODE] = df[CONTRACT_CODE].apply(_standardize_code)
|
|
77
|
+
df[REPORT_DATE] = pd.to_datetime(df[REPORT_DATE]).dt.tz_localize(None)
|
|
78
|
+
|
|
79
|
+
# Parquet cannot serialize mixed-type object columns (e.g. Traders_Tot_Old contains ints and strings)
|
|
80
|
+
for col in df.select_dtypes(include=['object']).columns:
|
|
81
|
+
if col not in [CONTRACT_CODE, REPORT_DATE]:
|
|
82
|
+
df[col] = df[col].astype(str)
|
|
83
|
+
|
|
84
|
+
return df
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def update(codes=None, first_year: int = FIRST_YEAR, last_year=None) -> None:
|
|
88
|
+
"""Download + parse CFTC Disaggregated futures COT; write full per-code history.
|
|
89
|
+
|
|
90
|
+
codes: iterable of CFTC codes; default = all registry codes.
|
|
91
|
+
Rebuilds the complete per-code table each run.
|
|
92
|
+
"""
|
|
93
|
+
last_year = last_year or dt.date.today().year
|
|
94
|
+
if codes:
|
|
95
|
+
want = set(codes)
|
|
96
|
+
else:
|
|
97
|
+
want = {s.cftc_code for s in all_symbols() if s.cftc_code}
|
|
98
|
+
for s in all_symbols(): # predecessor codes (migrated-contract history)
|
|
99
|
+
want.update(code for code, _ in hist_code_scales(s.hist_codes))
|
|
100
|
+
|
|
101
|
+
frames = []
|
|
102
|
+
|
|
103
|
+
# The CFTC bundles 2006-2016 in a single historical file
|
|
104
|
+
if first_year <= 2016:
|
|
105
|
+
url = "https://www.cftc.gov/files/dea/history/fut_disagg_txt_hist_2006_2016.zip"
|
|
106
|
+
zp = _download_url(url, "fut_disagg_txt_hist_2006_2016.zip")
|
|
107
|
+
if zp:
|
|
108
|
+
try:
|
|
109
|
+
df = _parse_zip(zp)
|
|
110
|
+
# Filter down to the requested year range so we don't accidentally pull earlier
|
|
111
|
+
# than first_year if the user explicitly wanted a shorter window.
|
|
112
|
+
df = df[(df[REPORT_DATE].dt.year >= first_year) & (df[REPORT_DATE].dt.year <= 2016)]
|
|
113
|
+
frames.append(df)
|
|
114
|
+
except Exception as e:
|
|
115
|
+
print(f" hist_2006_2016 (disagg): parse failed — {e}")
|
|
116
|
+
|
|
117
|
+
# Fetch individual years for 2017+ (or first_year if it's > 2016)
|
|
118
|
+
for year in range(max(2017, first_year), last_year + 1):
|
|
119
|
+
zp = _download_url(f"{URL_PREFIX}{year}.zip", f"fut_disagg_txt_{year}.zip")
|
|
120
|
+
if not zp:
|
|
121
|
+
continue
|
|
122
|
+
try:
|
|
123
|
+
frames.append(_parse_zip(zp))
|
|
124
|
+
except Exception as e: # noqa: BLE001
|
|
125
|
+
print(f" {year} (disagg): parse failed — {e}")
|
|
126
|
+
|
|
127
|
+
if not frames:
|
|
128
|
+
print("cftc_disagg: no data parsed")
|
|
129
|
+
return
|
|
130
|
+
|
|
131
|
+
allrows = pd.concat(frames, ignore_index=True)
|
|
132
|
+
|
|
133
|
+
# Parquet cannot serialize mixed-type object columns after concat (due to NaNs)
|
|
134
|
+
for col in allrows.select_dtypes(include=['object']).columns:
|
|
135
|
+
if col not in [CONTRACT_CODE, REPORT_DATE]:
|
|
136
|
+
allrows[col] = allrows[col].astype(str)
|
|
137
|
+
|
|
138
|
+
for code in sorted(want):
|
|
139
|
+
sub = allrows[allrows[CONTRACT_CODE] == code].copy()
|
|
140
|
+
if sub.empty:
|
|
141
|
+
continue
|
|
142
|
+
|
|
143
|
+
# Index by report date (DatetimeIndex → manifest last_date)
|
|
144
|
+
sub = sub.sort_values(REPORT_DATE).set_index(REPORT_DATE)
|
|
145
|
+
store.write_cot_disagg(code, sub, source="cftc_disagg")
|
|
146
|
+
print(f"{code}: {len(sub):5d} weeks (disagg) -> store")
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""CFTC COT Traders in Financial Futures (TFF) producer — cross-platform.
|
|
2
|
+
|
|
3
|
+
Downloads fut_fin_txt_{year}.zip, parses losslessly (preserving all Traders_*
|
|
4
|
+
and detailed entity columns), and writes per-code weekly positioning tables to
|
|
5
|
+
the store via store.write_cot_tff().
|
|
6
|
+
"""
|
|
7
|
+
import datetime as dt
|
|
8
|
+
import io
|
|
9
|
+
import zipfile
|
|
10
|
+
from email.utils import parsedate_to_datetime
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import pandas as pd
|
|
14
|
+
import requests
|
|
15
|
+
|
|
16
|
+
from .. import config, store
|
|
17
|
+
from ..registry import all_symbols, hist_code_scales
|
|
18
|
+
|
|
19
|
+
URL_PREFIX = "https://www.cftc.gov/files/dea/history/fut_fin_txt_"
|
|
20
|
+
FIRST_YEAR = 2006 # TFF futures history start
|
|
21
|
+
|
|
22
|
+
# CFTC TXT column names we must coerce/standardize (others pass through losslessly)
|
|
23
|
+
REPORT_DATE = "Report_Date_as_MM_DD_YYYY"
|
|
24
|
+
CONTRACT_CODE = "CFTC_Contract_Market_Code"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _cache_dir() -> Path:
|
|
28
|
+
d = config.store_root() / "_cache" / "cot_tff"
|
|
29
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
30
|
+
return d
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _standardize_code(val) -> str:
|
|
34
|
+
"""CFTC contract codes → 6-digit zero-padded string (matches CotSymbolCodeMap)."""
|
|
35
|
+
s = str(val).strip()
|
|
36
|
+
return s.zfill(6) if s.isdigit() else s
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _download_url(url: str, filename: str):
|
|
40
|
+
"""Download a zip to the cache; skip if the server copy isn't newer."""
|
|
41
|
+
zip_path = _cache_dir() / filename
|
|
42
|
+
try:
|
|
43
|
+
if zip_path.exists():
|
|
44
|
+
head = requests.head(url, timeout=30)
|
|
45
|
+
server_mtime = head.headers.get("Last-Modified")
|
|
46
|
+
if server_mtime and zip_path.stat().st_mtime >= parsedate_to_datetime(server_mtime).timestamp():
|
|
47
|
+
return zip_path # up to date
|
|
48
|
+
r = requests.get(url, timeout=180)
|
|
49
|
+
r.raise_for_status()
|
|
50
|
+
zip_path.write_bytes(r.content)
|
|
51
|
+
return zip_path
|
|
52
|
+
except Exception as e: # noqa: BLE001
|
|
53
|
+
print(f" {filename} (tff): download failed — {e}")
|
|
54
|
+
return zip_path if zip_path.exists() else None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _parse_zip(zip_path: Path) -> pd.DataFrame:
|
|
58
|
+
"""Extract the .txt (CSV) from a year zip → full lossless DataFrame."""
|
|
59
|
+
with zipfile.ZipFile(zip_path) as zf:
|
|
60
|
+
with zf.open(zf.namelist()[0]) as fh:
|
|
61
|
+
# The .txt files are actually CSVs. low_memory=False prevents dtype warnings.
|
|
62
|
+
df = pd.read_csv(fh, low_memory=False)
|
|
63
|
+
|
|
64
|
+
# Strip any trailing whitespace from column names BEFORE accessing them
|
|
65
|
+
df.columns = df.columns.str.strip()
|
|
66
|
+
|
|
67
|
+
# CFTC sometimes changes the date column name in TFF reports
|
|
68
|
+
if REPORT_DATE not in df.columns:
|
|
69
|
+
date_cols = [c for c in df.columns if "Report_Date" in c]
|
|
70
|
+
if date_cols:
|
|
71
|
+
df.rename(columns={date_cols[0]: REPORT_DATE}, inplace=True)
|
|
72
|
+
else:
|
|
73
|
+
print(f"AVAILABLE COLUMNS: {list(df.columns)}")
|
|
74
|
+
|
|
75
|
+
# Coerce the key schema columns to match the Legacy schema format
|
|
76
|
+
df[CONTRACT_CODE] = df[CONTRACT_CODE].apply(_standardize_code)
|
|
77
|
+
df[REPORT_DATE] = pd.to_datetime(df[REPORT_DATE]).dt.tz_localize(None)
|
|
78
|
+
|
|
79
|
+
# Parquet cannot serialize mixed-type object columns (e.g. Traders_Tot_Old contains ints and strings)
|
|
80
|
+
for col in df.select_dtypes(include=['object']).columns:
|
|
81
|
+
if col not in [CONTRACT_CODE, REPORT_DATE]:
|
|
82
|
+
df[col] = df[col].astype(str)
|
|
83
|
+
|
|
84
|
+
return df
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def update(codes=None, first_year: int = FIRST_YEAR, last_year=None) -> None:
|
|
88
|
+
"""Download + parse CFTC TFF futures COT; write full per-code history.
|
|
89
|
+
|
|
90
|
+
codes: iterable of CFTC codes; default = all registry codes.
|
|
91
|
+
Rebuilds the complete per-code table each run.
|
|
92
|
+
"""
|
|
93
|
+
last_year = last_year or dt.date.today().year
|
|
94
|
+
if codes:
|
|
95
|
+
want = set(codes)
|
|
96
|
+
else:
|
|
97
|
+
want = {s.cftc_code for s in all_symbols() if s.cftc_code}
|
|
98
|
+
for s in all_symbols(): # predecessor codes (migrated-contract history)
|
|
99
|
+
want.update(code for code, _ in hist_code_scales(s.hist_codes))
|
|
100
|
+
|
|
101
|
+
frames = []
|
|
102
|
+
|
|
103
|
+
# The CFTC bundles 2006-2016 in a single historical file
|
|
104
|
+
if first_year <= 2016:
|
|
105
|
+
url = "https://www.cftc.gov/files/dea/history/fin_fut_txt_2006_2016.zip"
|
|
106
|
+
zp = _download_url(url, "fin_fut_txt_2006_2016.zip")
|
|
107
|
+
if zp:
|
|
108
|
+
try:
|
|
109
|
+
df = _parse_zip(zp)
|
|
110
|
+
# Filter down to the requested year range so we don't accidentally pull earlier
|
|
111
|
+
# than first_year if the user explicitly wanted a shorter window.
|
|
112
|
+
df = df[(df[REPORT_DATE].dt.year >= first_year) & (df[REPORT_DATE].dt.year <= 2016)]
|
|
113
|
+
frames.append(df)
|
|
114
|
+
except Exception as e:
|
|
115
|
+
print(f" hist_2006_2016 (tff): parse failed — {e}")
|
|
116
|
+
|
|
117
|
+
# Fetch individual years for 2017+ (or first_year if it's > 2016)
|
|
118
|
+
for year in range(max(2017, first_year), last_year + 1):
|
|
119
|
+
zp = _download_url(f"{URL_PREFIX}{year}.zip", f"fut_fin_txt_{year}.zip")
|
|
120
|
+
if not zp:
|
|
121
|
+
continue
|
|
122
|
+
try:
|
|
123
|
+
frames.append(_parse_zip(zp))
|
|
124
|
+
except Exception as e: # noqa: BLE001
|
|
125
|
+
print(f" {year} (tff): parse failed — {e}")
|
|
126
|
+
|
|
127
|
+
if not frames:
|
|
128
|
+
print("cftc_tff: no data parsed")
|
|
129
|
+
return
|
|
130
|
+
|
|
131
|
+
allrows = pd.concat(frames, ignore_index=True)
|
|
132
|
+
|
|
133
|
+
# Parquet cannot serialize mixed-type object columns after concat (due to NaNs)
|
|
134
|
+
for col in allrows.select_dtypes(include=['object']).columns:
|
|
135
|
+
if col not in [CONTRACT_CODE, REPORT_DATE]:
|
|
136
|
+
allrows[col] = allrows[col].astype(str)
|
|
137
|
+
|
|
138
|
+
for code in sorted(want):
|
|
139
|
+
sub = allrows[allrows[CONTRACT_CODE] == code].copy()
|
|
140
|
+
if sub.empty:
|
|
141
|
+
continue
|
|
142
|
+
|
|
143
|
+
# Index by report date (DatetimeIndex → manifest last_date)
|
|
144
|
+
sub = sub.sort_values(REPORT_DATE).set_index(REPORT_DATE)
|
|
145
|
+
store.write_cot_tff(code, sub, source="cftc_tff")
|
|
146
|
+
print(f"{code}: {len(sub):5d} weeks (tff) -> store")
|