cotdata 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cotdata-0.3.0/PKG-INFO +587 -0
- cotdata-0.3.0/README.md +555 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/pyproject.toml +19 -4
- cotdata-0.3.0/src/cotdata/__init__.py +12 -0
- cotdata-0.3.0/src/cotdata/config.py +68 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/src/cotdata/cot.py +3 -3
- cotdata-0.3.0/src/cotdata/prices.py +186 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/src/cotdata/providers/cftc.py +40 -9
- {cotdata-0.1.0 → cotdata-0.3.0}/src/cotdata/providers/cftc_disagg.py +35 -18
- {cotdata-0.1.0 → cotdata-0.3.0}/src/cotdata/providers/cftc_tff.py +35 -18
- cotdata-0.3.0/src/cotdata/providers/databento.py +1099 -0
- cotdata-0.3.0/src/cotdata/providers/norgate.py +546 -0
- cotdata-0.3.0/src/cotdata/providers/yfinance.py +70 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/src/cotdata/registry.py +85 -2
- cotdata-0.3.0/src/cotdata/registry.yaml +155 -0
- cotdata-0.3.0/src/cotdata/status.py +218 -0
- cotdata-0.3.0/src/cotdata/store.py +347 -0
- cotdata-0.3.0/src/cotdata/update.py +264 -0
- cotdata-0.3.0/src/cotdata/vintage.py +749 -0
- cotdata-0.3.0/src/cotdata/vintage_cli.py +437 -0
- cotdata-0.3.0/src/cotdata/vintage_flow.py +144 -0
- cotdata-0.3.0/src/cotdata/vintage_ingest.py +707 -0
- cotdata-0.3.0/src/cotdata/vintage_schedule.py +629 -0
- cotdata-0.3.0/src/cotdata.egg-info/PKG-INFO +587 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/src/cotdata.egg-info/SOURCES.txt +25 -1
- cotdata-0.3.0/src/cotdata.egg-info/entry_points.txt +6 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/src/cotdata.egg-info/requires.txt +4 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/tests/test_adjustment.py +1 -1
- {cotdata-0.1.0 → cotdata-0.3.0}/tests/test_cftc_disagg_fidelity.py +7 -5
- {cotdata-0.1.0 → cotdata-0.3.0}/tests/test_cftc_reconciliation.py +16 -15
- {cotdata-0.1.0 → cotdata-0.3.0}/tests/test_cftc_tff_reconciliation.py +19 -18
- cotdata-0.3.0/tests/test_cli_exit.py +60 -0
- cotdata-0.3.0/tests/test_cot.py +76 -0
- cotdata-0.3.0/tests/test_current_baseline.py +96 -0
- cotdata-0.3.0/tests/test_databento_build.py +241 -0
- cotdata-0.3.0/tests/test_databento_ingest.py +514 -0
- cotdata-0.3.0/tests/test_databento_provider.py +193 -0
- cotdata-0.3.0/tests/test_manifest_seam.py +281 -0
- cotdata-0.3.0/tests/test_norgate_provider.py +520 -0
- cotdata-0.3.0/tests/test_propadj.py +142 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/tests/test_registry.py +104 -5
- cotdata-0.3.0/tests/test_status.py +151 -0
- cotdata-0.3.0/tests/test_store.py +179 -0
- cotdata-0.3.0/tests/test_validate_databento.py +107 -0
- cotdata-0.3.0/tests/test_vintage_canonicalize.py +261 -0
- cotdata-0.3.0/tests/test_vintage_capture.py +656 -0
- cotdata-0.3.0/tests/test_vintage_flow.py +87 -0
- cotdata-0.3.0/tests/test_vintage_ingest.py +453 -0
- cotdata-0.3.0/tests/test_vintage_schedule.py +451 -0
- cotdata-0.3.0/tests/test_yfinance_provider.py +61 -0
- cotdata-0.1.0/PKG-INFO +0 -246
- cotdata-0.1.0/README.md +0 -217
- cotdata-0.1.0/src/cotdata/__init__.py +0 -11
- cotdata-0.1.0/src/cotdata/config.py +0 -35
- cotdata-0.1.0/src/cotdata/prices.py +0 -43
- cotdata-0.1.0/src/cotdata/providers/databento.py +0 -280
- cotdata-0.1.0/src/cotdata/providers/norgate.py +0 -78
- cotdata-0.1.0/src/cotdata/registry.yaml +0 -105
- cotdata-0.1.0/src/cotdata/store.py +0 -96
- cotdata-0.1.0/src/cotdata/update.py +0 -42
- cotdata-0.1.0/src/cotdata.egg-info/PKG-INFO +0 -246
- cotdata-0.1.0/src/cotdata.egg-info/entry_points.txt +0 -2
- cotdata-0.1.0/tests/test_store.py +0 -48
- {cotdata-0.1.0 → cotdata-0.3.0}/LICENSE +0 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/setup.cfg +0 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/src/cotdata/providers/__init__.py +0 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/src/cotdata/providers/base.py +0 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/src/cotdata.egg-info/dependency_links.txt +0 -0
- {cotdata-0.1.0 → cotdata-0.3.0}/src/cotdata.egg-info/top_level.txt +0 -0
cotdata-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,587 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cotdata
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Canonical data layer for COT/futures strategies — a producer/consumer split over a file-based store.
|
|
5
|
+
Author-email: "Matt S." <cotanalyzer@gmail.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/mspinola/cotdata
|
|
7
|
+
Project-URL: Repository, https://github.com/mspinola/cotdata
|
|
8
|
+
Project-URL: Issues, https://github.com/mspinola/cotdata/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: pandas>=1.5
|
|
16
|
+
Requires-Dist: pyarrow>=10
|
|
17
|
+
Requires-Dist: requests>=2.28
|
|
18
|
+
Requires-Dist: python-dateutil>=2.8
|
|
19
|
+
Requires-Dist: packaging
|
|
20
|
+
Requires-Dist: xlrd>=2.0
|
|
21
|
+
Requires-Dist: PyYAML>=6.0
|
|
22
|
+
Provides-Extra: norgate
|
|
23
|
+
Requires-Dist: norgatedata; extra == "norgate"
|
|
24
|
+
Provides-Extra: databento
|
|
25
|
+
Requires-Dist: databento>=0.30; extra == "databento"
|
|
26
|
+
Provides-Extra: yahoo
|
|
27
|
+
Requires-Dist: yfinance; extra == "yahoo"
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
30
|
+
Requires-Dist: ruff==0.15.22; extra == "dev"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# cotdata
|
|
34
|
+
|
|
35
|
+
[](https://github.com/mspinola/cotdata/actions/workflows/python-test.yml)
|
|
36
|
+
[](https://pypi.org/project/cotdata/)
|
|
37
|
+
[](https://pypi.org/project/cotdata/)
|
|
38
|
+
[](https://opensource.org/licenses/MIT)
|
|
39
|
+
|
|
40
|
+
**A local, file-based data layer for futures prices and CFTC Commitments of Traders (COT) positioning.**
|
|
41
|
+
|
|
42
|
+
cotdata separates *fetching* data (a "producer" that talks to vendors) from *using* it (any number of "consumers" that just read Parquet through a small, stable API). Point every tool at one synced store, and none of them ever call a vendor SDK at runtime — so the same data feeds your research, backtests, and dashboards identically, on any OS.
|
|
43
|
+
|
|
44
|
+
- **One store, many readers.** Consumers `import cotdata` and read; they never touch a vendor SDK. Swapping a data vendor is a producer-only change.
|
|
45
|
+
- **Free COT, optional paid prices.** CFTC Commitments of Traders data (1986–present) downloads free from cftc.gov on any OS. Futures prices/specs come from [Norgate](https://norgatedata.com/) (paid, Windows) and are optional.
|
|
46
|
+
- **Cross-platform reads.** Produce on Windows (for Norgate); read anywhere (Mac/Linux/Windows), offline.
|
|
47
|
+
- **Predecessor stitching.** `get_cot()` transparently stitches migrated CFTC codes (e.g. the Russell 2000) and rescales tick-size changes (e.g. Lumber) into one continuous series.
|
|
48
|
+
- **Atomic writes.** Read the store safely even while the producer is downloading and writing.
|
|
49
|
+
- **New-data signal.** Every run writes a structured `status.json` so downstream tools can poll one file to detect fresh data.
|
|
50
|
+
|
|
51
|
+
## Data sources at a glance
|
|
52
|
+
|
|
53
|
+
| Data | Source | Cost | Runs on |
|
|
54
|
+
|------|--------|------|---------|
|
|
55
|
+
| CFTC COT — legacy / disaggregated / TFF | [cftc.gov](https://www.cftc.gov/) | **Free** | any OS |
|
|
56
|
+
| Futures prices + contract specs | [Norgate Data](https://norgatedata.com/) | Paid subscription | **Windows** (producer only) |
|
|
57
|
+
| Futures prices, back-adjusted | [Databento](https://databento.com/) GLBX.MDP3 | Paid per query | any OS |
|
|
58
|
+
| Futures prices, research-grade fallback | Yahoo Finance | **Free** | any OS |
|
|
59
|
+
| *Reading the store* (any of the above) | — | Free | any OS |
|
|
60
|
+
|
|
61
|
+
## Contents
|
|
62
|
+
|
|
63
|
+
- [Quickstart](#quickstart) · [How it works](#how-it-works) · [Reading data](#reading-data-consumer) · [Producing data](#producing-data-producer) · [Windows setup](docs/WINDOWS_SETUP.md) · [Scheduling on Windows](docs/WINDOWS_SCHEDULING.md) · [Scheduling on Linux](docs/LINUX_SCHEDULING.md) · [Syncing the store](docs/SYNCING.md) · [Operations](#operations) · [Concepts & design](#concepts--design) · [COT vintage tracking](#cot-vintage-tracking-as-published-history) · [Reference: schemas](#reference-data-schemas) · [Reference: COT formats](#reference-cot-formats-explained) · [Diagnostics](#diagnostics) · [Development](#development) · [Contributing](#contributing) · [License](#license)
|
|
64
|
+
|
|
65
|
+
## Quickstart
|
|
66
|
+
|
|
67
|
+
The fastest zero-cost path uses free CFTC COT data — no account, any OS:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install cotdata
|
|
71
|
+
export COTDATA_STORE=~/cotdata_store # where the shared store lives
|
|
72
|
+
cotdata-update --cot-legacy # free CFTC download (first run pulls history; cached after)
|
|
73
|
+
python -c "import cotdata; print(cotdata.get_cot('ES').tail())"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
That downloads the CFTC Legacy COT history and reads the S&P 500 (ES) positioning back out:
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
Open_Interest_All Comm_Positions_Long_All Comm_Positions_Short_All NonComm_Positions_Long_All NonComm_Positions_Short_All
|
|
80
|
+
Report_Date_as_MM_DD_YYYY
|
|
81
|
+
2026-06-23 1980254 1444102 1531232 251385 286833
|
|
82
|
+
2026-06-30 1967167 1422155 1509889 249934 287526
|
|
83
|
+
2026-07-07 1969636 1435736 1502199 244103 286994
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Futures **prices** additionally require a Norgate subscription on Windows — see [Producing data](#producing-data-producer).
|
|
87
|
+
|
|
88
|
+
## How it works
|
|
89
|
+
|
|
90
|
+
The **store is the API boundary** — not Python imports. Producers write Parquet + `manifest.json`; consumers only read. Nobody touches a vendor SDK at app runtime, so swapping a vendor is a producer-only change.
|
|
91
|
+
|
|
92
|
+
```
|
|
93
|
+
PRODUCER — runs where each source is reachable
|
|
94
|
+
Norgate export (Windows) CFTC COT download (any OS)
|
|
95
|
+
│ │
|
|
96
|
+
└──────────────┬───────────────┘
|
|
97
|
+
▼ write parquet + manifest
|
|
98
|
+
┌────────────────────────────────────────────────────────────┐
|
|
99
|
+
│ CANONICAL STORE ($COTDATA_STORE) │
|
|
100
|
+
│ prices/ cot_legacy/ cot_disagg/ cot_tff/ │
|
|
101
|
+
│ metadata/ manifest.json status.json │
|
|
102
|
+
└────────────────────────────────────────────────────────────┘
|
|
103
|
+
│ read (offline, any OS)
|
|
104
|
+
┌──────────────┴───────────────┐
|
|
105
|
+
▼ ▼
|
|
106
|
+
your signal research your backtest / dashboards
|
|
107
|
+
|
|
108
|
+
both just: import cotdata · store synced via rsync / Dropbox / S3
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The store layout:
|
|
112
|
+
|
|
113
|
+
- `prices/{symbol}_{adjustment}.parquet` — Open/High/Low/Close/Volume/Open Interest, tz-naive `Date` index. `adjustment` ∈ {`backadj`, `unadj`} on disk; `propadj` is a third view **derived on read** (not stored). Close = exchange settlement.
|
|
114
|
+
- `cot_legacy/{symbol}_{code}.parquet` — weekly CFTC Legacy positioning.
|
|
115
|
+
- `cot_disagg/{symbol}_{code}.parquet` — weekly CFTC Disaggregated positioning.
|
|
116
|
+
- `cot_tff/{symbol}_{code}.parquet` — weekly CFTC Traders in Financial Futures positioning.
|
|
117
|
+
- `metadata/contract_specs.parquet` — Norgate contract specifications (tick size, point value, margin).
|
|
118
|
+
- `manifest.json` — per-table `last_date`, `n_rows`, `source`, `updated_at`, `schema_version`.
|
|
119
|
+
- `status.json` — machine-readable new-data signal for downstream tools (see [Operations](#operations)).
|
|
120
|
+
- `vintage/` — optional as-published (vintage) capture: retained raw CFTC downloads plus
|
|
121
|
+
change-only observations and field-level revisions. Purely additive; the tables above are
|
|
122
|
+
unchanged whether or not it is enabled. See [COT vintage tracking](#cot-vintage-tracking-as-published-history).
|
|
123
|
+
|
|
124
|
+
## Reading data (consumer)
|
|
125
|
+
|
|
126
|
+
Set `COTDATA_STORE` to the synced store directory, then:
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
import cotdata
|
|
130
|
+
|
|
131
|
+
# Prices — pick the adjustment that matches your use:
|
|
132
|
+
signals = cotdata.get_prices("ES", adjustment="backadj") # signals + stops (gap-free rolls)
|
|
133
|
+
sizing = cotdata.get_prices("ES", adjustment="unadj") # position sizing (true dollar prices)
|
|
134
|
+
milk = cotdata.get_prices("DC", adjustment="propadj") # ratio-adjusted: strictly positive, %-return preserving
|
|
135
|
+
|
|
136
|
+
# COT — three CFTC report families:
|
|
137
|
+
legacy = cotdata.get_cot("ES", report="legacy") # Commercial / Non-Commercial
|
|
138
|
+
disagg = cotdata.get_cot("ES", report="disagg") # Managed Money, Swap Dealers, ... (commodities)
|
|
139
|
+
tff = cotdata.get_cot("ES", report="tff") # Leveraged Funds, Asset Managers, ... (financials)
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
A price frame (`get_prices("ES", adjustment="backadj").tail(3)`):
|
|
143
|
+
|
|
144
|
+
```
|
|
145
|
+
Open High Low Close Volume Open Interest
|
|
146
|
+
Date
|
|
147
|
+
2026-07-10 7587.25 7628.75 7552.75 7620.25 1078031.0 1966297.0
|
|
148
|
+
2026-07-13 7607.00 7615.25 7547.25 7563.00 1274520.0 1945908.0
|
|
149
|
+
2026-07-14 7557.00 7613.75 7531.50 7591.25 1139735.0 0.0
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
**Predecessor stitching & scaling:** `get_cot()` doesn't just read a file — it stitches historical CFTC codes for contracts that migrated exchanges (e.g. the Russell 2000) and rescales data for contracts that changed tick sizes (e.g. Lumber), so downstream models see one clean, continuous asset.
|
|
153
|
+
|
|
154
|
+
## Producing data (producer)
|
|
155
|
+
|
|
156
|
+
Run on the machine that can reach the source. Norgate prices require Windows, CFTC COT runs anywhere, and a server without Norgate can build prices from Databento instead (see [Cross-platform prices without Norgate](#cross-platform-prices-without-norgate-databento)).
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
COTDATA_STORE=/store cotdata-update --prices # Norgate prices, ALL registry symbols (Windows)
|
|
160
|
+
COTDATA_STORE=/store cotdata-update --prices --symbols ES NQ # ...or a subset
|
|
161
|
+
COTDATA_STORE=/store cotdata-update --metadata # Norgate contract specs (Windows)
|
|
162
|
+
COTDATA_STORE=/store cotdata-update --cot-legacy # CFTC Legacy (any OS)
|
|
163
|
+
COTDATA_STORE=/store cotdata-update --cot-disagg # CFTC Disaggregated (any OS)
|
|
164
|
+
COTDATA_STORE=/store cotdata-update --cot-tff # CFTC Traders in Financial Futures (any OS)
|
|
165
|
+
COTDATA_STORE=/store cotdata-update --cot-all # all three CFTC COT reports
|
|
166
|
+
COTDATA_STORE=/store cotdata-vintage fetch # optional: capture as-published COT (any OS)
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
`--prices` with no `--symbols` updates every symbol in the registry; add `--symbols` to scope it. Each run prints a per-symbol line with the date advance (e.g. `ES: … [2026-07-13 -> 2026-07-14]`) and a summary footer (OK/failed counts, rows written, elapsed, newest date). A run **exits non-zero** if a fetch hard-fails (Norgate/CFTC unreachable), so a scheduler can retry — see [Scheduling on Windows](#scheduling-on-windows-task-scheduler).
|
|
170
|
+
|
|
171
|
+
### Installation for the producer
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
pip install "cotdata[norgate]" # adds the norgatedata dependency (Windows)
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
The `norgatedata` package talks locally to the Norgate Data Updater application — there are no API keys. You just need the Updater installed, authenticated, and running.
|
|
178
|
+
|
|
179
|
+
### Cross-platform prices without Norgate (databento)
|
|
180
|
+
|
|
181
|
+
A server that cannot run Norgate (for example the public dashboard host) can build the price store from Databento instead. One provider owns each symbol end to end, so this is a full replacement, not a blend. Install the extras and set the environment:
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
pip install "cotdata[databento,yahoo]" # databento producer + the Yahoo fallback
|
|
185
|
+
|
|
186
|
+
export COTDATA_STORE=/path/to/store # the store the dashboard reads
|
|
187
|
+
export COTDATA_PRICE_SOURCE=databento # deployment default, so softs/MSCI fall to Yahoo
|
|
188
|
+
export DATABENTO_API_KEY=db-... # for the paid ingest step only
|
|
189
|
+
# optional: export COTDATA_DATABENTO_RAW=/path/to/raw # defaults to $COTDATA_STORE/_raw/databento
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Then build the store in order (ingest before build):
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
cotdata-update --ingest-databento # Stage 1 (PAID): raw .n.0/.n.1 ohlcv-1d + statistics -> raw store
|
|
196
|
+
cotdata-update --build-databento # Stage 2 (FREE): additive back-adjustment -> $COTDATA_STORE/prices
|
|
197
|
+
cotdata-update --prices-yahoo # softs, lumber, MSCI proxies (resolve to Yahoo on this deployment)
|
|
198
|
+
cotdata-update --cot-all # CFTC COT, the dashboard needs it too
|
|
199
|
+
cotdata-update --check # coverage, newest dates, staleness
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
- **Two stages, one paid.** Stage 1 is the only step that hits the API. It writes an append-only raw store and resumes from the last fetched date, so re-runs pull only new days. Stage 2 reads that raw store with no API cost, so the back-adjustment can be iterated offline. The raw store is producer-internal, so keep it out of any sync to consumers.
|
|
203
|
+
- **History starts 2010-06-06** (the GLBX floor), shallower than Norgate. Markets not on CME Globex (ICE softs, lumber, MSCI intl) fall back to Yahoo.
|
|
204
|
+
- **First-run check.** A healthy symbol prints `built unadj+backadj (N bars, K rolls)`. If it prints `no rolls detected`, back-adjustment is a no-op for that symbol, so investigate before trusting it.
|
|
205
|
+
- **Validate against Norgate** (optional gate) with `scripts/validate_databento_vs_norgate.py` if you have both stores.
|
|
206
|
+
- **Schedule** the two price commands nightly and `--cot-all` weekly — see [Scheduling on Linux](docs/LINUX_SCHEDULING.md).
|
|
207
|
+
|
|
208
|
+
### Producer halves: one host, one job
|
|
209
|
+
|
|
210
|
+
`cotdata` has two producers by design: the CFTC downloader (free, any OS) and the price
|
|
211
|
+
producer (Norgate needs Windows). Two entry points scope a host to one of them:
|
|
212
|
+
|
|
213
|
+
```bash
|
|
214
|
+
cotdata-cot --cot-all # CFTC half, any OS
|
|
215
|
+
cotdata-prices --prices --metadata --require-final # price half, Windows for Norgate
|
|
216
|
+
cotdata-update ... # both, for a single-machine deployment
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Each scoped entry point refuses the other half's flags, so a price box cannot quietly
|
|
220
|
+
become a second COT producer racing the first. `--check` and `--reconcile` are read-only
|
|
221
|
+
and work from either.
|
|
222
|
+
|
|
223
|
+
Each half also owns its own manifest (`manifests/cot.json`, `manifests/prices.json`).
|
|
224
|
+
The legacy top-level `manifest.json` held both halves in ONE file, which was unsafe two
|
|
225
|
+
ways: the update is a read-modify-write, so two producers lose each other's entries, and
|
|
226
|
+
a file-level sync between two stores resolves it last-writer-wins and silently discards
|
|
227
|
+
one side. The per-half files are disjoint, so both problems go away.
|
|
228
|
+
|
|
229
|
+
**Nothing writes `manifest.json` any more.** Migrate a store once:
|
|
230
|
+
|
|
231
|
+
```bash
|
|
232
|
+
cotdata-update --migrate-manifests
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Idempotent, and it never touches data. Until you run it, a domain missing from the
|
|
236
|
+
per-half files is still read from the aggregate with a warning. Delete `manifest.json`
|
|
237
|
+
once every consumer of that store is on this version. See ADR-0007.
|
|
238
|
+
|
|
239
|
+
### Scheduling on Windows (Task Scheduler)
|
|
240
|
+
|
|
241
|
+
Full setup, including wrapper scripts, the three-task layout (daily prices, daily COT catch-up, Friday release-window poller), `--require-final` event-driven pricing, restart-on-failure retry settings, and Norgate/Task-Scheduler troubleshooting (notably: NDU needs an interactive session), is in **[docs/WINDOWS_SCHEDULING.md](docs/WINDOWS_SCHEDULING.md)**. Start with the [Windows Setup Guide](docs/WINDOWS_SETUP.md) first if Python/the venv/`COTDATA_STORE` aren't configured yet.
|
|
242
|
+
|
|
243
|
+
The short version: prices fire once daily near the Norgate Continuous Futures Final (~8:55pm ET), COT gets a daily morning catch-up plus a tight Friday-afternoon poll around its ~3:30pm ET release, and every task uses restart-on-failure so idempotent, cheap re-runs absorb both transient errors and "not published yet."
|
|
244
|
+
|
|
245
|
+
### Scheduling on Linux (cron)
|
|
246
|
+
|
|
247
|
+
Full setup, including wrapper scripts, the crontab entries (nightly prices, daily COT catch-up, Friday release-window poller), `flock` overlap protection, and troubleshooting (cron's bare environment, timezone conversion, `DATABENTO_API_KEY` not being picked up), is in **[docs/LINUX_SCHEDULING.md](docs/LINUX_SCHEDULING.md)**.
|
|
248
|
+
|
|
249
|
+
The short version: a databento server schedules the same way as the Windows/Norgate producer — prices nightly, COT gets a daily morning catch-up plus a tight Friday-afternoon poll around its ~3:30pm ET release, all idempotent and safe to over-run.
|
|
250
|
+
|
|
251
|
+
### Syncing the store between machines
|
|
252
|
+
|
|
253
|
+
Norgate needs Windows, so a research Mac or a Linux dashboard is usually a **read-only
|
|
254
|
+
replica** of a store produced elsewhere. Prefer one producer writing everything and a
|
|
255
|
+
strictly one-directional sync.
|
|
256
|
+
|
|
257
|
+
Two directories must be **excluded** for size: `_cache/` (cotdata's cache of downloaded
|
|
258
|
+
CFTC source zips) and `_raw/` (the **paid** databento raw store) are producer-internal
|
|
259
|
+
and together are ~70% of the bytes. The legacy `manifest.json` should be excluded too.
|
|
260
|
+
|
|
261
|
+
Anything a consumer put in the store by hand is a **correctness** issue rather than a
|
|
262
|
+
saving. No producer creates it, so a mirroring sync deletes it. Exclude it, but the real
|
|
263
|
+
fix is to keep it out of the store: the store belongs to its producer.
|
|
264
|
+
|
|
265
|
+
The same rule bites hardest on `vintage/` if you enable it, because that data cannot be
|
|
266
|
+
re-fetched: capture it on the **producer** so it syncs outward, or keep it outside the
|
|
267
|
+
mirrored store with `COTDATA_VINTAGE_ROOT`. Its provenance index is deliberately named
|
|
268
|
+
`snapshots.json`, since the usual `manifest.json` exclusion matches by name at any depth
|
|
269
|
+
and would otherwise strip it in transit, delivering raw archives with no index.
|
|
270
|
+
|
|
271
|
+
Consumer cloud sync (Dropbox, Google Drive) is a poor fit here: conflict copies land
|
|
272
|
+
inside the store, and on-demand placeholder files break `read_parquet` on the machine
|
|
273
|
+
doing research.
|
|
274
|
+
|
|
275
|
+
Full guidance, exclusion table and example scripts: **[docs/SYNCING.md](docs/SYNCING.md)**.
|
|
276
|
+
|
|
277
|
+
## Operations
|
|
278
|
+
|
|
279
|
+
Read-only and maintenance commands, all cross-platform (they work off the store, no network):
|
|
280
|
+
|
|
281
|
+
```bash
|
|
282
|
+
cotdata-update --check # store status: row counts, newest data, staleness
|
|
283
|
+
cotdata-update --reconcile # prune stale manifest entries (see below)
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
`--check` reports per-domain row counts, newest data date, last write, and any entries lagging behind their peers (a partial-run signal):
|
|
287
|
+
|
|
288
|
+
```
|
|
289
|
+
domain entries rows newest data last write (UTC) behind
|
|
290
|
+
prices 84 829,096 2026-07-14 2026-07-15T10:15:24Z 1d
|
|
291
|
+
cot_legacy 44 70,201 2026-07-07 2026-07-14T04:26:55Z 8d
|
|
292
|
+
...
|
|
293
|
+
✓ all entries current (none lag behind their domain's newest).
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
### `status.json` — new-data signal for downstream tools
|
|
297
|
+
|
|
298
|
+
Every producer run writes `$COTDATA_STORE/status.json` (atomically, beside the data), so tools that trigger on fresh data poll one small structured file instead of scanning the store:
|
|
299
|
+
|
|
300
|
+
```json
|
|
301
|
+
{
|
|
302
|
+
"generated_at": "2026-07-15T10:15:24Z",
|
|
303
|
+
"schema_version": 2,
|
|
304
|
+
"newest_data": { "prices": "2026-07-14", "cot_legacy": "2026-07-07", "cot_disagg": "2026-07-07", "cot_tff": "2026-07-07" },
|
|
305
|
+
"domains": { "prices": { "newest_data": "2026-07-14", "last_write": "2026-07-15T10:15:24Z", "entries": 84, "rows": 829096, "lagging": 0 }, "...": {} },
|
|
306
|
+
"last_run": { "kinds": ["prices"], "ok": ["ES", "..."], "symbols_failed": [], "rows": 1658000, "seconds": 88, "at": "2026-07-15T10:15:24Z" }
|
|
307
|
+
}
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
**Polling contract:**
|
|
311
|
+
- To detect **new data**, compare `newest_data.<domain>` (e.g. `newest_data.prices`, `newest_data.cot_legacy`) against your last-seen value. It advances **only when genuinely new daily data arrives** — a no-op run leaves it unchanged.
|
|
312
|
+
- To detect that **a run happened at all** (new data or not), use `generated_at`.
|
|
313
|
+
- `last_run` carries the most recent run's outcome (which domains, per-symbol failures) for alerting.
|
|
314
|
+
|
|
315
|
+
Prices and each COT report are separate domains, so a price-triggered tool and a COT-triggered tool each watch their own key.
|
|
316
|
+
|
|
317
|
+
### `--reconcile` — manifest hygiene
|
|
318
|
+
|
|
319
|
+
COT tables are stored per code as **`{symbol}_{code}`** (e.g. `RTY_23977A`), so a symbol's current and predecessor (`hist_codes`) contracts are both attributable to it. `--reconcile` drops manifest entries whose parquet file is missing — bare-code ghosts and retired domains left by older naming schemes — so `--check` and `status.json` show only real, consistently-named entries. It never touches data (only removes bookkeeping for files that don't exist).
|
|
320
|
+
|
|
321
|
+
## Concepts & design
|
|
322
|
+
|
|
323
|
+
### Back-adjusted vs unadjusted prices
|
|
324
|
+
|
|
325
|
+
Futures contracts expire, forcing traders to "roll" into the next contract, which usually trades at a slightly different price. Simply stitching contracts together creates artificial price gaps, so cotdata stores two series and derives a third:
|
|
326
|
+
|
|
327
|
+
- **`backadj` (signals & stops).** Gap-free *arithmetic* (additive) rolls shift historical prices to align with the new contract, preserving *absolute* daily point moves. Use this for indicators, signals, and stop-losses to avoid false triggers on rollover gaps.
|
|
328
|
+
- **`unadj` (position sizing).** Back-adjustment shifts historical prices (sometimes negative), so you can't use it for dollar values. Use `unadj` (raw, real-life prices) for that day to compute true dollar risk and contract counts.
|
|
329
|
+
- **`propadj` (proportional / ratio adjustment — strictly positive).** Derived on read from `unadj` + `backadj`; preserves daily *percentage* returns and never goes non-positive. Use it for **low-priced, long-history contracts where additive back-adjustment accumulates roll gaps below zero** and breaks price-based stops and R-multiples. See *Class III Milk (DC)* below.
|
|
330
|
+
|
|
331
|
+
#### Why `propadj` exists — Class III Milk (DC)
|
|
332
|
+
|
|
333
|
+
Norgate publishes continuous futures in only two forms: unadjusted and **additive** back-adjusted (`_CCB`) — there is no native ratio-adjusted series. Additive adjustment subtracts each roll's calendar spread from all prior history, and for a low-priced, seasonal, ~29-year contract like **DC (Class III Milk, ~$15–20/cwt)** those gaps accumulate past zero: **46.7% of `DC_backadj` closes are ≤ 0** (range −9.83 to 23.09). A price-based stop, an R-multiple, or a percentage return is meaningless on a non-positive series, so CMR cannot use DC's `backadj` at all — even though DC is the flagship *new-asset-class* (Dairy) held-out generalization market.
|
|
334
|
+
|
|
335
|
+
`propadj` salvages it. Because the additive series `B` and unadjusted series `U` differ by an offset `O = B − U` that steps only at rolls, each roll's calendar spread is recoverable (`s = O[r−1] − O[r]`) and convertible to a multiplicative roll ratio `k = (U[r−1] + s)/U[r−1]`. Scaling each historical segment by the cumulative product of `k` (most-recent segment anchored to actual prices) yields a series that is **strictly positive over the full 1997–2026 history** (DC range 4.68–25.01), preserves within-segment percentage returns exactly, and is sign-identical to `backadj` on every day including rolls. It is a pure function of two already-stored series, so it needs no producer re-run — `get_prices("DC", adjustment="propadj")` works today. Recommendation: **CMR reads DC (and any similarly low-priced contract) with `adjustment="propadj"`.** Restricting DC to its positive-price era (2011→present, ~15y) or dropping it were the fallbacks; neither is needed.
|
|
336
|
+
|
|
337
|
+
### Providers & authentication
|
|
338
|
+
|
|
339
|
+
Which vendor prices a symbol is a deployment choice, not a fixed fact (the same ES is Norgate for local research and databento on a public-dash server). It is resolved when a producer runs, from three inputs: the deployment default `COTDATA_PRICE_SOURCE` (`norgate` if unset), per-symbol capability (the `norgate` / `databento` / `yahoo` mappings in the registry, `null` where a vendor has no series), and an optional per-symbol `price_source` override. A symbol uses its override if set, otherwise the default when that vendor can serve it, otherwise a Yahoo fallback where a ticker exists. Each producer writes only the symbols that resolve to it, so a symbol is never blended across vendors. One provider owns each symbol end to end.
|
|
340
|
+
|
|
341
|
+
- **Norgate Data (paid, Windows).** No Python API key. The `norgatedata` package talks locally to the Norgate Data Updater app, which must be installed, authenticated, and running on Windows. This is the default for local research (`cotdata-update --prices`).
|
|
342
|
+
- **Databento (paid, cross-platform).** A two-stage producer for a server that cannot run Norgate. Stage 1 (`cotdata-update --ingest-databento`) pulls raw `.n.0` / `.n.1` `ohlcv-1d` and `statistics` into an append-only raw store (`$COTDATA_DATABENTO_RAW`, else `_raw/databento` under the store). This is the only paid step, and it is resumable, so re-runs fetch only new dates. Stage 2 (`cotdata-update --build-databento`) derives the back-adjusted prices from the raw store with no API cost, so the build logic can be iterated offline. Set `DATABENTO_API_KEY` and `COTDATA_PRICE_SOURCE=databento`. History starts 2010-06-06, and markets not on CME Globex (ICE softs, lumber, MSCI intl) fall back to Yahoo. The raw store is producer-internal, so exclude it from any consumer sync.
|
|
343
|
+
- **Yahoo Finance (free, research-grade).** `cotdata-update --prices-yahoo` prices the markets that resolve to yfinance on this deployment: the MSCI ETF proxies always, plus the softs and lumber on a databento server. Expect gaps and silent revisions. This is not a production replacement for the paid feeds. Requires the `[yahoo]` extra.
|
|
344
|
+
|
|
345
|
+
### The symbol registry
|
|
346
|
+
|
|
347
|
+
The supported futures contracts are defined in a YAML registry, so adding a market needs no code:
|
|
348
|
+
|
|
349
|
+
- **Add a market:** edit `src/cotdata/registry.yaml` under its asset class. The registry handles metadata like `is_equity` and predecessor `hist_codes`.
|
|
350
|
+
- **Centralize it:** set `COTDATA_REGISTRY` to a shared `registry.yaml` (e.g. inside `$COTDATA_STORE`) so producer and consumers use identical asset definitions without a `git pull`.
|
|
351
|
+
|
|
352
|
+
### Atomic store
|
|
353
|
+
|
|
354
|
+
The store uses **atomic writes** (write-temp-then-rename). Consumers can safely query via `get_prices` / `get_cot` even while `cotdata-update` is actively downloading and writing.
|
|
355
|
+
|
|
356
|
+
### COT vintage tracking (as-published history)
|
|
357
|
+
|
|
358
|
+
CFTC revises COT data after publication — most consequentially through **trader
|
|
359
|
+
reclassification**, which moves positions between categories retroactively. Because
|
|
360
|
+
downstream signals are rolling z-scores and percentiles against years of history, a
|
|
361
|
+
restatement silently rewrites the baseline every historical reading was computed against.
|
|
362
|
+
There is precedent: in July 2008 the Commission revised reports back to July 3, 2007.
|
|
363
|
+
|
|
364
|
+
**CFTC serves current state only.** There is no vintage archive and no as-published
|
|
365
|
+
endpoint, so vintage data can only be accumulated going forward — every uncaptured week is
|
|
366
|
+
a permanent blind spot in the part of the series most likely to have been revised.
|
|
367
|
+
|
|
368
|
+
This is **opt-in and purely additive**: if you never run it, the store behaves exactly as
|
|
369
|
+
before. Enabling it adds a `vintage/` subtree.
|
|
370
|
+
|
|
371
|
+
```bash
|
|
372
|
+
cotdata-vintage fetch # capture current + prior year + weekly static (daily)
|
|
373
|
+
cotdata-vintage fetch --all # every year 1986-present (see below)
|
|
374
|
+
cotdata-vintage ingest --pending # parse retained raw -> observations + revisions
|
|
375
|
+
cotdata-vintage diff --since 2026-01-01 # field-level revisions, with revision depth
|
|
376
|
+
cotdata-vintage asof --as-of 2026-07-24T18:00:00 --report-date 2026-07-21
|
|
377
|
+
cotdata-vintage flow --market 088691 # weekly flow decomposition (see below)
|
|
378
|
+
cotdata-schedule sync # CFTC Special Announcements
|
|
379
|
+
cotdata-schedule published # true publication dates from retained weekly statics
|
|
380
|
+
cotdata-schedule backfill # resolve release_date + its provenance
|
|
381
|
+
```
|
|
382
|
+
|
|
383
|
+
How it works:
|
|
384
|
+
|
|
385
|
+
- **Immutable landing zone.** Every fetch is recorded (including 304s) and raw bytes are
|
|
386
|
+
retained permanently under `vintage/raw/`, written atomically and never rewritten. A
|
|
387
|
+
byte-identical regeneration is deduped — a changed download is not itself a revision.
|
|
388
|
+
- **All three reports.** Legacy, Disaggregated and TFF canonicalise into one long schema.
|
|
389
|
+
Disagg and TFF also populate per-category spreading, per-category trader counts and
|
|
390
|
+
CR4/CR8 concentration, none of which the Legacy file carries, and they are where
|
|
391
|
+
**Managed Money** and **Leveraged Funds** live. A suppressed trader count (CFTC writes
|
|
392
|
+
`.`) canonicalises to null rather than to a string.
|
|
393
|
+
- **Change-only observations.** A row is written only when its value hash differs from the
|
|
394
|
+
latest for its natural key `(report_date, market_code, report_type, combined, category)`,
|
|
395
|
+
so storage grows with actual revisions rather than with time.
|
|
396
|
+
- **Field-level revisions** carry `age_days` (revision depth): whether revisions stay in
|
|
397
|
+
recent weeks or reach back into the calibration window determines how much the rest of
|
|
398
|
+
a system has to care.
|
|
399
|
+
- **Point-in-time reads.** `asof(t)` returns each key's latest value observed at or before
|
|
400
|
+
`t`, reconstructing what was actually knowable then.
|
|
401
|
+
- **Release dates with provenance.** `report_date` is stored exactly as reported (never
|
|
402
|
+
normalized to Tuesday), and `release_date` is resolved through
|
|
403
|
+
`published > observed > announced > scheduled > derived`, with the source recorded — a
|
|
404
|
+
release date without provenance is worse than none, since indexing on `report_date`
|
|
405
|
+
embeds a lookahead (three days normally, weeks during a backlog). `published` is the
|
|
406
|
+
weekly static's HTTP `Last-Modified`, a true publication timestamp; it is forward-only
|
|
407
|
+
(that file holds one week and is overwritten), so weeks predating capture fall back
|
|
408
|
+
down the chain. `announced` comes from the republication tables CFTC posts on the
|
|
409
|
+
Special Announcements page after a disruption, and is what puts the Oct–Dec 2025
|
|
410
|
+
appropriations-lapse backlog on its real dates: 36,296 stored rows that `derived`
|
|
411
|
+
otherwise places up to 47 days early, before the lapse that stopped them being
|
|
412
|
+
published at all.
|
|
413
|
+
|
|
414
|
+
- **Flow decomposition.** `cotdata-vintage flow` labels each week's ΔLong versus ΔShort as
|
|
415
|
+
`new_longs` / `short_covering` / `new_shorts` / `long_liquidation`, by dominant leg, with
|
|
416
|
+
no parameters to tune. A rally driven by short covering has a finite fuel supply and one
|
|
417
|
+
driven by fresh longs does not, and they are indistinguishable on a price chart. It also
|
|
418
|
+
emits `days_elapsed`, because COT was **fortnightly until 1992-10-13** and holidays shift
|
|
419
|
+
the rest, so a "weekly" change is not always weekly.
|
|
420
|
+
|
|
421
|
+
**Run capture on the producer, not a replica**, and schedule it **daily**: nearly every
|
|
422
|
+
request returns 304, so a daily run is close to free while catching holiday-shifted and
|
|
423
|
+
backlog releases with no schedule logic.
|
|
424
|
+
|
|
425
|
+
**The frozen-year tripwire.** CFTC regenerates a rolling two-year window (current plus the
|
|
426
|
+
immediately-prior year) and nothing older. The prior year is therefore re-served every week
|
|
427
|
+
but byte-identical, which is the one place a **content** check on closed data comes free.
|
|
428
|
+
It is in the default fetch set for that reason: it costs one roughly 7 MB transfer per week
|
|
429
|
+
and zero bytes on disk, and it is the only automated retroactive-restatement detector here.
|
|
430
|
+
Anything other than `unchanged bytes (deduped)` on it raises an alert, which `ingest`
|
|
431
|
+
re-raises as a non-zero exit and a `REVISIONS_<date>.txt` marker file. `--no-prior-year`
|
|
432
|
+
turns it off. `--all` extends the same check to every year, but since nothing older is ever
|
|
433
|
+
re-served it mostly confirms 304s; monthly or quarterly is the right cadence for that. Full
|
|
434
|
+
design notes, including the measured CFTC caching behaviour, are in
|
|
435
|
+
[docs/design/cot_vintage.md](docs/design/cot_vintage.md).
|
|
436
|
+
|
|
437
|
+
> **Replica warning.** The vintage tree must not be written on a machine whose store is
|
|
438
|
+
> mirrored (`robocopy /MIR`, `rsync --delete`) from a producer: the mirror deletes
|
|
439
|
+
> destination-only files and the data is irreplaceable. Capture on the producer, or set
|
|
440
|
+
> `COTDATA_VINTAGE_ROOT` to a path outside the mirrored store. See [docs/SYNCING.md](docs/SYNCING.md).
|
|
441
|
+
|
|
442
|
+
## Local development
|
|
443
|
+
|
|
444
|
+
```bash
|
|
445
|
+
uv venv # create .venv
|
|
446
|
+
uv pip install -e . # install cotdata + deps
|
|
447
|
+
export COTDATA_STORE=/path/to/synced/store # the shared store
|
|
448
|
+
uv run pytest # run the tests
|
|
449
|
+
```
|
|
450
|
+
|
|
451
|
+
On the Windows producer, install the Norgate extra with `uv pip install -e ".[norgate]"` (tested on Python 3.10, within Norgate's supported versions). Use `uv run <cmd>`, or activate with `source .venv/bin/activate` (Mac/Linux) / `.venv\Scripts\activate` (Windows).
|
|
452
|
+
|
|
453
|
+
## Reference: Data schemas
|
|
454
|
+
|
|
455
|
+
The canonical store uses standard Parquet files. Loaded with `pd.read_parquet()`, they conform to the following schemas.
|
|
456
|
+
|
|
457
|
+
### Price Data (`prices/{symbol}_{adjustment}.parquet`)
|
|
458
|
+
Primary price history (Norgate Data), indexed by tz-naive `Date`. The pipeline downloads both the back-adjusted (`backadj`) series for signals/stops and the unadjusted (`unadj`) series for true transaction-cost modeling.
|
|
459
|
+
|
|
460
|
+
**Reading reconstructed volume:** the reconstruction columns below are internal storage. Consumers should not read `Volume_Reconstructed` directly — call `get_prices(symbol, volume="reconstructed")` and the `Volume` column is served as reconstructed-with-per-row-raw-fallback, plus a `Volume_Source` column for audit. The default `volume="front"` returns the front-month series unchanged (byte-identical to the pre-v2 API).
|
|
461
|
+
|
|
462
|
+
**Schema versioning:** `schema_version` in `manifest.json` records the on-disk data version (v2 = reconstructed volume promoted). Consumers key cache invalidation on `cotdata.schema_version()` and can guard with `cotdata.require_schema(min_version)`.
|
|
463
|
+
|
|
464
|
+
| Column | Type | Description |
|
|
465
|
+
|--------|------|-------------|
|
|
466
|
+
| `Date` | DatetimeIndex | Trading day (tz-naive, normalized to midnight). |
|
|
467
|
+
| `Open` | float | Opening price. |
|
|
468
|
+
| `High` | float | High price. |
|
|
469
|
+
| `Low` | float | Low price. |
|
|
470
|
+
| `Close` | float | Settlement Close price. |
|
|
471
|
+
| `Volume` | float | Continuous contract trading volume (front-month only). |
|
|
472
|
+
| `Open Interest` | float | Continuous contract open interest. |
|
|
473
|
+
| `Volume_Reconstructed` | float | True market volume (sum of First and Second contract). Differs from raw `Volume` by symbol — typically higher for products whose rolls spread volume across contracts, but roughly equal or lower for symbols with a near-empty back month (e.g. crypto). Not a drop-in replacement. |
|
|
474
|
+
| `Volume_Source` | string | `reconstructed` if First+Second available, `raw` fallback if not. |
|
|
475
|
+
| `FirstVolume` / `SecondVolume` | float | Trading volume of the specific first and second expiring contracts. |
|
|
476
|
+
| `FirstContract` / `SecondContract` | string | Contract names for the first and second expirations (e.g., `ES-2024H`). |
|
|
477
|
+
| `Delivery Month` | float | Expiration month of the active contract (e.g. `202609`). Used to detect contract rolls. |
|
|
478
|
+
|
|
479
|
+
### Contract Specifications (`metadata/contract_specs.parquet`)
|
|
480
|
+
Contract metadata (Norgate Data), used for exact point-value risk sizing and transaction cost models.
|
|
481
|
+
|
|
482
|
+
| Column | Type | Description |
|
|
483
|
+
|--------|------|-------------|
|
|
484
|
+
| `Symbol` | string | Internal ticker symbol (e.g., `ES`). |
|
|
485
|
+
| `Norgate_Symbol` | string | Raw Norgate symbol used to query the API (e.g., `&ES_CCB`). |
|
|
486
|
+
| `Name` | string | Full name of the contract. |
|
|
487
|
+
| `Exchange` | string | Name of the listing exchange. |
|
|
488
|
+
| `Group` | string | Norgate asset classification group. |
|
|
489
|
+
| `Contract Size` | float | Size multiplier (e.g., $50 for ES). Also called Point Value. |
|
|
490
|
+
| `Tick Size` | float | Minimum price fluctuation (e.g., 0.25 for ES). |
|
|
491
|
+
| `Tick Value` | float | Dollar value of one tick (`Tick Size` * `Contract Size`). |
|
|
492
|
+
| `Point Value` | float | Same as `Contract Size`. |
|
|
493
|
+
| `Currency` | string | Base currency of the contract. |
|
|
494
|
+
| `Margin` | float | Initial margin requirement (if provided by Norgate). |
|
|
495
|
+
|
|
496
|
+
### COT Legacy Data (`cot_legacy/{symbol}_{code}.parquet`)
|
|
497
|
+
Legacy positioning data (CFTC Legacy Futures Report). **History starts in 1986.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
|
|
498
|
+
|
|
499
|
+
> [!NOTE]
|
|
500
|
+
> **Legacy Reports**: broken down by exchange, with futures-only and combined futures-and-options variants. Legacy classifies reportable open interest into non-commercial and commercial traders. The `cotdata` pipeline strictly downloads the **Futures-only** reports (`https://www.cftc.gov/files/dea/history/dea_fut_xls_{YEAR}.zip`).
|
|
501
|
+
|
|
502
|
+
> [!NOTE]
|
|
503
|
+
> **Column Subset**: The raw CFTC `.xls` files contain [well over 100 columns](https://www.cftc.gov/MarketReports/CommitmentsofTraders/HistoricalViewable/cotvariableslegacy.html); the pipeline keeps the focused 15-column subset below to keep files small. To include more, add the exact CFTC column name to `TARGET_COLS` in `src/cotdata/providers/cftc.py`.
|
|
504
|
+
|
|
505
|
+
| Column | Type | Description |
|
|
506
|
+
|--------|------|-------------|
|
|
507
|
+
| `Report_Date_as_MM_DD_YYYY` | DatetimeIndex | Reporting date (typically Tuesday). |
|
|
508
|
+
| `Market_and_Exchange_Names` | string | Name of the contract and exchange. |
|
|
509
|
+
| `CFTC_Contract_Market_Code` | string | 6-digit CFTC contract code. |
|
|
510
|
+
| `Open_Interest_All` | float | Total open interest for the contract. |
|
|
511
|
+
| `Comm_Positions_Long_All` | float | Commercial Long positions. |
|
|
512
|
+
| `Comm_Positions_Short_All` | float | Commercial Short positions. |
|
|
513
|
+
| `NonComm_Positions_Long_All` | float | Non-Commercial (Large Speculator) Long positions. |
|
|
514
|
+
| `NonComm_Positions_Short_All` | float | Non-Commercial (Large Speculator) Short positions. |
|
|
515
|
+
| `NonRept_Positions_Long_All` | float | Non-Reportable (Small Speculator) Long positions. |
|
|
516
|
+
| `NonRept_Positions_Short_All` | float | Non-Reportable (Small Speculator) Short positions. |
|
|
517
|
+
| `Traders_Tot_All` | float | Total number of reportable traders. |
|
|
518
|
+
| `Traders_Comm_Long_All` | float | Number of Commercial Long traders. |
|
|
519
|
+
| `Traders_Comm_Short_All` | float | Number of Commercial Short traders. |
|
|
520
|
+
| `Traders_NonComm_Long_All` | float | Number of Non-Commercial Long traders. |
|
|
521
|
+
| `Traders_NonComm_Short_All` | float | Number of Non-Commercial Short traders. |
|
|
522
|
+
|
|
523
|
+
### COT Disaggregated Data (`cot_disagg/{symbol}_{code}.parquet`)
|
|
524
|
+
Entity-specific positioning and trader counts (CFTC Disaggregated Futures-Only Report). **History starts in 2006.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
|
|
525
|
+
|
|
526
|
+
> [!NOTE]
|
|
527
|
+
> **Lossless Image**: Unlike the filtered Legacy schema, the Disaggregated parquets are a **lossless image** of the source CFTC `txt` files — all granular entity groups (Money Manager, Swap Dealer, Producer/Merchant, Other Reportable) and their `Traders_*` counts. Required for computing Position Size and Clustering metrics.
|
|
528
|
+
|
|
529
|
+
### COT Traders in Financial Futures (TFF) Data (`cot_tff/{symbol}_{code}.parquet`)
|
|
530
|
+
Entity-specific positioning and trader counts for financial markets (CFTC TFF Futures-Only Report). **History starts in 2006.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
|
|
531
|
+
|
|
532
|
+
> [!NOTE]
|
|
533
|
+
> **Financials Counterpart**: TFF is the exact counterpart to Disaggregated, used for financial markets (Equities, FX, Rates), which have no Disaggregated report.
|
|
534
|
+
|
|
535
|
+
> [!NOTE]
|
|
536
|
+
> **Lossless Image**: Like Disaggregated, TFF parquets are a **lossless image** of the source CFTC `txt` files — the financial entity groups (`Dealer`, `Asset_Mgr`, `Lev_Money`, `Other_Rept`) and their `Traders_*` counts.
|
|
537
|
+
|
|
538
|
+
## Reference: COT formats explained
|
|
539
|
+
|
|
540
|
+
The CFTC publishes positioning data in three formats; `cotdata` manages all three for complete coverage and the deepest history.
|
|
541
|
+
|
|
542
|
+
1. **Legacy (1986–Present)** — *all markets.* Divides traders into **Commercial** (hedgers) and **Non-Commercial** (large speculators). The only format with pre-2006 data, so it's essential for long-term backtesting.
|
|
543
|
+
2. **Disaggregated / DIS (2006–Present)** — *physical commodities only* (Agriculture, Energy, Metals). Splits traders into **Producer/Merchant**, **Swap Dealers**, **Managed Money**, and **Other Reportables** — a clearer view of "smart money" (Managed Money) in commodities.
|
|
544
|
+
3. **Traders in Financial Futures / TFF (2006–Present)** — *financial markets only* (Equities, Rates, Currencies). Splits traders into **Dealer/Intermediary**, **Asset Manager**, **Leveraged Funds**, and **Other Reportables** — the definitive source for speculative flow (Leveraged Funds) in financials.
|
|
545
|
+
|
|
546
|
+
## Diagnostics
|
|
547
|
+
|
|
548
|
+
Verify your Norgate subscription and configuration with the included smoke test, on the Windows producer:
|
|
549
|
+
|
|
550
|
+
```bash
|
|
551
|
+
python tests/test_adjustment.py
|
|
552
|
+
```
|
|
553
|
+
|
|
554
|
+
It checks: (1) **Local communication** — Python can reach the Norgate Data Updater; (2) **Subscription access** — your subscription includes the required CME futures package; (3) **Roll-gap validation** — proves whether the Updater is returning back-adjusted (gap-free) vs unadjusted continuous contracts, by hunting for calendar-spread gaps at roll dates. Gap-free data is vital for accurate stop-loss modeling.
|
|
555
|
+
|
|
556
|
+
## Ecosystem
|
|
557
|
+
|
|
558
|
+
cotdata is the *data* layer of a small, unbundled toolchain — it stops at "clean
|
|
559
|
+
data behind a stable API" on purpose. What you do with that data is a separate,
|
|
560
|
+
swappable step:
|
|
561
|
+
|
|
562
|
+
- **cotdata** *(this package)* — the *data* layer. One synced store of futures
|
|
563
|
+
prices and CFTC COT positioning; many readers, no vendor SDK at read time.
|
|
564
|
+
- **[crucible](https://github.com/mspinola/crucible)** — the *edge* layer. Feed a
|
|
565
|
+
signal built on cotdata frames into crucible and it tells you — with a
|
|
566
|
+
confidence interval and a p-value — whether the trade-level edge is real, before
|
|
567
|
+
you open a funded account.
|
|
568
|
+
|
|
569
|
+
The flow runs one direction: **`cotdata` (data) → your signal → `crucible`
|
|
570
|
+
(edge)**. Neither imports the other, so cotdata stays useful on its own for any
|
|
571
|
+
COT/futures research — crucible is just the most common thing to point at it next.
|
|
572
|
+
|
|
573
|
+
## Development
|
|
574
|
+
|
|
575
|
+
Want to contribute or work on cotdata locally? See [CONTRIBUTING.md](CONTRIBUTING.md) for:
|
|
576
|
+
- Virtual environment setup with `uv` or standard `pip`
|
|
577
|
+
- Running the test suite
|
|
578
|
+
- Platform-specific notes (Norgate is Windows-only; CFTC parsing runs anywhere)
|
|
579
|
+
- Code style guidelines
|
|
580
|
+
|
|
581
|
+
## Contributing
|
|
582
|
+
|
|
583
|
+
Issues and pull requests are welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, tests, and conventions. When filing a bug, include your OS — Norgate features require Windows, while store reads and CFTC COT run anywhere.
|
|
584
|
+
|
|
585
|
+
## License
|
|
586
|
+
|
|
587
|
+
Released under the MIT License — see [LICENSE](LICENSE).
|