cotdata 0.3.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cotdata-0.3.0/src/cotdata.egg-info → cotdata-0.5.0}/PKG-INFO +182 -181
- {cotdata-0.3.0 → cotdata-0.5.0}/README.md +180 -172
- {cotdata-0.3.0 → cotdata-0.5.0}/pyproject.toml +8 -11
- cotdata-0.5.0/src/cotdata/__init__.py +18 -0
- cotdata-0.5.0/src/cotdata/cit.py +177 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/config.py +32 -5
- cotdata-0.5.0/src/cotdata/cot.py +228 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/providers/cftc.py +60 -31
- cotdata-0.5.0/src/cotdata/providers/cftc_cit.py +194 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/providers/cftc_disagg.py +63 -28
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/providers/cftc_tff.py +61 -26
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/registry.py +52 -87
- cotdata-0.5.0/src/cotdata/registry.yaml +218 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/status.py +55 -7
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/store.py +71 -43
- cotdata-0.5.0/src/cotdata/update.py +185 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/vintage.py +72 -13
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/vintage_cli.py +96 -12
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/vintage_ingest.py +333 -9
- {cotdata-0.3.0 → cotdata-0.5.0/src/cotdata.egg-info}/PKG-INFO +182 -181
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata.egg-info/SOURCES.txt +10 -13
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata.egg-info/entry_points.txt +0 -1
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata.egg-info/requires.txt +0 -10
- cotdata-0.5.0/tests/test_cit_decomposition.py +154 -0
- cotdata-0.5.0/tests/test_cit_supplemental.py +507 -0
- cotdata-0.5.0/tests/test_cli_exit.py +45 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_cot.py +19 -14
- cotdata-0.5.0/tests/test_cot_combined_basis.py +353 -0
- cotdata-0.5.0/tests/test_cot_stitch.py +87 -0
- cotdata-0.5.0/tests/test_cot_sum.py +133 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_manifest_seam.py +105 -90
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_registry.py +64 -95
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_status.py +78 -15
- cotdata-0.5.0/tests/test_store.py +123 -0
- cotdata-0.5.0/tests/test_sync_preflight.py +179 -0
- cotdata-0.5.0/tests/test_verify_replicas.py +156 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_vintage_capture.py +10 -3
- cotdata-0.3.0/src/cotdata/__init__.py +0 -12
- cotdata-0.3.0/src/cotdata/cot.py +0 -62
- cotdata-0.3.0/src/cotdata/prices.py +0 -186
- cotdata-0.3.0/src/cotdata/providers/databento.py +0 -1099
- cotdata-0.3.0/src/cotdata/providers/norgate.py +0 -546
- cotdata-0.3.0/src/cotdata/providers/yfinance.py +0 -70
- cotdata-0.3.0/src/cotdata/registry.yaml +0 -155
- cotdata-0.3.0/src/cotdata/update.py +0 -264
- cotdata-0.3.0/tests/test_adjustment.py +0 -111
- cotdata-0.3.0/tests/test_cli_exit.py +0 -60
- cotdata-0.3.0/tests/test_databento_build.py +0 -241
- cotdata-0.3.0/tests/test_databento_ingest.py +0 -514
- cotdata-0.3.0/tests/test_databento_provider.py +0 -193
- cotdata-0.3.0/tests/test_norgate_provider.py +0 -520
- cotdata-0.3.0/tests/test_propadj.py +0 -142
- cotdata-0.3.0/tests/test_store.py +0 -179
- cotdata-0.3.0/tests/test_validate_databento.py +0 -107
- cotdata-0.3.0/tests/test_yfinance_provider.py +0 -61
- {cotdata-0.3.0 → cotdata-0.5.0}/LICENSE +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/setup.cfg +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/providers/__init__.py +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/providers/base.py +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/vintage_flow.py +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata/vintage_schedule.py +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata.egg-info/dependency_links.txt +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/src/cotdata.egg-info/top_level.txt +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_cftc_disagg_fidelity.py +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_cftc_reconciliation.py +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_cftc_tff_reconciliation.py +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_current_baseline.py +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_vintage_canonicalize.py +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_vintage_flow.py +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_vintage_ingest.py +0 -0
- {cotdata-0.3.0 → cotdata-0.5.0}/tests/test_vintage_schedule.py +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cotdata
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: Canonical
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Canonical CFTC positioning (COT) data layer — a producer/consumer split over a file-based store.
|
|
5
5
|
Author-email: "Matt S." <cotanalyzer@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/mspinola/cotdata
|
|
7
7
|
Project-URL: Repository, https://github.com/mspinola/cotdata
|
|
@@ -16,15 +16,8 @@ Requires-Dist: pandas>=1.5
|
|
|
16
16
|
Requires-Dist: pyarrow>=10
|
|
17
17
|
Requires-Dist: requests>=2.28
|
|
18
18
|
Requires-Dist: python-dateutil>=2.8
|
|
19
|
-
Requires-Dist: packaging
|
|
20
19
|
Requires-Dist: xlrd>=2.0
|
|
21
20
|
Requires-Dist: PyYAML>=6.0
|
|
22
|
-
Provides-Extra: norgate
|
|
23
|
-
Requires-Dist: norgatedata; extra == "norgate"
|
|
24
|
-
Provides-Extra: databento
|
|
25
|
-
Requires-Dist: databento>=0.30; extra == "databento"
|
|
26
|
-
Provides-Extra: yahoo
|
|
27
|
-
Requires-Dist: yfinance; extra == "yahoo"
|
|
28
21
|
Provides-Extra: dev
|
|
29
22
|
Requires-Dist: pytest>=7; extra == "dev"
|
|
30
23
|
Requires-Dist: ruff==0.15.22; extra == "dev"
|
|
@@ -37,26 +30,35 @@ Dynamic: license-file
|
|
|
37
30
|
[](https://pypi.org/project/cotdata/)
|
|
38
31
|
[](https://opensource.org/licenses/MIT)
|
|
39
32
|
|
|
40
|
-
**A local, file-based data layer for
|
|
33
|
+
**A local, file-based data layer for CFTC Commitments of Traders (COT) positioning.**
|
|
41
34
|
|
|
42
35
|
cotdata separates *fetching* data (a "producer" that talks to vendors) from *using* it (any number of "consumers" that just read Parquet through a small, stable API). Point every tool at one synced store, and none of them ever call a vendor SDK at runtime — so the same data feeds your research, backtests, and dashboards identically, on any OS.
|
|
43
36
|
|
|
44
37
|
- **One store, many readers.** Consumers `import cotdata` and read; they never touch a vendor SDK. Swapping a data vendor is a producer-only change.
|
|
45
|
-
- **Free
|
|
46
|
-
- **Cross-platform reads.** Produce on Windows (for Norgate); read anywhere (Mac/Linux/Windows), offline.
|
|
38
|
+
- **Free, on any OS.** CFTC Commitments of Traders data (1986–present) downloads free from cftc.gov. No account, no subscription.
|
|
47
39
|
- **Predecessor stitching.** `get_cot()` transparently stitches migrated CFTC codes (e.g. the Russell 2000) and rescales tick-size changes (e.g. Lumber) into one continuous series.
|
|
48
40
|
- **Atomic writes.** Read the store safely even while the producer is downloading and writing.
|
|
49
41
|
- **New-data signal.** Every run writes a structured `status.json` so downstream tools can poll one file to detect fresh data.
|
|
50
42
|
|
|
43
|
+
> [!IMPORTANT]
|
|
44
|
+
> **Price bars are not in this package.** Under [ADR-0007](#ecosystem), cotdata is CFTC
|
|
45
|
+
> positioning only, and every daily bar — Norgate futures, databento futures, Yahoo
|
|
46
|
+
> equities/ETFs, the `backadj`/`unadj`/`propadj` tiers, contract specifications — lives in
|
|
47
|
+
> [**crucible-marketdata**](https://pypi.org/project/crucible-marketdata/). Replace
|
|
48
|
+
> `cotdata.get_prices(sym, adjustment)` with `marketdata.get_bars(sym, tier)` and point
|
|
49
|
+
> `MARKETDATA_STORE` at that store. The two packages keep separate stores and separate
|
|
50
|
+
> producers; nothing about `get_cot()` changed.
|
|
51
|
+
|
|
51
52
|
## Data sources at a glance
|
|
52
53
|
|
|
53
54
|
| Data | Source | Cost | Runs on |
|
|
54
55
|
|------|--------|------|---------|
|
|
55
|
-
| CFTC COT — legacy / disaggregated / TFF | [cftc.gov](https://www.cftc.gov/) | **Free** | any OS |
|
|
56
|
-
|
|
|
57
|
-
|
|
|
58
|
-
|
|
59
|
-
|
|
56
|
+
| CFTC COT — legacy / disaggregated / TFF / supplemental | [cftc.gov](https://www.cftc.gov/) | **Free** | any OS |
|
|
57
|
+
| *Reading the store* | — | Free | any OS |
|
|
58
|
+
| Daily bars + contract specs | → [crucible-marketdata](https://pypi.org/project/crucible-marketdata/) | — | — |
|
|
59
|
+
|
|
60
|
+
No paid dependency, and no optional data extra: this package downloads public CFTC
|
|
61
|
+
files over HTTP and reads its own store.
|
|
60
62
|
|
|
61
63
|
## Contents
|
|
62
64
|
|
|
@@ -83,7 +85,7 @@ Report_Date_as_MM_DD_YYYY
|
|
|
83
85
|
2026-07-07 1969636 1435736 1502199 244103 286994
|
|
84
86
|
```
|
|
85
87
|
|
|
86
|
-
|
|
88
|
+
For daily **bars**, install [crucible-marketdata](https://pypi.org/project/crucible-marketdata/) alongside this and read `marketdata.get_bars("ES", "backadj")`.
|
|
87
89
|
|
|
88
90
|
## How it works
|
|
89
91
|
|
|
@@ -91,14 +93,15 @@ The **store is the API boundary** — not Python imports. Producers write Parque
|
|
|
91
93
|
|
|
92
94
|
```
|
|
93
95
|
PRODUCER — runs where each source is reachable
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
└──────────────┬───────────────┘
|
|
96
|
+
CFTC COT download — free, any OS, no vendor SDK
|
|
97
|
+
│
|
|
97
98
|
▼ write parquet + manifest
|
|
98
99
|
┌────────────────────────────────────────────────────────────┐
|
|
99
100
|
│ CANONICAL STORE ($COTDATA_STORE) │
|
|
100
|
-
│
|
|
101
|
-
│
|
|
101
|
+
│ cot_legacy/ cot_disagg/ cot_tff/ cot_supplemental/ │
|
|
102
|
+
│ cot_legacy_combined/ cot_disagg_combined/ │
|
|
103
|
+
│ cot_tff_combined/ │
|
|
104
|
+
│ manifests/ status.json │
|
|
102
105
|
└────────────────────────────────────────────────────────────┘
|
|
103
106
|
│ read (offline, any OS)
|
|
104
107
|
┌──────────────┴───────────────┐
|
|
@@ -106,16 +109,19 @@ The **store is the API boundary** — not Python imports. Producers write Parque
|
|
|
106
109
|
your signal research your backtest / dashboards
|
|
107
110
|
|
|
108
111
|
both just: import cotdata · store synced via rsync / Dropbox / S3
|
|
112
|
+
|
|
113
|
+
Daily bars live in a SEPARATE store: $MARKETDATA_STORE, read with
|
|
114
|
+
`import marketdata`. Two stores, two producers, one seam (ADR-0007).
|
|
109
115
|
```
|
|
110
116
|
|
|
111
117
|
The store layout:
|
|
112
118
|
|
|
113
|
-
- `prices/{symbol}_{adjustment}.parquet` — Open/High/Low/Close/Volume/Open Interest, tz-naive `Date` index. `adjustment` ∈ {`backadj`, `unadj`} on disk; `propadj` is a third view **derived on read** (not stored). Close = exchange settlement.
|
|
114
119
|
- `cot_legacy/{symbol}_{code}.parquet` — weekly CFTC Legacy positioning.
|
|
115
120
|
- `cot_disagg/{symbol}_{code}.parquet` — weekly CFTC Disaggregated positioning.
|
|
116
121
|
- `cot_tff/{symbol}_{code}.parquet` — weekly CFTC Traders in Financial Futures positioning.
|
|
117
|
-
- `
|
|
118
|
-
- `
|
|
122
|
+
- `cot_supplemental/{symbol}_{code}.parquet` — weekly CFTC Supplemental (Commodity Index Trader) positioning, 13 agricultural markets. **Futures-and-options combined**; the CFTC publishes no futures-only variant.
|
|
123
|
+
- `cot_legacy_combined/`, `cot_disagg_combined/`, `cot_tff_combined/` — the same three reports as the first three rows, on the **futures-and-options combined** basis (Legacy from 1995, the other two from 2006). Same columns, same `{symbol}_{code}.parquet` naming, a different quantity: combined open interest counts delta-weighted option equivalents, so on 2026-10-27 ES is 2,843,639 combined against 1,955,711 futures-only. The two bases never share a directory. Read with `get_cot(..., combined=True)`.
|
|
124
|
+
- `manifests/{cot,prices}.json` — per-table `last_date`, `n_rows`, `source`, `updated_at`, `schema_version`, one file per producer half.
|
|
119
125
|
- `status.json` — machine-readable new-data signal for downstream tools (see [Operations](#operations)).
|
|
120
126
|
- `vintage/` — optional as-published (vintage) capture: retained raw CFTC downloads plus
|
|
121
127
|
change-only observations and field-level revisions. Purely additive; the tables above are
|
|
@@ -128,18 +134,38 @@ Set `COTDATA_STORE` to the synced store directory, then:
|
|
|
128
134
|
```python
|
|
129
135
|
import cotdata
|
|
130
136
|
|
|
131
|
-
#
|
|
132
|
-
signals = cotdata.get_prices("ES", adjustment="backadj") # signals + stops (gap-free rolls)
|
|
133
|
-
sizing = cotdata.get_prices("ES", adjustment="unadj") # position sizing (true dollar prices)
|
|
134
|
-
milk = cotdata.get_prices("DC", adjustment="propadj") # ratio-adjusted: strictly positive, %-return preserving
|
|
135
|
-
|
|
136
|
-
# COT — three CFTC report families:
|
|
137
|
+
# COT — four CFTC report families, futures-only by default:
|
|
137
138
|
legacy = cotdata.get_cot("ES", report="legacy") # Commercial / Non-Commercial
|
|
138
139
|
disagg = cotdata.get_cot("ES", report="disagg") # Managed Money, Swap Dealers, ... (commodities)
|
|
139
140
|
tff = cotdata.get_cot("ES", report="tff") # Leveraged Funds, Asset Managers, ... (financials)
|
|
141
|
+
cit = cotdata.get_cot("CC", report="supplemental") # Index Traders (13 ags, combined basis)
|
|
142
|
+
|
|
143
|
+
# ...and the futures-and-options COMBINED basis of the first three:
|
|
144
|
+
legacy_fo = cotdata.get_cot("ES", report="legacy", combined=True)
|
|
145
|
+
tff_fo = cotdata.get_cot("ES", report="tff", combined=True)
|
|
146
|
+
|
|
147
|
+
# ...and, for the 13 Supplemental markets, Legacy combined joined to the Supplemental
|
|
148
|
+
# week by week, so each Legacy bucket splits into "ex index" and "index traders from it":
|
|
149
|
+
from cotdata import cit
|
|
150
|
+
parts = cit.decompose("ZW") # comm_long, comm_long_nocit, cit_from_comm_long, ... (cotdata/cit.py)
|
|
140
151
|
```
|
|
141
152
|
|
|
142
|
-
|
|
153
|
+
> [!WARNING]
|
|
154
|
+
> `combined=None` (the default) is each report's natural basis: futures-only for `legacy`,
|
|
155
|
+
> `disagg` and `tff`, combined for `supplemental`, which is published on no other basis
|
|
156
|
+
> (`combined=False` on it raises). Combined `Open_Interest_All` is a different quantity from
|
|
157
|
+
> futures-only for the same market and week. Do not difference or ratio across bases, or
|
|
158
|
+
> across reports on different bases, without accounting for that.
|
|
159
|
+
|
|
160
|
+
Daily **bars** come from the sibling package, against its own store:
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
import marketdata # pip install crucible-marketdata
|
|
164
|
+
|
|
165
|
+
signals = marketdata.get_bars("ES", "backadj") # signals + stops (gap-free rolls)
|
|
166
|
+
sizing = marketdata.get_bars("ES", "unadj") # position sizing (true dollar prices)
|
|
167
|
+
milk = marketdata.get_bars("DC", "propadj") # ratio-adjusted, %-return preserving
|
|
168
|
+
```
|
|
143
169
|
|
|
144
170
|
```
|
|
145
171
|
Open High Low Close Volume Open Interest
|
|
@@ -149,78 +175,42 @@ Date
|
|
|
149
175
|
2026-07-14 7557.00 7613.75 7531.50 7591.25 1139735.0 0.0
|
|
150
176
|
```
|
|
151
177
|
|
|
178
|
+
Same frames, same symbols, same tier names — the import and the environment variable
|
|
179
|
+
(`MARKETDATA_STORE`) are what changed. `cotdata.get_prices` and `cotdata.roll_dates` are
|
|
180
|
+
gone rather than deprecated: left importable they would read a store the nightly job no
|
|
181
|
+
longer fills, and stale data is harder to notice than an `AttributeError`.
|
|
182
|
+
|
|
152
183
|
**Predecessor stitching & scaling:** `get_cot()` doesn't just read a file — it stitches historical CFTC codes for contracts that migrated exchanges (e.g. the Russell 2000) and rescales data for contracts that changed tick sizes (e.g. Lumber), so downstream models see one clean, continuous asset.
|
|
153
184
|
|
|
154
185
|
## Producing data (producer)
|
|
155
186
|
|
|
156
|
-
|
|
187
|
+
Runs anywhere — the CFTC files are a public HTTP download. Bars of every vendor are produced by `marketdata-update` in the [sibling package](https://pypi.org/project/crucible-marketdata/).
|
|
157
188
|
|
|
158
189
|
```bash
|
|
159
|
-
COTDATA_STORE=/store cotdata-update --prices # Norgate prices, ALL registry symbols (Windows)
|
|
160
|
-
COTDATA_STORE=/store cotdata-update --prices --symbols ES NQ # ...or a subset
|
|
161
|
-
COTDATA_STORE=/store cotdata-update --metadata # Norgate contract specs (Windows)
|
|
162
190
|
COTDATA_STORE=/store cotdata-update --cot-legacy # CFTC Legacy (any OS)
|
|
163
191
|
COTDATA_STORE=/store cotdata-update --cot-disagg # CFTC Disaggregated (any OS)
|
|
164
192
|
COTDATA_STORE=/store cotdata-update --cot-tff # CFTC Traders in Financial Futures (any OS)
|
|
165
|
-
COTDATA_STORE=/store cotdata-update --cot-
|
|
193
|
+
COTDATA_STORE=/store cotdata-update --cot-supplemental # CFTC Supplemental / index traders (any OS)
|
|
194
|
+
COTDATA_STORE=/store cotdata-update --cot-legacy-combined # Legacy, futures-and-options combined (from 1995)
|
|
195
|
+
COTDATA_STORE=/store cotdata-update --cot-disagg-combined # Disaggregated, combined (from 2006)
|
|
196
|
+
COTDATA_STORE=/store cotdata-update --cot-tff-combined # TFF, combined (from 2006)
|
|
197
|
+
COTDATA_STORE=/store cotdata-update --cot-all # all seven: four reports + three combined
|
|
166
198
|
COTDATA_STORE=/store cotdata-vintage fetch # optional: capture as-published COT (any OS)
|
|
167
199
|
```
|
|
168
200
|
|
|
169
|
-
|
|
201
|
+
Each run prints a per-domain line and a summary footer. A run **exits non-zero** if a fetch hard-fails (CFTC or Databento unreachable), so a scheduler can retry — see [Scheduling on Linux](docs/LINUX_SCHEDULING.md).
|
|
170
202
|
|
|
171
|
-
###
|
|
172
|
-
|
|
173
|
-
```bash
|
|
174
|
-
pip install "cotdata[norgate]" # adds the norgatedata dependency (Windows)
|
|
175
|
-
```
|
|
203
|
+
### One producer, and the manifest split it left behind
|
|
176
204
|
|
|
177
|
-
|
|
205
|
+
`cotdata` used to have two producers — the CFTC downloader and a price producer — and two
|
|
206
|
+
entry points that scoped a host to one of them, so a price box could not quietly become a
|
|
207
|
+
second COT producer racing the first. Every price producer moved to `marketdata`, so there
|
|
208
|
+
is one producer here now and nothing to scope. `cotdata-cot` survives as an alias of
|
|
209
|
+
`cotdata-update` because the scheduled jobs call it by name; `cotdata-prices` is gone.
|
|
178
210
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
```bash
|
|
184
|
-
pip install "cotdata[databento,yahoo]" # databento producer + the Yahoo fallback
|
|
185
|
-
|
|
186
|
-
export COTDATA_STORE=/path/to/store # the store the dashboard reads
|
|
187
|
-
export COTDATA_PRICE_SOURCE=databento # deployment default, so softs/MSCI fall to Yahoo
|
|
188
|
-
export DATABENTO_API_KEY=db-... # for the paid ingest step only
|
|
189
|
-
# optional: export COTDATA_DATABENTO_RAW=/path/to/raw # defaults to $COTDATA_STORE/_raw/databento
|
|
190
|
-
```
|
|
191
|
-
|
|
192
|
-
Then build the store in order (ingest before build):
|
|
193
|
-
|
|
194
|
-
```bash
|
|
195
|
-
cotdata-update --ingest-databento # Stage 1 (PAID): raw .n.0/.n.1 ohlcv-1d + statistics -> raw store
|
|
196
|
-
cotdata-update --build-databento # Stage 2 (FREE): additive back-adjustment -> $COTDATA_STORE/prices
|
|
197
|
-
cotdata-update --prices-yahoo # softs, lumber, MSCI proxies (resolve to Yahoo on this deployment)
|
|
198
|
-
cotdata-update --cot-all # CFTC COT, the dashboard needs it too
|
|
199
|
-
cotdata-update --check # coverage, newest dates, staleness
|
|
200
|
-
```
|
|
201
|
-
|
|
202
|
-
- **Two stages, one paid.** Stage 1 is the only step that hits the API. It writes an append-only raw store and resumes from the last fetched date, so re-runs pull only new days. Stage 2 reads that raw store with no API cost, so the back-adjustment can be iterated offline. The raw store is producer-internal, so keep it out of any sync to consumers.
|
|
203
|
-
- **History starts 2010-06-06** (the GLBX floor), shallower than Norgate. Markets not on CME Globex (ICE softs, lumber, MSCI intl) fall back to Yahoo.
|
|
204
|
-
- **First-run check.** A healthy symbol prints `built unadj+backadj (N bars, K rolls)`. If it prints `no rolls detected`, back-adjustment is a no-op for that symbol, so investigate before trusting it.
|
|
205
|
-
- **Validate against Norgate** (optional gate) with `scripts/validate_databento_vs_norgate.py` if you have both stores.
|
|
206
|
-
- **Schedule** the two price commands nightly and `--cot-all` weekly — see [Scheduling on Linux](docs/LINUX_SCHEDULING.md).
|
|
207
|
-
|
|
208
|
-
### Producer halves: one host, one job
|
|
209
|
-
|
|
210
|
-
`cotdata` has two producers by design: the CFTC downloader (free, any OS) and the price
|
|
211
|
-
producer (Norgate needs Windows). Two entry points scope a host to one of them:
|
|
212
|
-
|
|
213
|
-
```bash
|
|
214
|
-
cotdata-cot --cot-all # CFTC half, any OS
|
|
215
|
-
cotdata-prices --prices --metadata --require-final # price half, Windows for Norgate
|
|
216
|
-
cotdata-update ... # both, for a single-machine deployment
|
|
217
|
-
```
|
|
218
|
-
|
|
219
|
-
Each scoped entry point refuses the other half's flags, so a price box cannot quietly
|
|
220
|
-
become a second COT producer racing the first. `--check` and `--reconcile` are read-only
|
|
221
|
-
and work from either.
|
|
222
|
-
|
|
223
|
-
Each half also owns its own manifest (`manifests/cot.json`, `manifests/prices.json`).
|
|
211
|
+
The manifest is still split per half (`manifests/cot.json`, `manifests/prices.json`), and
|
|
212
|
+
the `prices` half is now **read-only history**: a store built before the move still carries
|
|
213
|
+
those entries, and they have to keep migrating and reconciling rather than being stranded.
|
|
224
214
|
The legacy top-level `manifest.json` held both halves in ONE file, which was unsafe two
|
|
225
215
|
ways: the update is a read-modify-write, so two producers lose each other's entries, and
|
|
226
216
|
a file-level sync between two stores resolves it last-writer-wins and silently discards
|
|
@@ -238,25 +228,30 @@ once every consumer of that store is on this version. See ADR-0007.
|
|
|
238
228
|
|
|
239
229
|
### Scheduling on Windows (Task Scheduler)
|
|
240
230
|
|
|
241
|
-
|
|
231
|
+
The COT tasks (daily catch-up, Friday release-window poller), repeating-trigger poll settings, and Task-Scheduler troubleshooting are in **[docs/WINDOWS_SCHEDULING.md](docs/WINDOWS_SCHEDULING.md)**. Start with the [Windows Setup Guide](docs/WINDOWS_SETUP.md) first if Python/the venv/`COTDATA_STORE` aren't configured yet.
|
|
232
|
+
|
|
233
|
+
The short version: COT gets a daily morning catch-up plus a tight Friday-afternoon poll around its ~3:30pm ET release, and the polling tasks use a **repeating trigger** so idempotent, cheap re-runs absorb both transient errors and "not published yet." Note that Task Scheduler's *"if the task fails, restart every N minutes"* is **not** the mechanism for this — it does not fire on a non-zero exit code from your script, only on a failure to launch it. See [Polling with a repeating trigger](docs/WINDOWS_SCHEDULING.md#polling-with-a-repeating-trigger).
|
|
242
234
|
|
|
243
|
-
The
|
|
235
|
+
The Windows box is also where the **Norgate bar** job runs, but that is now `marketdata-update --bars --domain futures --require-final` from the sibling package, on its own schedule near the Norgate Continuous Futures Final (~8:55pm ET). Both packages' docs cover their own half.
|
|
244
236
|
|
|
245
237
|
### Scheduling on Linux (cron)
|
|
246
238
|
|
|
247
|
-
Full setup, including wrapper
|
|
239
|
+
Full setup, including the wrapper script, the crontab entries (daily COT catch-up plus a Friday release-window poller), `flock` overlap protection, and troubleshooting (cron's bare environment, timezone conversion), is in **[docs/LINUX_SCHEDULING.md](docs/LINUX_SCHEDULING.md)**.
|
|
248
240
|
|
|
249
|
-
The short version:
|
|
241
|
+
The short version: COT gets a daily morning catch-up plus a tight Friday-afternoon poll around its ~3:30pm ET release, all idempotent and safe to over-run.
|
|
250
242
|
|
|
251
243
|
### Syncing the store between machines
|
|
252
244
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
245
|
+
A research Mac or a Linux dashboard is usually a **read-only replica** of a store
|
|
246
|
+
produced elsewhere. Prefer one producer writing everything and a strictly
|
|
247
|
+
one-directional sync. (The bar store syncs the same way, separately — it is a different
|
|
248
|
+
directory with a different producer.)
|
|
256
249
|
|
|
257
|
-
|
|
258
|
-
CFTC source zips
|
|
259
|
-
|
|
250
|
+
One directory must be **excluded** for size: `_cache/`, cotdata's cache of downloaded
|
|
251
|
+
CFTC source zips, is producer-internal and free to rebuild. The legacy `manifest.json`
|
|
252
|
+
should be excluded too. (A store built before ADR-0007 also has `_raw/` — databento's
|
|
253
|
+
paid bronze store — and `prices/`; both belong to `marketdata` now and neither is
|
|
254
|
+
written here any more.)
|
|
260
255
|
|
|
261
256
|
Anything a consumer put in the store by hand is a **correctness** issue rather than a
|
|
262
257
|
saving. No producer creates it, so a mirroring sync deletes it. Exclude it, but the real
|
|
@@ -286,13 +281,27 @@ cotdata-update --reconcile # prune stale manifest entries (see below)
|
|
|
286
281
|
`--check` reports per-domain row counts, newest data date, last write, and any entries lagging behind their peers (a partial-run signal):
|
|
287
282
|
|
|
288
283
|
```
|
|
289
|
-
domain
|
|
290
|
-
prices
|
|
291
|
-
cot_legacy
|
|
284
|
+
domain entries rows newest data last write (UTC) behind
|
|
285
|
+
prices 98 954,524 2026-08-07 2026-08-08T15:40:20Z 14d ← RETIRED
|
|
286
|
+
cot_legacy 53 81,821 2026-08-11 2026-08-21T12:10:42Z 10d
|
|
287
|
+
cot_disagg 29 29,187 2026-08-11 2026-08-21T12:10:51Z 10d
|
|
292
288
|
...
|
|
293
|
-
|
|
289
|
+
|
|
290
|
+
⚠ prices, metadata: RETIRED — this store still holds the files, but cotdata stopped writing
|
|
291
|
+
them at ADR-0007. Their dates are frozen and will never advance again.
|
|
292
|
+
Daily bars and contract specs now live in the marketdata store ($MARKETDATA_STORE);
|
|
293
|
+
read them with `marketdata-update --check`.
|
|
294
|
+
|
|
295
|
+
✓ every entry in the live domains was written by the latest producer pass.
|
|
294
296
|
```
|
|
295
297
|
|
|
298
|
+
**A `RETIRED` domain is not a warning you can wave away.** ADR-0007 moved bars and contract
|
|
299
|
+
specs to marketdata; a store created before that move still holds `prices/` and `metadata/`,
|
|
300
|
+
and nothing advances them. They earn a label rather than silence because lag is measured
|
|
301
|
+
*within* a domain — a uniformly frozen tree is perfectly self-consistent, so it scores
|
|
302
|
+
`lagging: 0` and reads exactly like a domain that ran cleanly minutes ago. See
|
|
303
|
+
[SYNCING.md](docs/SYNCING.md#retiring-the-moved-domains-prices-and-metadata) for removing them.
|
|
304
|
+
|
|
296
305
|
### `status.json` — new-data signal for downstream tools
|
|
297
306
|
|
|
298
307
|
Every producer run writes `$COTDATA_STORE/status.json` (atomically, beside the data), so tools that trigger on fresh data poll one small structured file instead of scanning the store:
|
|
@@ -301,14 +310,16 @@ Every producer run writes `$COTDATA_STORE/status.json` (atomically, beside the d
|
|
|
301
310
|
{
|
|
302
311
|
"generated_at": "2026-07-15T10:15:24Z",
|
|
303
312
|
"schema_version": 2,
|
|
304
|
-
"newest_data": { "
|
|
305
|
-
"domains": { "
|
|
306
|
-
|
|
313
|
+
"newest_data": { "cot_legacy": "2026-08-11", "cot_disagg": "2026-08-11", "cot_tff": "2026-08-11", "cot_supplemental": "2026-08-11", "prices": "2026-08-07" },
|
|
314
|
+
"domains": { "cot_legacy": { "newest_data": "2026-08-11", "last_write": "2026-08-21T12:10:42Z", "entries": 53, "rows": 81821, "lagging": 0, "retired": false },
|
|
315
|
+
"prices": { "newest_data": "2026-08-07", "last_write": "2026-08-08T15:40:20Z", "entries": 98, "rows": 954524, "lagging": 0, "retired": true }, "...": {} },
|
|
316
|
+
"last_run": { "kinds": ["cot_legacy", "cot_disagg"], "ok": ["001602", "..."], "symbols_failed": [], "rows": 111008, "seconds": 22, "at": "2026-08-21T12:10:58Z" }
|
|
307
317
|
}
|
|
308
318
|
```
|
|
309
319
|
|
|
310
320
|
**Polling contract:**
|
|
311
|
-
- To detect **new data**, compare `newest_data.<domain>` (e.g. `newest_data.
|
|
321
|
+
- To detect **new data**, compare `newest_data.<domain>` (e.g. `newest_data.cot_legacy`) against your last-seen value. It advances **only when genuinely new daily data arrives** — a no-op run leaves it unchanged.
|
|
322
|
+
- **Check `domains.<domain>.retired` first.** A retired domain's date is frozen, so a poller keyed on it waits forever and never errors — it just quietly reports data that stopped moving. This bullet used to give `newest_data.prices` as its example, and on 2026-08-21 a consumer following it read a two-week-old date from a store whose bars had moved to marketdata. **Bars are not in this store; poll `marketdata-update --check` instead.**
|
|
312
323
|
- To detect that **a run happened at all** (new data or not), use `generated_at`.
|
|
313
324
|
- `last_run` carries the most recent run's outcome (which domains, per-symbol failures) for alerting.
|
|
314
325
|
|
|
@@ -320,27 +331,14 @@ COT tables are stored per code as **`{symbol}_{code}`** (e.g. `RTY_23977A`), so
|
|
|
320
331
|
|
|
321
332
|
## Concepts & design
|
|
322
333
|
|
|
323
|
-
###
|
|
324
|
-
|
|
325
|
-
Futures contracts expire, forcing traders to "roll" into the next contract, which usually trades at a slightly different price. Simply stitching contracts together creates artificial price gaps, so cotdata stores two series and derives a third:
|
|
326
|
-
|
|
327
|
-
- **`backadj` (signals & stops).** Gap-free *arithmetic* (additive) rolls shift historical prices to align with the new contract, preserving *absolute* daily point moves. Use this for indicators, signals, and stop-losses to avoid false triggers on rollover gaps.
|
|
328
|
-
- **`unadj` (position sizing).** Back-adjustment shifts historical prices (sometimes negative), so you can't use it for dollar values. Use `unadj` (raw, real-life prices) for that day to compute true dollar risk and contract counts.
|
|
329
|
-
- **`propadj` (proportional / ratio adjustment — strictly positive).** Derived on read from `unadj` + `backadj`; preserves daily *percentage* returns and never goes non-positive. Use it for **low-priced, long-history contracts where additive back-adjustment accumulates roll gaps below zero** and breaks price-based stops and R-multiples. See *Class III Milk (DC)* below.
|
|
330
|
-
|
|
331
|
-
#### Why `propadj` exists — Class III Milk (DC)
|
|
332
|
-
|
|
333
|
-
Norgate publishes continuous futures in only two forms: unadjusted and **additive** back-adjusted (`_CCB`) — there is no native ratio-adjusted series. Additive adjustment subtracts each roll's calendar spread from all prior history, and for a low-priced, seasonal, ~29-year contract like **DC (Class III Milk, ~$15–20/cwt)** those gaps accumulate past zero: **46.7% of `DC_backadj` closes are ≤ 0** (range −9.83 to 23.09). A price-based stop, an R-multiple, or a percentage return is meaningless on a non-positive series, so CMR cannot use DC's `backadj` at all — even though DC is the flagship *new-asset-class* (Dairy) held-out generalization market.
|
|
334
|
-
|
|
335
|
-
`propadj` salvages it. Because the additive series `B` and unadjusted series `U` differ by an offset `O = B − U` that steps only at rolls, each roll's calendar spread is recoverable (`s = O[r−1] − O[r]`) and convertible to a multiplicative roll ratio `k = (U[r−1] + s)/U[r−1]`. Scaling each historical segment by the cumulative product of `k` (most-recent segment anchored to actual prices) yields a series that is **strictly positive over the full 1997–2026 history** (DC range 4.68–25.01), preserves within-segment percentage returns exactly, and is sign-identical to `backadj` on every day including rolls. It is a pure function of two already-stored series, so it needs no producer re-run — `get_prices("DC", adjustment="propadj")` works today. Recommendation: **CMR reads DC (and any similarly low-priced contract) with `adjustment="propadj"`.** Restricting DC to its positive-price era (2011→present, ~15y) or dropping it were the fallbacks; neither is needed.
|
|
336
|
-
|
|
337
|
-
### Providers & authentication
|
|
338
|
-
|
|
339
|
-
Which vendor prices a symbol is a deployment choice, not a fixed fact (the same ES is Norgate for local research and databento on a public-dash server). It is resolved when a producer runs, from three inputs: the deployment default `COTDATA_PRICE_SOURCE` (`norgate` if unset), per-symbol capability (the `norgate` / `databento` / `yahoo` mappings in the registry, `null` where a vendor has no series), and an optional per-symbol `price_source` override. A symbol uses its override if set, otherwise the default when that vendor can serve it, otherwise a Yahoo fallback where a ticker exists. Each producer writes only the symbols that resolve to it, so a symbol is never blended across vendors. One provider owns each symbol end to end.
|
|
334
|
+
### Price adjustment tiers
|
|
340
335
|
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
336
|
+
Not here. Futures roll, and stitching contracts creates artificial gaps, so bars come in
|
|
337
|
+
three tiers — `backadj` for signals and stops, `unadj` for position sizing, `propadj` for
|
|
338
|
+
anything measuring percent returns. All three, and the reasoning behind them (including
|
|
339
|
+
why `propadj` is not optional: additive back-adjustment drives 46.7% of Class III Milk's
|
|
340
|
+
closes below zero), live in
|
|
341
|
+
[crucible-marketdata's README](https://github.com/mspinola/marketdata).
|
|
344
342
|
|
|
345
343
|
### The symbol registry
|
|
346
344
|
|
|
@@ -351,7 +349,7 @@ The supported futures contracts are defined in a YAML registry, so adding a mark
|
|
|
351
349
|
|
|
352
350
|
### Atomic store
|
|
353
351
|
|
|
354
|
-
The store uses **atomic writes** (write-temp-then-rename). Consumers can safely query via `
|
|
352
|
+
The store uses **atomic writes** (write-temp-then-rename). Consumers can safely query via `get_cot` even while `cotdata-update` is actively downloading and writing.
|
|
355
353
|
|
|
356
354
|
### COT vintage tracking (as-published history)
|
|
357
355
|
|
|
@@ -374,7 +372,8 @@ cotdata-vintage fetch --all # every year 1986-present (see below)
|
|
|
374
372
|
cotdata-vintage ingest --pending # parse retained raw -> observations + revisions
|
|
375
373
|
cotdata-vintage diff --since 2026-01-01 # field-level revisions, with revision depth
|
|
376
374
|
cotdata-vintage asof --as-of 2026-07-24T18:00:00 --report-date 2026-07-21
|
|
377
|
-
cotdata-vintage
|
|
375
|
+
cotdata-vintage zero-sum --market 088691 # long/short/open-interest identity per week
|
|
376
|
+
cotdata-vintage coverage # which markets a report covers, per year
|
|
378
377
|
cotdata-schedule sync # CFTC Special Announcements
|
|
379
378
|
cotdata-schedule published # true publication dates from retained weekly statics
|
|
380
379
|
cotdata-schedule backfill # resolve release_date + its provenance
|
|
@@ -385,11 +384,17 @@ How it works:
|
|
|
385
384
|
- **Immutable landing zone.** Every fetch is recorded (including 304s) and raw bytes are
|
|
386
385
|
retained permanently under `vintage/raw/`, written atomically and never rewritten. A
|
|
387
386
|
byte-identical regeneration is deduped — a changed download is not itself a revision.
|
|
388
|
-
- **All
|
|
389
|
-
Disagg and TFF also populate per-category spreading, per-category trader
|
|
390
|
-
CR4/CR8 concentration, none of which the Legacy file carries, and they are
|
|
391
|
-
**Managed Money** and **Leveraged Funds** live. A suppressed trader count (CFTC
|
|
392
|
-
`.`) canonicalises to null rather than to a string.
|
|
387
|
+
- **All four reports.** Legacy, Disaggregated, TFF and Supplemental canonicalise into one
|
|
388
|
+
long schema. Disagg and TFF also populate per-category spreading, per-category trader
|
|
389
|
+
counts and CR4/CR8 concentration, none of which the Legacy file carries, and they are
|
|
390
|
+
where **Managed Money** and **Leveraged Funds** live. A suppressed trader count (CFTC
|
|
391
|
+
writes `.`) canonicalises to null rather than to a string. Supplemental is where **Index
|
|
392
|
+
Traders** live, and it is the one report where `combined` is `True`: the category
|
|
393
|
+
vocabulary is checked per `report_type`, so its `commercial` (which is net of index
|
|
394
|
+
traders) can never be mistaken for Legacy's.
|
|
395
|
+
- **Coverage is not constant, and `cotdata-vintage coverage` says so.** A report's market
|
|
396
|
+
set is derived from the stored observations and every entry or exit is printed. The live
|
|
397
|
+
case is Supplemental, which covered 12 markets until Soybean Meal entered in 2013.
|
|
393
398
|
- **Change-only observations.** A row is written only when its value hash differs from the
|
|
394
399
|
latest for its natural key `(report_date, market_code, report_type, combined, category)`,
|
|
395
400
|
so storage grows with actual revisions rather than with time.
|
|
@@ -448,56 +453,30 @@ export COTDATA_STORE=/path/to/synced/store # the shared store
|
|
|
448
453
|
uv run pytest # run the tests
|
|
449
454
|
```
|
|
450
455
|
|
|
451
|
-
|
|
456
|
+
There are no optional data extras — every vendor SDK moved out with the producers. Use `uv run <cmd>`, or activate with `source .venv/bin/activate` (Mac/Linux) / `.venv\Scripts\activate` (Windows).
|
|
452
457
|
|
|
453
458
|
## Reference: Data schemas
|
|
454
459
|
|
|
455
460
|
The canonical store uses standard Parquet files. Loaded with `pd.read_parquet()`, they conform to the following schemas.
|
|
456
461
|
|
|
457
|
-
### Price Data
|
|
458
|
-
Primary price history (Norgate Data), indexed by tz-naive `Date`. The pipeline downloads both the back-adjusted (`backadj`) series for signals/stops and the unadjusted (`unadj`) series for true transaction-cost modeling.
|
|
462
|
+
### Price Data
|
|
459
463
|
|
|
460
|
-
|
|
464
|
+
Not in this store. Every bar, and the schema describing it, moved to
|
|
465
|
+
[crucible-marketdata](https://pypi.org/project/crucible-marketdata/) under ADR-0007.
|
|
466
|
+
A store built before that move still has a `prices/` directory: it is history, nothing
|
|
467
|
+
here reads or writes it, and `--reconcile` will keep its manifest entries honest.
|
|
461
468
|
|
|
462
|
-
|
|
469
|
+
### Contract Specifications
|
|
463
470
|
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
| `Open` | float | Opening price. |
|
|
468
|
-
| `High` | float | High price. |
|
|
469
|
-
| `Low` | float | Low price. |
|
|
470
|
-
| `Close` | float | Settlement Close price. |
|
|
471
|
-
| `Volume` | float | Continuous contract trading volume (front-month only). |
|
|
472
|
-
| `Open Interest` | float | Continuous contract open interest. |
|
|
473
|
-
| `Volume_Reconstructed` | float | True market volume (sum of First and Second contract). Differs from raw `Volume` by symbol — typically higher for products whose rolls spread volume across contracts, but roughly equal or lower for symbols with a near-empty back month (e.g. crypto). Not a drop-in replacement. |
|
|
474
|
-
| `Volume_Source` | string | `reconstructed` if First+Second available, `raw` fallback if not. |
|
|
475
|
-
| `FirstVolume` / `SecondVolume` | float | Trading volume of the specific first and second expiring contracts. |
|
|
476
|
-
| `FirstContract` / `SecondContract` | string | Contract names for the first and second expirations (e.g., `ES-2024H`). |
|
|
477
|
-
| `Delivery Month` | float | Expiration month of the active contract (e.g. `202609`). Used to detect contract rolls. |
|
|
478
|
-
|
|
479
|
-
### Contract Specifications (`metadata/contract_specs.parquet`)
|
|
480
|
-
Contract metadata (Norgate Data), used for exact point-value risk sizing and transaction cost models.
|
|
481
|
-
|
|
482
|
-
| Column | Type | Description |
|
|
483
|
-
|--------|------|-------------|
|
|
484
|
-
| `Symbol` | string | Internal ticker symbol (e.g., `ES`). |
|
|
485
|
-
| `Norgate_Symbol` | string | Raw Norgate symbol used to query the API (e.g., `&ES_CCB`). |
|
|
486
|
-
| `Name` | string | Full name of the contract. |
|
|
487
|
-
| `Exchange` | string | Name of the listing exchange. |
|
|
488
|
-
| `Group` | string | Norgate asset classification group. |
|
|
489
|
-
| `Contract Size` | float | Size multiplier (e.g., $50 for ES). Also called Point Value. |
|
|
490
|
-
| `Tick Size` | float | Minimum price fluctuation (e.g., 0.25 for ES). |
|
|
491
|
-
| `Tick Value` | float | Dollar value of one tick (`Tick Size` * `Contract Size`). |
|
|
492
|
-
| `Point Value` | float | Same as `Contract Size`. |
|
|
493
|
-
| `Currency` | string | Base currency of the contract. |
|
|
494
|
-
| `Margin` | float | Initial margin requirement (if provided by Norgate). |
|
|
471
|
+
Moved to `marketdata`'s store (`metadata/contract_specs.parquet` under
|
|
472
|
+
`$MARKETDATA_STORE`) with the Norgate producer that writes them — read them with
|
|
473
|
+
`marketdata.read_metadata()`. Nothing in this package writes contract specs any more.
|
|
495
474
|
|
|
496
475
|
### COT Legacy Data (`cot_legacy/{symbol}_{code}.parquet`)
|
|
497
476
|
Legacy positioning data (CFTC Legacy Futures Report). **History starts in 1986.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
|
|
498
477
|
|
|
499
478
|
> [!NOTE]
|
|
500
|
-
> **Legacy Reports**: broken down by exchange, with futures-only and combined futures-and-options variants. Legacy classifies reportable open interest into non-commercial and commercial traders.
|
|
479
|
+
> **Legacy Reports**: broken down by exchange, with futures-only and combined futures-and-options variants. Legacy classifies reportable open interest into non-commercial and commercial traders. This directory holds the **futures-only** files (`dea_fut_xls_{YEAR}.zip`); the combined files (`dea_com_xls_{YEAR}.zip`, from 1995) land in `cot_legacy_combined/` with the same columns.
|
|
501
480
|
|
|
502
481
|
> [!NOTE]
|
|
503
482
|
> **Column Subset**: The raw CFTC `.xls` files contain [well over 100 columns](https://www.cftc.gov/MarketReports/CommitmentsofTraders/HistoricalViewable/cotvariableslegacy.html); the pipeline keeps the focused 15-column subset below to keep files small. To include more, add the exact CFTC column name to `TARGET_COLS` in `src/cotdata/providers/cftc.py`.
|
|
@@ -535,23 +514,41 @@ Entity-specific positioning and trader counts for financial markets (CFTC TFF Fu
|
|
|
535
514
|
> [!NOTE]
|
|
536
515
|
> **Lossless Image**: Like Disaggregated, TFF parquets are a **lossless image** of the source CFTC `txt` files — the financial entity groups (`Dealer`, `Asset_Mgr`, `Lev_Money`, `Other_Rept`) and their `Traders_*` counts.
|
|
537
516
|
|
|
517
|
+
### COT combined-basis data (`cot_legacy_combined/`, `cot_disagg_combined/`, `cot_tff_combined/`)
|
|
518
|
+
The three reports above on the CFTC's **futures-and-options combined** basis (`dea_com_xls_*`, `com_disagg_txt_*`, `com_fin_txt_*`). Same parsers, same columns, same per-code files; Legacy combined starts in **1995**, the other two in 2006. The Disaggregated and TFF files state their basis in `FutOnly_or_Combined` and the producer refuses a file whose statement disagrees with the directory it is writing to; the Legacy files carry no such column, so for them the URL is the only evidence and the two bases are kept apart by construction (separate zip names, caches and writers behind one flag). Measured on 2026-10-27: ES **2,843,639 combined / 1,955,711 futures-only**, GC 518,784 / 396,810, CL 2,606,478 / 1,886,280. See [ADR-0003](docs/adr/ADR-0003-combined-basis-is-in-scope.md).
|
|
519
|
+
|
|
520
|
+
### COT Supplemental Data (`cot_supplemental/{symbol}_{code}.parquet`)
|
|
521
|
+
Index-trader positioning for 13 select agricultural markets (CFTC Supplemental / Commodity Index Trader Report). **History starts in January 2006**, and the covered set was **12 markets until Soybean Meal entered on 2013-04-02**. Indexed by tz-naive `Report_Date_as_MM_DD_YYYY` (renamed at parse from CFTC's `As_of_Date_In_Form_*`, whose own name changed in 2013).
|
|
522
|
+
|
|
523
|
+
> [!WARNING]
|
|
524
|
+
> **Combined basis, and the file does not say so.** This is the only report with no futures-only variant, and unlike Disaggregated and TFF it carries no `FutOnly_or_Combined` column. Verified by matching open interest against both Legacy series: 390/390 against futures-and-options combined, 0/390 against futures-only. Its open interest is therefore **not** comparable with the other three for the same market and week.
|
|
525
|
+
|
|
526
|
+
> [!NOTE]
|
|
527
|
+
> **Positions are net of index traders.** `Comm_Positions_Long_All_NoCIT` is commercial *minus* the index book, not Legacy's commercial. The index book is carved out of commercial, non-commercial **and non-commercial spreading** — three buckets, where CFTC's own prose names two. Non-Reportable is untouched, because index traders are reportable by definition.
|
|
528
|
+
|
|
529
|
+
> [!TIP]
|
|
530
|
+
> **It joins to Legacy combined, exactly.** The `_NoCIT` buckets are `cot_legacy_combined/`'s buckets with the index traders removed, for the same market and week, and open interest and non-reportable agree to the contract on all 13,753 shared weeks 2006-2026. `cotdata.cit.decompose(name)` makes the join and splits each Legacy bucket into its ex-index part and the index position carved from it (spreading as the residual, since the store does not keep Legacy's spread column); it refuses a market where the exact identities fail, which is a basis or vintage error. Measurements, including how much of Legacy's "commercial" is index money per market: [docs/analysis/2026-10-31-cit-legacy-combined-decomposition.md](docs/analysis/2026-10-31-cit-legacy-combined-decomposition.md).
|
|
531
|
+
|
|
532
|
+
> [!NOTE]
|
|
533
|
+
> **Lossless Image**: Like Disaggregated and TFF, these parquets are a lossless image of the source CFTC `txt` files, including CFTC's own header typos (`NComm_Postions_Spread_All_NoCIT`).
|
|
534
|
+
|
|
538
535
|
## Reference: COT formats explained
|
|
539
536
|
|
|
540
|
-
The CFTC publishes positioning data in three
|
|
537
|
+
The CFTC publishes positioning data in four formats, the first three of them on two bases (futures-only and futures-and-options combined); `cotdata` manages all seven series for complete coverage and the deepest history.
|
|
541
538
|
|
|
542
539
|
1. **Legacy (1986–Present)** — *all markets.* Divides traders into **Commercial** (hedgers) and **Non-Commercial** (large speculators). The only format with pre-2006 data, so it's essential for long-term backtesting.
|
|
543
540
|
2. **Disaggregated / DIS (2006–Present)** — *physical commodities only* (Agriculture, Energy, Metals). Splits traders into **Producer/Merchant**, **Swap Dealers**, **Managed Money**, and **Other Reportables** — a clearer view of "smart money" (Managed Money) in commodities.
|
|
544
541
|
3. **Traders in Financial Futures / TFF (2006–Present)** — *financial markets only* (Equities, Rates, Currencies). Splits traders into **Dealer/Intermediary**, **Asset Manager**, **Leveraged Funds**, and **Other Reportables** — the definitive source for speculative flow (Leveraged Funds) in financials.
|
|
542
|
+
4. **Supplemental / CIT (2006–Present)** — *13 select agricultural markets only.* Takes the Legacy split and carves **Index Traders** out of it, the only public source that separates index flow. Futures-and-options **combined** only. The taxonomy is Legacy, not Disaggregated, so Index Traders does **not** nest inside Disaggregated's Swap Dealer and the two cannot be differenced to isolate levered swap flow.
|
|
545
543
|
|
|
546
544
|
## Diagnostics
|
|
547
545
|
|
|
548
|
-
Verify your Norgate subscription and configuration with the included smoke test, on the Windows producer:
|
|
549
|
-
|
|
550
546
|
```bash
|
|
551
|
-
|
|
547
|
+
cotdata-update --check # coverage, newest dates, staleness — no network
|
|
548
|
+
python scripts/vintage_alert_selftest.py # the vintage capture's alerting path
|
|
552
549
|
```
|
|
553
550
|
|
|
554
|
-
|
|
551
|
+
Norgate subscription and roll-gap diagnostics moved to `marketdata` with the provider.
|
|
555
552
|
|
|
556
553
|
## Ecosystem
|
|
557
554
|
|
|
@@ -559,8 +556,12 @@ cotdata is the *data* layer of a small, unbundled toolchain — it stops at "cle
|
|
|
559
556
|
data behind a stable API" on purpose. What you do with that data is a separate,
|
|
560
557
|
swappable step:
|
|
561
558
|
|
|
562
|
-
- **cotdata** *(this package)* —
|
|
563
|
-
|
|
559
|
+
- **cotdata** *(this package)* — CFTC COT positioning. One synced store, many
|
|
560
|
+
readers, no vendor SDK at read time.
|
|
561
|
+
- **[crucible-marketdata](https://pypi.org/project/crucible-marketdata/)** — the
|
|
562
|
+
daily bars that used to live here: Norgate futures, Yahoo equities/ETFs, the
|
|
563
|
+
adjustment tiers, contract specs. Split out by ADR-0007 so a positioning question
|
|
564
|
+
and a price question have separate stores, producers and schedules.
|
|
564
565
|
- **[crucible](https://github.com/mspinola/crucible)** — the *edge* layer. Feed a
|
|
565
566
|
signal built on cotdata frames into crucible and it tells you — with a
|
|
566
567
|
confidence interval and a p-value — whether the trade-level edge is real, before
|
|
@@ -575,12 +576,12 @@ COT/futures research — crucible is just the most common thing to point at it n
|
|
|
575
576
|
Want to contribute or work on cotdata locally? See [CONTRIBUTING.md](CONTRIBUTING.md) for:
|
|
576
577
|
- Virtual environment setup with `uv` or standard `pip`
|
|
577
578
|
- Running the test suite
|
|
578
|
-
- Platform-specific notes (
|
|
579
|
+
- Platform-specific notes (everything here runs anywhere — no vendor SDK)
|
|
579
580
|
- Code style guidelines
|
|
580
581
|
|
|
581
582
|
## Contributing
|
|
582
583
|
|
|
583
|
-
Issues and pull requests are welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, tests, and conventions. When filing a bug, include your OS
|
|
584
|
+
Issues and pull requests are welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, tests, and conventions. When filing a bug, include your OS. If it is about price bars, it probably belongs on [crucible-marketdata](https://github.com/mspinola/marketdata) instead.
|
|
584
585
|
|
|
585
586
|
## License
|
|
586
587
|
|