cotdata 0.1.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cotdata-0.5.0/PKG-INFO +588 -0
- cotdata-0.5.0/README.md +563 -0
- {cotdata-0.1.0 → cotdata-0.5.0}/pyproject.toml +21 -9
- cotdata-0.5.0/src/cotdata/__init__.py +18 -0
- cotdata-0.5.0/src/cotdata/cit.py +177 -0
- cotdata-0.5.0/src/cotdata/config.py +95 -0
- cotdata-0.5.0/src/cotdata/cot.py +228 -0
- cotdata-0.5.0/src/cotdata/providers/cftc.py +189 -0
- cotdata-0.5.0/src/cotdata/providers/cftc_cit.py +194 -0
- cotdata-0.5.0/src/cotdata/providers/cftc_disagg.py +198 -0
- cotdata-0.5.0/src/cotdata/providers/cftc_tff.py +198 -0
- {cotdata-0.1.0 → cotdata-0.5.0}/src/cotdata/registry.py +54 -6
- cotdata-0.5.0/src/cotdata/registry.yaml +218 -0
- cotdata-0.5.0/src/cotdata/status.py +266 -0
- cotdata-0.5.0/src/cotdata/store.py +375 -0
- cotdata-0.5.0/src/cotdata/update.py +185 -0
- cotdata-0.5.0/src/cotdata/vintage.py +808 -0
- cotdata-0.5.0/src/cotdata/vintage_cli.py +521 -0
- cotdata-0.5.0/src/cotdata/vintage_flow.py +144 -0
- cotdata-0.5.0/src/cotdata/vintage_ingest.py +1031 -0
- cotdata-0.5.0/src/cotdata/vintage_schedule.py +629 -0
- cotdata-0.5.0/src/cotdata.egg-info/PKG-INFO +588 -0
- cotdata-0.5.0/src/cotdata.egg-info/SOURCES.txt +51 -0
- cotdata-0.5.0/src/cotdata.egg-info/entry_points.txt +5 -0
- {cotdata-0.1.0 → cotdata-0.5.0}/src/cotdata.egg-info/requires.txt +1 -7
- {cotdata-0.1.0 → cotdata-0.5.0}/tests/test_cftc_disagg_fidelity.py +7 -5
- {cotdata-0.1.0 → cotdata-0.5.0}/tests/test_cftc_reconciliation.py +16 -15
- {cotdata-0.1.0 → cotdata-0.5.0}/tests/test_cftc_tff_reconciliation.py +19 -18
- cotdata-0.5.0/tests/test_cit_decomposition.py +154 -0
- cotdata-0.5.0/tests/test_cit_supplemental.py +507 -0
- cotdata-0.5.0/tests/test_cli_exit.py +45 -0
- cotdata-0.5.0/tests/test_cot.py +81 -0
- cotdata-0.5.0/tests/test_cot_combined_basis.py +353 -0
- cotdata-0.5.0/tests/test_cot_stitch.py +87 -0
- cotdata-0.5.0/tests/test_cot_sum.py +133 -0
- cotdata-0.5.0/tests/test_current_baseline.py +96 -0
- cotdata-0.5.0/tests/test_manifest_seam.py +296 -0
- {cotdata-0.1.0 → cotdata-0.5.0}/tests/test_registry.py +76 -8
- cotdata-0.5.0/tests/test_status.py +214 -0
- cotdata-0.5.0/tests/test_store.py +123 -0
- cotdata-0.5.0/tests/test_sync_preflight.py +179 -0
- cotdata-0.5.0/tests/test_verify_replicas.py +156 -0
- cotdata-0.5.0/tests/test_vintage_canonicalize.py +261 -0
- cotdata-0.5.0/tests/test_vintage_capture.py +663 -0
- cotdata-0.5.0/tests/test_vintage_flow.py +87 -0
- cotdata-0.5.0/tests/test_vintage_ingest.py +453 -0
- cotdata-0.5.0/tests/test_vintage_schedule.py +451 -0
- cotdata-0.1.0/PKG-INFO +0 -246
- cotdata-0.1.0/README.md +0 -217
- cotdata-0.1.0/src/cotdata/__init__.py +0 -11
- cotdata-0.1.0/src/cotdata/config.py +0 -35
- cotdata-0.1.0/src/cotdata/cot.py +0 -62
- cotdata-0.1.0/src/cotdata/prices.py +0 -43
- cotdata-0.1.0/src/cotdata/providers/cftc.py +0 -129
- cotdata-0.1.0/src/cotdata/providers/cftc_disagg.py +0 -146
- cotdata-0.1.0/src/cotdata/providers/cftc_tff.py +0 -146
- cotdata-0.1.0/src/cotdata/providers/databento.py +0 -280
- cotdata-0.1.0/src/cotdata/providers/norgate.py +0 -78
- cotdata-0.1.0/src/cotdata/registry.yaml +0 -105
- cotdata-0.1.0/src/cotdata/store.py +0 -96
- cotdata-0.1.0/src/cotdata/update.py +0 -42
- cotdata-0.1.0/src/cotdata.egg-info/PKG-INFO +0 -246
- cotdata-0.1.0/src/cotdata.egg-info/SOURCES.txt +0 -30
- cotdata-0.1.0/src/cotdata.egg-info/entry_points.txt +0 -2
- cotdata-0.1.0/tests/test_adjustment.py +0 -111
- cotdata-0.1.0/tests/test_store.py +0 -48
- {cotdata-0.1.0 → cotdata-0.5.0}/LICENSE +0 -0
- {cotdata-0.1.0 → cotdata-0.5.0}/setup.cfg +0 -0
- {cotdata-0.1.0 → cotdata-0.5.0}/src/cotdata/providers/__init__.py +0 -0
- {cotdata-0.1.0 → cotdata-0.5.0}/src/cotdata/providers/base.py +0 -0
- {cotdata-0.1.0 → cotdata-0.5.0}/src/cotdata.egg-info/dependency_links.txt +0 -0
- {cotdata-0.1.0 → cotdata-0.5.0}/src/cotdata.egg-info/top_level.txt +0 -0
cotdata-0.5.0/PKG-INFO
ADDED
|
@@ -0,0 +1,588 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cotdata
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Canonical CFTC positioning (COT) data layer — a producer/consumer split over a file-based store.
|
|
5
|
+
Author-email: "Matt S." <cotanalyzer@gmail.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/mspinola/cotdata
|
|
7
|
+
Project-URL: Repository, https://github.com/mspinola/cotdata
|
|
8
|
+
Project-URL: Issues, https://github.com/mspinola/cotdata/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: pandas>=1.5
|
|
16
|
+
Requires-Dist: pyarrow>=10
|
|
17
|
+
Requires-Dist: requests>=2.28
|
|
18
|
+
Requires-Dist: python-dateutil>=2.8
|
|
19
|
+
Requires-Dist: xlrd>=2.0
|
|
20
|
+
Requires-Dist: PyYAML>=6.0
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
23
|
+
Requires-Dist: ruff==0.15.22; extra == "dev"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# cotdata
|
|
27
|
+
|
|
28
|
+
[](https://github.com/mspinola/cotdata/actions/workflows/python-test.yml)
|
|
29
|
+
[](https://pypi.org/project/cotdata/)
|
|
30
|
+
[](https://pypi.org/project/cotdata/)
|
|
31
|
+
[](https://opensource.org/licenses/MIT)
|
|
32
|
+
|
|
33
|
+
**A local, file-based data layer for CFTC Commitments of Traders (COT) positioning.**
|
|
34
|
+
|
|
35
|
+
cotdata separates *fetching* data (a "producer" that talks to vendors) from *using* it (any number of "consumers" that just read Parquet through a small, stable API). Point every tool at one synced store, and none of them ever call a vendor SDK at runtime — so the same data feeds your research, backtests, and dashboards identically, on any OS.
|
|
36
|
+
|
|
37
|
+
- **One store, many readers.** Consumers `import cotdata` and read; they never touch a vendor SDK. Swapping a data vendor is a producer-only change.
|
|
38
|
+
- **Free, on any OS.** CFTC Commitments of Traders data (1986–present) downloads free from cftc.gov. No account, no subscription.
|
|
39
|
+
- **Predecessor stitching.** `get_cot()` transparently stitches migrated CFTC codes (e.g. the Russell 2000) and rescales tick-size changes (e.g. Lumber) into one continuous series.
|
|
40
|
+
- **Atomic writes.** Read the store safely even while the producer is downloading and writing.
|
|
41
|
+
- **New-data signal.** Every run writes a structured `status.json` so downstream tools can poll one file to detect fresh data.
|
|
42
|
+
|
|
43
|
+
> [!IMPORTANT]
|
|
44
|
+
> **Price bars are not in this package.** Under [ADR-0007](#ecosystem), cotdata is CFTC
|
|
45
|
+
> positioning only, and every daily bar — Norgate futures, databento futures, Yahoo
|
|
46
|
+
> equities/ETFs, the `backadj`/`unadj`/`propadj` tiers, contract specifications — lives in
|
|
47
|
+
> [**crucible-marketdata**](https://pypi.org/project/crucible-marketdata/). Replace
|
|
48
|
+
> `cotdata.get_prices(sym, adjustment)` with `marketdata.get_bars(sym, tier)` and point
|
|
49
|
+
> `MARKETDATA_STORE` at that store. The two packages keep separate stores and separate
|
|
50
|
+
> producers; nothing about `get_cot()` changed.
|
|
51
|
+
|
|
52
|
+
## Data sources at a glance
|
|
53
|
+
|
|
54
|
+
| Data | Source | Cost | Runs on |
|
|
55
|
+
|------|--------|------|---------|
|
|
56
|
+
| CFTC COT — legacy / disaggregated / TFF / supplemental | [cftc.gov](https://www.cftc.gov/) | **Free** | any OS |
|
|
57
|
+
| *Reading the store* | — | Free | any OS |
|
|
58
|
+
| Daily bars + contract specs | → [crucible-marketdata](https://pypi.org/project/crucible-marketdata/) | — | — |
|
|
59
|
+
|
|
60
|
+
No paid dependency, and no optional data extra: this package downloads public CFTC
|
|
61
|
+
files over HTTP and reads its own store.
|
|
62
|
+
|
|
63
|
+
## Contents
|
|
64
|
+
|
|
65
|
+
- [Quickstart](#quickstart) · [How it works](#how-it-works) · [Reading data](#reading-data-consumer) · [Producing data](#producing-data-producer) · [Windows setup](docs/WINDOWS_SETUP.md) · [Scheduling on Windows](docs/WINDOWS_SCHEDULING.md) · [Scheduling on Linux](docs/LINUX_SCHEDULING.md) · [Syncing the store](docs/SYNCING.md) · [Operations](#operations) · [Concepts & design](#concepts--design) · [COT vintage tracking](#cot-vintage-tracking-as-published-history) · [Reference: schemas](#reference-data-schemas) · [Reference: COT formats](#reference-cot-formats-explained) · [Diagnostics](#diagnostics) · [Development](#development) · [Contributing](#contributing) · [License](#license)
|
|
66
|
+
|
|
67
|
+
## Quickstart
|
|
68
|
+
|
|
69
|
+
The fastest zero-cost path uses free CFTC COT data — no account, any OS:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pip install cotdata
|
|
73
|
+
export COTDATA_STORE=~/cotdata_store # where the shared store lives
|
|
74
|
+
cotdata-update --cot-legacy # free CFTC download (first run pulls history; cached after)
|
|
75
|
+
python -c "import cotdata; print(cotdata.get_cot('ES').tail())"
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
That downloads the CFTC Legacy COT history and reads the S&P 500 (ES) positioning back out:
|
|
79
|
+
|
|
80
|
+
```
|
|
81
|
+
Open_Interest_All Comm_Positions_Long_All Comm_Positions_Short_All NonComm_Positions_Long_All NonComm_Positions_Short_All
|
|
82
|
+
Report_Date_as_MM_DD_YYYY
|
|
83
|
+
2026-06-23 1980254 1444102 1531232 251385 286833
|
|
84
|
+
2026-06-30 1967167 1422155 1509889 249934 287526
|
|
85
|
+
2026-07-07 1969636 1435736 1502199 244103 286994
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
For daily **bars**, install [crucible-marketdata](https://pypi.org/project/crucible-marketdata/) alongside this and read `marketdata.get_bars("ES", "backadj")`.
|
|
89
|
+
|
|
90
|
+
## How it works
|
|
91
|
+
|
|
92
|
+
The **store is the API boundary** — not Python imports. Producers write Parquet + `manifest.json`; consumers only read. Nobody touches a vendor SDK at app runtime, so swapping a vendor is a producer-only change.
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
PRODUCER — runs where each source is reachable
|
|
96
|
+
CFTC COT download — free, any OS, no vendor SDK
|
|
97
|
+
│
|
|
98
|
+
▼ write parquet + manifest
|
|
99
|
+
┌────────────────────────────────────────────────────────────┐
|
|
100
|
+
│ CANONICAL STORE ($COTDATA_STORE) │
|
|
101
|
+
│ cot_legacy/ cot_disagg/ cot_tff/ cot_supplemental/ │
|
|
102
|
+
│ cot_legacy_combined/ cot_disagg_combined/ │
|
|
103
|
+
│ cot_tff_combined/ │
|
|
104
|
+
│ manifests/ status.json │
|
|
105
|
+
└────────────────────────────────────────────────────────────┘
|
|
106
|
+
│ read (offline, any OS)
|
|
107
|
+
┌──────────────┴───────────────┐
|
|
108
|
+
▼ ▼
|
|
109
|
+
your signal research your backtest / dashboards
|
|
110
|
+
|
|
111
|
+
both just: import cotdata · store synced via rsync / Dropbox / S3
|
|
112
|
+
|
|
113
|
+
Daily bars live in a SEPARATE store: $MARKETDATA_STORE, read with
|
|
114
|
+
`import marketdata`. Two stores, two producers, one seam (ADR-0007).
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
The store layout:
|
|
118
|
+
|
|
119
|
+
- `cot_legacy/{symbol}_{code}.parquet` — weekly CFTC Legacy positioning.
|
|
120
|
+
- `cot_disagg/{symbol}_{code}.parquet` — weekly CFTC Disaggregated positioning.
|
|
121
|
+
- `cot_tff/{symbol}_{code}.parquet` — weekly CFTC Traders in Financial Futures positioning.
|
|
122
|
+
- `cot_supplemental/{symbol}_{code}.parquet` — weekly CFTC Supplemental (Commodity Index Trader) positioning, 13 agricultural markets. **Futures-and-options combined**; the CFTC publishes no futures-only variant.
|
|
123
|
+
- `cot_legacy_combined/`, `cot_disagg_combined/`, `cot_tff_combined/` — the same three reports as the first three rows, on the **futures-and-options combined** basis (Legacy from 1995, the other two from 2006). Same columns, same `{symbol}_{code}.parquet` naming, a different quantity: combined open interest counts delta-weighted option equivalents, so on 2026-10-27 ES is 2,843,639 combined against 1,955,711 futures-only. The two bases never share a directory. Read with `get_cot(..., combined=True)`.
|
|
124
|
+
- `manifests/{cot,prices}.json` — per-table `last_date`, `n_rows`, `source`, `updated_at`, `schema_version`, one file per producer half.
|
|
125
|
+
- `status.json` — machine-readable new-data signal for downstream tools (see [Operations](#operations)).
|
|
126
|
+
- `vintage/` — optional as-published (vintage) capture: retained raw CFTC downloads plus
|
|
127
|
+
change-only observations and field-level revisions. Purely additive; the tables above are
|
|
128
|
+
unchanged whether or not it is enabled. See [COT vintage tracking](#cot-vintage-tracking-as-published-history).
|
|
129
|
+
|
|
130
|
+
## Reading data (consumer)
|
|
131
|
+
|
|
132
|
+
Set `COTDATA_STORE` to the synced store directory, then:
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
import cotdata
|
|
136
|
+
|
|
137
|
+
# COT — four CFTC report families, futures-only by default:
|
|
138
|
+
legacy = cotdata.get_cot("ES", report="legacy") # Commercial / Non-Commercial
|
|
139
|
+
disagg = cotdata.get_cot("ES", report="disagg") # Managed Money, Swap Dealers, ... (commodities)
|
|
140
|
+
tff = cotdata.get_cot("ES", report="tff") # Leveraged Funds, Asset Managers, ... (financials)
|
|
141
|
+
cit = cotdata.get_cot("CC", report="supplemental") # Index Traders (13 ags, combined basis)
|
|
142
|
+
|
|
143
|
+
# ...and the futures-and-options COMBINED basis of the first three:
|
|
144
|
+
legacy_fo = cotdata.get_cot("ES", report="legacy", combined=True)
|
|
145
|
+
tff_fo = cotdata.get_cot("ES", report="tff", combined=True)
|
|
146
|
+
|
|
147
|
+
# ...and, for the 13 Supplemental markets, Legacy combined joined to the Supplemental
|
|
148
|
+
# week by week, so each Legacy bucket splits into "ex index" and "index traders from it":
|
|
149
|
+
from cotdata import cit
|
|
150
|
+
parts = cit.decompose("ZW") # comm_long, comm_long_nocit, cit_from_comm_long, ... (cotdata/cit.py)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
> [!WARNING]
|
|
154
|
+
> `combined=None` (the default) is each report's natural basis: futures-only for `legacy`,
|
|
155
|
+
> `disagg` and `tff`, combined for `supplemental`, which is published on no other basis
|
|
156
|
+
> (`combined=False` on it raises). Combined `Open_Interest_All` is a different quantity from
|
|
157
|
+
> futures-only for the same market and week. Do not difference or ratio across bases, or
|
|
158
|
+
> across reports on different bases, without accounting for that.
|
|
159
|
+
|
|
160
|
+
Daily **bars** come from the sibling package, against its own store:
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
import marketdata # pip install crucible-marketdata
|
|
164
|
+
|
|
165
|
+
signals = marketdata.get_bars("ES", "backadj") # signals + stops (gap-free rolls)
|
|
166
|
+
sizing = marketdata.get_bars("ES", "unadj") # position sizing (true dollar prices)
|
|
167
|
+
milk = marketdata.get_bars("DC", "propadj") # ratio-adjusted, %-return preserving
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
```
|
|
171
|
+
Open High Low Close Volume Open Interest
|
|
172
|
+
Date
|
|
173
|
+
2026-07-10 7587.25 7628.75 7552.75 7620.25 1078031.0 1966297.0
|
|
174
|
+
2026-07-13 7607.00 7615.25 7547.25 7563.00 1274520.0 1945908.0
|
|
175
|
+
2026-07-14 7557.00 7613.75 7531.50 7591.25 1139735.0 0.0
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Same frames, same symbols, same tier names — the import and the environment variable
|
|
179
|
+
(`MARKETDATA_STORE`) are what changed. `cotdata.get_prices` and `cotdata.roll_dates` are
|
|
180
|
+
gone rather than deprecated: left importable they would read a store the nightly job no
|
|
181
|
+
longer fills, and stale data is harder to notice than an `AttributeError`.
|
|
182
|
+
|
|
183
|
+
**Predecessor stitching & scaling:** `get_cot()` doesn't just read a file — it stitches historical CFTC codes for contracts that migrated exchanges (e.g. the Russell 2000) and rescales data for contracts that changed tick sizes (e.g. Lumber), so downstream models see one clean, continuous asset.
|
|
184
|
+
|
|
185
|
+
## Producing data (producer)
|
|
186
|
+
|
|
187
|
+
Runs anywhere — the CFTC files are a public HTTP download. Bars of every vendor are produced by `marketdata-update` in the [sibling package](https://pypi.org/project/crucible-marketdata/).
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
COTDATA_STORE=/store cotdata-update --cot-legacy # CFTC Legacy (any OS)
|
|
191
|
+
COTDATA_STORE=/store cotdata-update --cot-disagg # CFTC Disaggregated (any OS)
|
|
192
|
+
COTDATA_STORE=/store cotdata-update --cot-tff # CFTC Traders in Financial Futures (any OS)
|
|
193
|
+
COTDATA_STORE=/store cotdata-update --cot-supplemental # CFTC Supplemental / index traders (any OS)
|
|
194
|
+
COTDATA_STORE=/store cotdata-update --cot-legacy-combined # Legacy, futures-and-options combined (from 1995)
|
|
195
|
+
COTDATA_STORE=/store cotdata-update --cot-disagg-combined # Disaggregated, combined (from 2006)
|
|
196
|
+
COTDATA_STORE=/store cotdata-update --cot-tff-combined # TFF, combined (from 2006)
|
|
197
|
+
COTDATA_STORE=/store cotdata-update --cot-all # all seven: four reports + three combined
|
|
198
|
+
COTDATA_STORE=/store cotdata-vintage fetch # optional: capture as-published COT (any OS)
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Each run prints a per-domain line and a summary footer. A run **exits non-zero** if a fetch hard-fails (CFTC or Databento unreachable), so a scheduler can retry — see [Scheduling on Linux](docs/LINUX_SCHEDULING.md).
|
|
202
|
+
|
|
203
|
+
### One producer, and the manifest split it left behind
|
|
204
|
+
|
|
205
|
+
`cotdata` used to have two producers — the CFTC downloader and a price producer — and two
|
|
206
|
+
entry points that scoped a host to one of them, so a price box could not quietly become a
|
|
207
|
+
second COT producer racing the first. Every price producer moved to `marketdata`, so there
|
|
208
|
+
is one producer here now and nothing to scope. `cotdata-cot` survives as an alias of
|
|
209
|
+
`cotdata-update` because the scheduled jobs call it by name; `cotdata-prices` is gone.
|
|
210
|
+
|
|
211
|
+
The manifest is still split per half (`manifests/cot.json`, `manifests/prices.json`), and
|
|
212
|
+
the `prices` half is now **read-only history**: a store built before the move still carries
|
|
213
|
+
those entries, and they have to keep migrating and reconciling rather than being stranded.
|
|
214
|
+
The legacy top-level `manifest.json` held both halves in ONE file, which was unsafe two
|
|
215
|
+
ways: the update is a read-modify-write, so two producers lose each other's entries, and
|
|
216
|
+
a file-level sync between two stores resolves it last-writer-wins and silently discards
|
|
217
|
+
one side. The per-half files are disjoint, so both problems go away.
|
|
218
|
+
|
|
219
|
+
**Nothing writes `manifest.json` any more.** Migrate a store once:
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
cotdata-update --migrate-manifests
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Idempotent, and it never touches data. Until you run it, a domain missing from the
|
|
226
|
+
per-half files is still read from the aggregate with a warning. Delete `manifest.json`
|
|
227
|
+
once every consumer of that store is on this version. See ADR-0007.
|
|
228
|
+
|
|
229
|
+
### Scheduling on Windows (Task Scheduler)
|
|
230
|
+
|
|
231
|
+
The COT tasks (daily catch-up, Friday release-window poller), repeating-trigger poll settings, and Task-Scheduler troubleshooting are in **[docs/WINDOWS_SCHEDULING.md](docs/WINDOWS_SCHEDULING.md)**. Start with the [Windows Setup Guide](docs/WINDOWS_SETUP.md) first if Python/the venv/`COTDATA_STORE` aren't configured yet.
|
|
232
|
+
|
|
233
|
+
The short version: COT gets a daily morning catch-up plus a tight Friday-afternoon poll around its ~3:30pm ET release, and the polling tasks use a **repeating trigger** so idempotent, cheap re-runs absorb both transient errors and "not published yet." Note that Task Scheduler's *"if the task fails, restart every N minutes"* is **not** the mechanism for this — it does not fire on a non-zero exit code from your script, only on a failure to launch it. See [Polling with a repeating trigger](docs/WINDOWS_SCHEDULING.md#polling-with-a-repeating-trigger).
|
|
234
|
+
|
|
235
|
+
The Windows box is also where the **Norgate bar** job runs, but that is now `marketdata-update --bars --domain futures --require-final` from the sibling package, on its own schedule near the Norgate Continuous Futures Final (~8:55pm ET). Both packages' docs cover their own half.
|
|
236
|
+
|
|
237
|
+
### Scheduling on Linux (cron)
|
|
238
|
+
|
|
239
|
+
Full setup, including the wrapper script, the crontab entries (daily COT catch-up plus a Friday release-window poller), `flock` overlap protection, and troubleshooting (cron's bare environment, timezone conversion), is in **[docs/LINUX_SCHEDULING.md](docs/LINUX_SCHEDULING.md)**.
|
|
240
|
+
|
|
241
|
+
The short version: COT gets a daily morning catch-up plus a tight Friday-afternoon poll around its ~3:30pm ET release, all idempotent and safe to over-run.
|
|
242
|
+
|
|
243
|
+
### Syncing the store between machines
|
|
244
|
+
|
|
245
|
+
A research Mac or a Linux dashboard is usually a **read-only replica** of a store
|
|
246
|
+
produced elsewhere. Prefer one producer writing everything and a strictly
|
|
247
|
+
one-directional sync. (The bar store syncs the same way, separately — it is a different
|
|
248
|
+
directory with a different producer.)
|
|
249
|
+
|
|
250
|
+
One directory must be **excluded** for size: `_cache/`, cotdata's cache of downloaded
|
|
251
|
+
CFTC source zips, is producer-internal and free to rebuild. The legacy `manifest.json`
|
|
252
|
+
should be excluded too. (A store built before ADR-0007 also has `_raw/` — databento's
|
|
253
|
+
paid bronze store — and `prices/`; both belong to `marketdata` now and neither is
|
|
254
|
+
written here any more.)
|
|
255
|
+
|
|
256
|
+
Anything a consumer put in the store by hand is a **correctness** issue rather than a
|
|
257
|
+
saving. No producer creates it, so a mirroring sync deletes it. Exclude it, but the real
|
|
258
|
+
fix is to keep it out of the store: the store belongs to its producer.
|
|
259
|
+
|
|
260
|
+
The same rule bites hardest on `vintage/` if you enable it, because that data cannot be
|
|
261
|
+
re-fetched: capture it on the **producer** so it syncs outward, or keep it outside the
|
|
262
|
+
mirrored store with `COTDATA_VINTAGE_ROOT`. Its provenance index is deliberately named
|
|
263
|
+
`snapshots.json`, since the usual `manifest.json` exclusion matches by name at any depth
|
|
264
|
+
and would otherwise strip it in transit, delivering raw archives with no index.
|
|
265
|
+
|
|
266
|
+
Consumer cloud sync (Dropbox, Google Drive) is a poor fit here: conflict copies land
|
|
267
|
+
inside the store, and on-demand placeholder files break `read_parquet` on the machine
|
|
268
|
+
doing research.
|
|
269
|
+
|
|
270
|
+
Full guidance, exclusion table and example scripts: **[docs/SYNCING.md](docs/SYNCING.md)**.
|
|
271
|
+
|
|
272
|
+
## Operations
|
|
273
|
+
|
|
274
|
+
Read-only and maintenance commands, all cross-platform (they work off the store, no network):
|
|
275
|
+
|
|
276
|
+
```bash
|
|
277
|
+
cotdata-update --check # store status: row counts, newest data, staleness
|
|
278
|
+
cotdata-update --reconcile # prune stale manifest entries (see below)
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
`--check` reports per-domain row counts, newest data date, last write, and any entries lagging behind their peers (a partial-run signal):
|
|
282
|
+
|
|
283
|
+
```
|
|
284
|
+
domain entries rows newest data last write (UTC) behind
|
|
285
|
+
prices 98 954,524 2026-08-07 2026-08-08T15:40:20Z 14d ← RETIRED
|
|
286
|
+
cot_legacy 53 81,821 2026-08-11 2026-08-21T12:10:42Z 10d
|
|
287
|
+
cot_disagg 29 29,187 2026-08-11 2026-08-21T12:10:51Z 10d
|
|
288
|
+
...
|
|
289
|
+
|
|
290
|
+
⚠ prices, metadata: RETIRED — this store still holds the files, but cotdata stopped writing
|
|
291
|
+
them at ADR-0007. Their dates are frozen and will never advance again.
|
|
292
|
+
Daily bars and contract specs now live in the marketdata store ($MARKETDATA_STORE);
|
|
293
|
+
read them with `marketdata-update --check`.
|
|
294
|
+
|
|
295
|
+
✓ every entry in the live domains was written by the latest producer pass.
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
**A `RETIRED` domain is not a warning you can wave away.** ADR-0007 moved bars and contract
|
|
299
|
+
specs to marketdata; a store created before that move still holds `prices/` and `metadata/`,
|
|
300
|
+
and nothing advances them. They earn a label rather than silence because lag is measured
|
|
301
|
+
*within* a domain — a uniformly frozen tree is perfectly self-consistent, so it scores
|
|
302
|
+
`lagging: 0` and reads exactly like a domain that ran cleanly minutes ago. See
|
|
303
|
+
[SYNCING.md](docs/SYNCING.md#retiring-the-moved-domains-prices-and-metadata) for removing them.
|
|
304
|
+
|
|
305
|
+
### `status.json` — new-data signal for downstream tools
|
|
306
|
+
|
|
307
|
+
Every producer run writes `$COTDATA_STORE/status.json` (atomically, beside the data), so tools that trigger on fresh data poll one small structured file instead of scanning the store:
|
|
308
|
+
|
|
309
|
+
```json
|
|
310
|
+
{
|
|
311
|
+
"generated_at": "2026-07-15T10:15:24Z",
|
|
312
|
+
"schema_version": 2,
|
|
313
|
+
"newest_data": { "cot_legacy": "2026-08-11", "cot_disagg": "2026-08-11", "cot_tff": "2026-08-11", "cot_supplemental": "2026-08-11", "prices": "2026-08-07" },
|
|
314
|
+
"domains": { "cot_legacy": { "newest_data": "2026-08-11", "last_write": "2026-08-21T12:10:42Z", "entries": 53, "rows": 81821, "lagging": 0, "retired": false },
|
|
315
|
+
"prices": { "newest_data": "2026-08-07", "last_write": "2026-08-08T15:40:20Z", "entries": 98, "rows": 954524, "lagging": 0, "retired": true }, "...": {} },
|
|
316
|
+
"last_run": { "kinds": ["cot_legacy", "cot_disagg"], "ok": ["001602", "..."], "symbols_failed": [], "rows": 111008, "seconds": 22, "at": "2026-08-21T12:10:58Z" }
|
|
317
|
+
}
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
**Polling contract:**
|
|
321
|
+
- To detect **new data**, compare `newest_data.<domain>` (e.g. `newest_data.cot_legacy`) against your last-seen value. It advances **only when genuinely new daily data arrives** — a no-op run leaves it unchanged.
|
|
322
|
+
- **Check `domains.<domain>.retired` first.** A retired domain's date is frozen, so a poller keyed on it waits forever and never errors — it just quietly reports data that stopped moving. This bullet used to give `newest_data.prices` as its example, and on 2026-08-21 a consumer following it read a two-week-old date from a store whose bars had moved to marketdata. **Bars are not in this store; poll `marketdata-update --check` instead.**
|
|
323
|
+
- To detect that **a run happened at all** (new data or not), use `generated_at`.
|
|
324
|
+
- `last_run` carries the most recent run's outcome (which domains, per-symbol failures) for alerting.
|
|
325
|
+
|
|
326
|
+
Prices and each COT report are separate domains, so a price-triggered tool and a COT-triggered tool each watch their own key.
|
|
327
|
+
|
|
328
|
+
### `--reconcile` — manifest hygiene
|
|
329
|
+
|
|
330
|
+
COT tables are stored per code as **`{symbol}_{code}`** (e.g. `RTY_23977A`), so a symbol's current and predecessor (`hist_codes`) contracts are both attributable to it. `--reconcile` drops manifest entries whose parquet file is missing — bare-code ghosts and retired domains left by older naming schemes — so `--check` and `status.json` show only real, consistently-named entries. It never touches data (only removes bookkeeping for files that don't exist).
|
|
331
|
+
|
|
332
|
+
## Concepts & design
|
|
333
|
+
|
|
334
|
+
### Price adjustment tiers
|
|
335
|
+
|
|
336
|
+
Not here. Futures roll, and stitching contracts creates artificial gaps, so bars come in
|
|
337
|
+
three tiers — `backadj` for signals and stops, `unadj` for position sizing, `propadj` for
|
|
338
|
+
anything measuring percent returns. All three, and the reasoning behind them (including
|
|
339
|
+
why `propadj` is not optional: additive back-adjustment drives 46.7% of Class III Milk's
|
|
340
|
+
closes below zero), live in
|
|
341
|
+
[crucible-marketdata's README](https://github.com/mspinola/marketdata).
|
|
342
|
+
|
|
343
|
+
### The symbol registry
|
|
344
|
+
|
|
345
|
+
The supported futures contracts are defined in a YAML registry, so adding a market needs no code:
|
|
346
|
+
|
|
347
|
+
- **Add a market:** edit `src/cotdata/registry.yaml` under its asset class. The registry handles metadata like `is_equity` and predecessor `hist_codes`.
|
|
348
|
+
- **Centralize it:** set `COTDATA_REGISTRY` to a shared `registry.yaml` (e.g. inside `$COTDATA_STORE`) so producer and consumers use identical asset definitions without a `git pull`.
|
|
349
|
+
|
|
350
|
+
### Atomic store
|
|
351
|
+
|
|
352
|
+
The store uses **atomic writes** (write-temp-then-rename). Consumers can safely query via `get_cot` even while `cotdata-update` is actively downloading and writing.
|
|
353
|
+
|
|
354
|
+
### COT vintage tracking (as-published history)
|
|
355
|
+
|
|
356
|
+
CFTC revises COT data after publication — most consequentially through **trader
|
|
357
|
+
reclassification**, which moves positions between categories retroactively. Because
|
|
358
|
+
downstream signals are rolling z-scores and percentiles against years of history, a
|
|
359
|
+
restatement silently rewrites the baseline every historical reading was computed against.
|
|
360
|
+
There is precedent: in July 2008 the Commission revised reports back to July 3, 2007.
|
|
361
|
+
|
|
362
|
+
**CFTC serves current state only.** There is no vintage archive and no as-published
|
|
363
|
+
endpoint, so vintage data can only be accumulated going forward — every uncaptured week is
|
|
364
|
+
a permanent blind spot in the part of the series most likely to have been revised.
|
|
365
|
+
|
|
366
|
+
This is **opt-in and purely additive**: if you never run it, the store behaves exactly as
|
|
367
|
+
before. Enabling it adds a `vintage/` subtree.
|
|
368
|
+
|
|
369
|
+
```bash
|
|
370
|
+
cotdata-vintage fetch # capture current + prior year + weekly static (daily)
|
|
371
|
+
cotdata-vintage fetch --all # every year 1986-present (see below)
|
|
372
|
+
cotdata-vintage ingest --pending # parse retained raw -> observations + revisions
|
|
373
|
+
cotdata-vintage diff --since 2026-01-01 # field-level revisions, with revision depth
|
|
374
|
+
cotdata-vintage asof --as-of 2026-07-24T18:00:00 --report-date 2026-07-21
|
|
375
|
+
cotdata-vintage zero-sum --market 088691 # long/short/open-interest identity per week
|
|
376
|
+
cotdata-vintage coverage # which markets a report covers, per year
|
|
377
|
+
cotdata-schedule sync # CFTC Special Announcements
|
|
378
|
+
cotdata-schedule published # true publication dates from retained weekly statics
|
|
379
|
+
cotdata-schedule backfill # resolve release_date + its provenance
|
|
380
|
+
```
|
|
381
|
+
|
|
382
|
+
How it works:
|
|
383
|
+
|
|
384
|
+
- **Immutable landing zone.** Every fetch is recorded (including 304s) and raw bytes are
|
|
385
|
+
retained permanently under `vintage/raw/`, written atomically and never rewritten. A
|
|
386
|
+
byte-identical regeneration is deduped — a changed download is not itself a revision.
|
|
387
|
+
- **All four reports.** Legacy, Disaggregated, TFF and Supplemental canonicalise into one
|
|
388
|
+
long schema. Disagg and TFF also populate per-category spreading, per-category trader
|
|
389
|
+
counts and CR4/CR8 concentration, none of which the Legacy file carries, and they are
|
|
390
|
+
where **Managed Money** and **Leveraged Funds** live. A suppressed trader count (CFTC
|
|
391
|
+
writes `.`) canonicalises to null rather than to a string. Supplemental is where **Index
|
|
392
|
+
Traders** live, and it is the one report where `combined` is `True`: the category
|
|
393
|
+
vocabulary is checked per `report_type`, so its `commercial` (which is net of index
|
|
394
|
+
traders) can never be mistaken for Legacy's.
|
|
395
|
+
- **Coverage is not constant, and `cotdata-vintage coverage` says so.** A report's market
|
|
396
|
+
set is derived from the stored observations and every entry or exit is printed. The live
|
|
397
|
+
case is Supplemental, which covered 12 markets until Soybean Meal entered in 2013.
|
|
398
|
+
- **Change-only observations.** A row is written only when its value hash differs from the
|
|
399
|
+
latest for its natural key `(report_date, market_code, report_type, combined, category)`,
|
|
400
|
+
so storage grows with actual revisions rather than with time.
|
|
401
|
+
- **Field-level revisions** carry `age_days` (revision depth): whether revisions stay in
|
|
402
|
+
recent weeks or reach back into the calibration window determines how much the rest of
|
|
403
|
+
a system has to care.
|
|
404
|
+
- **Point-in-time reads.** `asof(t)` returns each key's latest value observed at or before
|
|
405
|
+
`t`, reconstructing what was actually knowable then.
|
|
406
|
+
- **Release dates with provenance.** `report_date` is stored exactly as reported (never
|
|
407
|
+
normalized to Tuesday), and `release_date` is resolved through
|
|
408
|
+
`published > observed > announced > scheduled > derived`, with the source recorded — a
|
|
409
|
+
release date without provenance is worse than none, since indexing on `report_date`
|
|
410
|
+
embeds a lookahead (three days normally, weeks during a backlog). `published` is the
|
|
411
|
+
weekly static's HTTP `Last-Modified`, a true publication timestamp; it is forward-only
|
|
412
|
+
(that file holds one week and is overwritten), so weeks predating capture fall back
|
|
413
|
+
down the chain. `announced` comes from the republication tables CFTC posts on the
|
|
414
|
+
Special Announcements page after a disruption, and is what puts the Oct–Dec 2025
|
|
415
|
+
appropriations-lapse backlog on its real dates: 36,296 stored rows that `derived`
|
|
416
|
+
otherwise places up to 47 days early, before the lapse that stopped them being
|
|
417
|
+
published at all.
|
|
418
|
+
|
|
419
|
+
- **Flow decomposition.** `cotdata-vintage flow` labels each week's ΔLong versus ΔShort as
|
|
420
|
+
`new_longs` / `short_covering` / `new_shorts` / `long_liquidation`, by dominant leg, with
|
|
421
|
+
no parameters to tune. A rally driven by short covering has a finite fuel supply and one
|
|
422
|
+
driven by fresh longs does not, and they are indistinguishable on a price chart. It also
|
|
423
|
+
emits `days_elapsed`, because COT was **fortnightly until 1992-10-13** and holidays shift
|
|
424
|
+
the rest, so a "weekly" change is not always weekly.
|
|
425
|
+
|
|
426
|
+
**Run capture on the producer, not a replica**, and schedule it **daily**: nearly every
|
|
427
|
+
request returns 304, so a daily run is close to free while catching holiday-shifted and
|
|
428
|
+
backlog releases with no schedule logic.
|
|
429
|
+
|
|
430
|
+
**The frozen-year tripwire.** CFTC regenerates a rolling two-year window (current plus the
|
|
431
|
+
immediately-prior year) and nothing older. The prior year is therefore re-served every week
|
|
432
|
+
but byte-identical, which is the one place a **content** check on closed data comes free.
|
|
433
|
+
It is in the default fetch set for that reason: it costs one roughly 7 MB transfer per week
|
|
434
|
+
and zero bytes on disk, and it is the only automated retroactive-restatement detector here.
|
|
435
|
+
Anything other than `unchanged bytes (deduped)` on it raises an alert, which `ingest`
|
|
436
|
+
re-raises as a non-zero exit and a `REVISIONS_<date>.txt` marker file. `--no-prior-year`
|
|
437
|
+
turns it off. `--all` extends the same check to every year, but since nothing older is ever
|
|
438
|
+
re-served it mostly confirms 304s; monthly or quarterly is the right cadence for that. Full
|
|
439
|
+
design notes, including the measured CFTC caching behaviour, are in
|
|
440
|
+
[docs/design/cot_vintage.md](docs/design/cot_vintage.md).
|
|
441
|
+
|
|
442
|
+
> **Replica warning.** The vintage tree must not be written on a machine whose store is
|
|
443
|
+
> mirrored (`robocopy /MIR`, `rsync --delete`) from a producer: the mirror deletes
|
|
444
|
+
> destination-only files and the data is irreplaceable. Capture on the producer, or set
|
|
445
|
+
> `COTDATA_VINTAGE_ROOT` to a path outside the mirrored store. See [docs/SYNCING.md](docs/SYNCING.md).
|
|
446
|
+
|
|
447
|
+
## Local development
|
|
448
|
+
|
|
449
|
+
```bash
|
|
450
|
+
uv venv # create .venv
|
|
451
|
+
uv pip install -e . # install cotdata + deps
|
|
452
|
+
export COTDATA_STORE=/path/to/synced/store # the shared store
|
|
453
|
+
uv run pytest # run the tests
|
|
454
|
+
```
|
|
455
|
+
|
|
456
|
+
There are no optional data extras — every vendor SDK moved out with the producers. Use `uv run <cmd>`, or activate with `source .venv/bin/activate` (Mac/Linux) / `.venv\Scripts\activate` (Windows).
|
|
457
|
+
|
|
458
|
+
## Reference: Data schemas
|
|
459
|
+
|
|
460
|
+
The canonical store uses standard Parquet files. Loaded with `pd.read_parquet()`, they conform to the following schemas.
|
|
461
|
+
|
|
462
|
+
### Price Data
|
|
463
|
+
|
|
464
|
+
Not in this store. Every bar, and the schema describing it, moved to
|
|
465
|
+
[crucible-marketdata](https://pypi.org/project/crucible-marketdata/) under ADR-0007.
|
|
466
|
+
A store built before that move still has a `prices/` directory: it is history, nothing
|
|
467
|
+
here reads or writes it, and `--reconcile` will keep its manifest entries honest.
|
|
468
|
+
|
|
469
|
+
### Contract Specifications
|
|
470
|
+
|
|
471
|
+
Moved to `marketdata`'s store (`metadata/contract_specs.parquet` under
|
|
472
|
+
`$MARKETDATA_STORE`) with the Norgate producer that writes them — read them with
|
|
473
|
+
`marketdata.read_metadata()`. Nothing in this package writes contract specs any more.
|
|
474
|
+
|
|
475
|
+
### COT Legacy Data (`cot_legacy/{symbol}_{code}.parquet`)
|
|
476
|
+
Legacy positioning data (CFTC Legacy Futures Report). **History starts in 1986.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
|
|
477
|
+
|
|
478
|
+
> [!NOTE]
|
|
479
|
+
> **Legacy Reports**: broken down by exchange, with futures-only and combined futures-and-options variants. Legacy classifies reportable open interest into non-commercial and commercial traders. This directory holds the **futures-only** files (`dea_fut_xls_{YEAR}.zip`); the combined files (`dea_com_xls_{YEAR}.zip`, from 1995) land in `cot_legacy_combined/` with the same columns.
|
|
480
|
+
|
|
481
|
+
> [!NOTE]
|
|
482
|
+
> **Column Subset**: The raw CFTC `.xls` files contain [well over 100 columns](https://www.cftc.gov/MarketReports/CommitmentsofTraders/HistoricalViewable/cotvariableslegacy.html); the pipeline keeps the focused 15-column subset below to keep files small. To include more, add the exact CFTC column name to `TARGET_COLS` in `src/cotdata/providers/cftc.py`.
|
|
483
|
+
|
|
484
|
+
| Column | Type | Description |
|
|
485
|
+
|--------|------|-------------|
|
|
486
|
+
| `Report_Date_as_MM_DD_YYYY` | DatetimeIndex | Reporting date (typically Tuesday). |
|
|
487
|
+
| `Market_and_Exchange_Names` | string | Name of the contract and exchange. |
|
|
488
|
+
| `CFTC_Contract_Market_Code` | string | 6-digit CFTC contract code. |
|
|
489
|
+
| `Open_Interest_All` | float | Total open interest for the contract. |
|
|
490
|
+
| `Comm_Positions_Long_All` | float | Commercial Long positions. |
|
|
491
|
+
| `Comm_Positions_Short_All` | float | Commercial Short positions. |
|
|
492
|
+
| `NonComm_Positions_Long_All` | float | Non-Commercial (Large Speculator) Long positions. |
|
|
493
|
+
| `NonComm_Positions_Short_All` | float | Non-Commercial (Large Speculator) Short positions. |
|
|
494
|
+
| `NonRept_Positions_Long_All` | float | Non-Reportable (Small Speculator) Long positions. |
|
|
495
|
+
| `NonRept_Positions_Short_All` | float | Non-Reportable (Small Speculator) Short positions. |
|
|
496
|
+
| `Traders_Tot_All` | float | Total number of reportable traders. |
|
|
497
|
+
| `Traders_Comm_Long_All` | float | Number of Commercial Long traders. |
|
|
498
|
+
| `Traders_Comm_Short_All` | float | Number of Commercial Short traders. |
|
|
499
|
+
| `Traders_NonComm_Long_All` | float | Number of Non-Commercial Long traders. |
|
|
500
|
+
| `Traders_NonComm_Short_All` | float | Number of Non-Commercial Short traders. |
|
|
501
|
+
|
|
502
|
+
### COT Disaggregated Data (`cot_disagg/{symbol}_{code}.parquet`)
|
|
503
|
+
Entity-specific positioning and trader counts (CFTC Disaggregated Futures-Only Report). **History starts in 2006.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
|
|
504
|
+
|
|
505
|
+
> [!NOTE]
|
|
506
|
+
> **Lossless Image**: Unlike the filtered Legacy schema, the Disaggregated parquets are a **lossless image** of the source CFTC `txt` files — all granular entity groups (Money Manager, Swap Dealer, Producer/Merchant, Other Reportable) and their `Traders_*` counts. Required for computing Position Size and Clustering metrics.
|
|
507
|
+
|
|
508
|
+
### COT Traders in Financial Futures (TFF) Data (`cot_tff/{symbol}_{code}.parquet`)
|
|
509
|
+
Entity-specific positioning and trader counts for financial markets (CFTC TFF Futures-Only Report). **History starts in 2006.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
|
|
510
|
+
|
|
511
|
+
> [!NOTE]
|
|
512
|
+
> **Financials Counterpart**: TFF is the exact counterpart to Disaggregated, used for financial markets (Equities, FX, Rates), which have no Disaggregated report.
|
|
513
|
+
|
|
514
|
+
> [!NOTE]
|
|
515
|
+
> **Lossless Image**: Like Disaggregated, TFF parquets are a **lossless image** of the source CFTC `txt` files — the financial entity groups (`Dealer`, `Asset_Mgr`, `Lev_Money`, `Other_Rept`) and their `Traders_*` counts.
|
|
516
|
+
|
|
517
|
+
### COT combined-basis data (`cot_legacy_combined/`, `cot_disagg_combined/`, `cot_tff_combined/`)
|
|
518
|
+
The three reports above on the CFTC's **futures-and-options combined** basis (`dea_com_xls_*`, `com_disagg_txt_*`, `com_fin_txt_*`). Same parsers, same columns, same per-code files; Legacy combined starts in **1995**, the other two in 2006. The Disaggregated and TFF files state their basis in `FutOnly_or_Combined` and the producer refuses a file whose statement disagrees with the directory it is writing to; the Legacy files carry no such column, so for them the URL is the only evidence and the two bases are kept apart by construction (separate zip names, caches and writers behind one flag). Measured on 2026-10-27: ES **2,843,639 combined / 1,955,711 futures-only**, GC 518,784 / 396,810, CL 2,606,478 / 1,886,280. See [ADR-0003](docs/adr/ADR-0003-combined-basis-is-in-scope.md).
|
|
519
|
+
|
|
520
|
+
### COT Supplemental Data (`cot_supplemental/{symbol}_{code}.parquet`)
|
|
521
|
+
Index-trader positioning for 13 select agricultural markets (CFTC Supplemental / Commodity Index Trader Report). **History starts in January 2006**, and the covered set was **12 markets until Soybean Meal entered on 2013-04-02**. Indexed by tz-naive `Report_Date_as_MM_DD_YYYY` (renamed at parse from CFTC's `As_of_Date_In_Form_*`, whose own name changed in 2013).
|
|
522
|
+
|
|
523
|
+
> [!WARNING]
|
|
524
|
+
> **Combined basis, and the file does not say so.** This is the only report with no futures-only variant, and unlike Disaggregated and TFF it carries no `FutOnly_or_Combined` column. Verified by matching open interest against both Legacy series: 390/390 against futures-and-options combined, 0/390 against futures-only. Its open interest is therefore **not** comparable with the other three for the same market and week.
|
|
525
|
+
|
|
526
|
+
> [!NOTE]
|
|
527
|
+
> **Positions are net of index traders.** `Comm_Positions_Long_All_NoCIT` is commercial *minus* the index book, not Legacy's commercial. The index book is carved out of commercial, non-commercial **and non-commercial spreading** — three buckets, where CFTC's own prose names two. Non-Reportable is untouched, because index traders are reportable by definition.
|
|
528
|
+
|
|
529
|
+
> [!TIP]
|
|
530
|
+
> **It joins to Legacy combined, exactly.** The `_NoCIT` buckets are `cot_legacy_combined/`'s buckets with the index traders removed, for the same market and week, and open interest and non-reportable agree to the contract on all 13,753 shared weeks 2006-2026. `cotdata.cit.decompose(name)` makes the join and splits each Legacy bucket into its ex-index part and the index position carved from it (spreading as the residual, since the store does not keep Legacy's spread column); it refuses a market where the exact identities fail, which is a basis or vintage error. Measurements, including how much of Legacy's "commercial" is index money per market: [docs/analysis/2026-10-31-cit-legacy-combined-decomposition.md](docs/analysis/2026-10-31-cit-legacy-combined-decomposition.md).
|
|
531
|
+
|
|
532
|
+
> [!NOTE]
|
|
533
|
+
> **Lossless Image**: Like Disaggregated and TFF, these parquets are a lossless image of the source CFTC `txt` files, including CFTC's own header typos (`NComm_Postions_Spread_All_NoCIT`).
|
|
534
|
+
|
|
535
|
+
## Reference: COT formats explained
|
|
536
|
+
|
|
537
|
+
The CFTC publishes positioning data in four formats, the first three of them on two bases (futures-only and futures-and-options combined); `cotdata` manages all seven series for complete coverage and the deepest history.
|
|
538
|
+
|
|
539
|
+
1. **Legacy (1986–Present)** — *all markets.* Divides traders into **Commercial** (hedgers) and **Non-Commercial** (large speculators). The only format with pre-2006 data, so it's essential for long-term backtesting.
|
|
540
|
+
2. **Disaggregated / DIS (2006–Present)** — *physical commodities only* (Agriculture, Energy, Metals). Splits traders into **Producer/Merchant**, **Swap Dealers**, **Managed Money**, and **Other Reportables** — a clearer view of "smart money" (Managed Money) in commodities.
|
|
541
|
+
3. **Traders in Financial Futures / TFF (2006–Present)** — *financial markets only* (Equities, Rates, Currencies). Splits traders into **Dealer/Intermediary**, **Asset Manager**, **Leveraged Funds**, and **Other Reportables** — the definitive source for speculative flow (Leveraged Funds) in financials.
|
|
542
|
+
4. **Supplemental / CIT (2006–Present)** — *13 select agricultural markets only.* Takes the Legacy split and carves **Index Traders** out of it, the only public source that separates index flow. Futures-and-options **combined** only. The taxonomy is Legacy, not Disaggregated, so Index Traders does **not** nest inside Disaggregated's Swap Dealer and the two cannot be differenced to isolate levered swap flow.
|
|
543
|
+
|
|
544
|
+
## Diagnostics
|
|
545
|
+
|
|
546
|
+
```bash
|
|
547
|
+
cotdata-update --check # coverage, newest dates, staleness — no network
|
|
548
|
+
python scripts/vintage_alert_selftest.py # the vintage capture's alerting path
|
|
549
|
+
```
|
|
550
|
+
|
|
551
|
+
Norgate subscription and roll-gap diagnostics moved to `marketdata` with the provider.
|
|
552
|
+
|
|
553
|
+
## Ecosystem
|
|
554
|
+
|
|
555
|
+
cotdata is the *data* layer of a small, unbundled toolchain — it stops at "clean
|
|
556
|
+
data behind a stable API" on purpose. What you do with that data is a separate,
|
|
557
|
+
swappable step:
|
|
558
|
+
|
|
559
|
+
- **cotdata** *(this package)* — CFTC COT positioning. One synced store, many
|
|
560
|
+
readers, no vendor SDK at read time.
|
|
561
|
+
- **[crucible-marketdata](https://pypi.org/project/crucible-marketdata/)** — the
|
|
562
|
+
daily bars that used to live here: Norgate futures, Yahoo equities/ETFs, the
|
|
563
|
+
adjustment tiers, contract specs. Split out by ADR-0007 so a positioning question
|
|
564
|
+
and a price question have separate stores, producers and schedules.
|
|
565
|
+
- **[crucible](https://github.com/mspinola/crucible)** — the *edge* layer. Feed a
|
|
566
|
+
signal built on cotdata frames into crucible and it tells you — with a
|
|
567
|
+
confidence interval and a p-value — whether the trade-level edge is real, before
|
|
568
|
+
you open a funded account.
|
|
569
|
+
|
|
570
|
+
The flow runs one direction: **`cotdata` (data) → your signal → `crucible`
|
|
571
|
+
(edge)**. Neither imports the other, so cotdata stays useful on its own for any
|
|
572
|
+
COT/futures research — crucible is just the most common thing to point at it next.
|
|
573
|
+
|
|
574
|
+
## Development
|
|
575
|
+
|
|
576
|
+
Want to contribute or work on cotdata locally? See [CONTRIBUTING.md](CONTRIBUTING.md) for:
|
|
577
|
+
- Virtual environment setup with `uv` or standard `pip`
|
|
578
|
+
- Running the test suite
|
|
579
|
+
- Platform-specific notes (everything here runs anywhere — no vendor SDK)
|
|
580
|
+
- Code style guidelines
|
|
581
|
+
|
|
582
|
+
## Contributing
|
|
583
|
+
|
|
584
|
+
Issues and pull requests are welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, tests, and conventions. When filing a bug, include your OS. If it is about price bars, it probably belongs on [crucible-marketdata](https://github.com/mspinola/marketdata) instead.
|
|
585
|
+
|
|
586
|
+
## License
|
|
587
|
+
|
|
588
|
+
Released under the MIT License — see [LICENSE](LICENSE).
|