cotdata 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. cotdata-0.1.0/LICENSE +21 -0
  2. cotdata-0.1.0/PKG-INFO +246 -0
  3. cotdata-0.1.0/README.md +217 -0
  4. cotdata-0.1.0/pyproject.toml +48 -0
  5. cotdata-0.1.0/setup.cfg +4 -0
  6. cotdata-0.1.0/src/cotdata/__init__.py +11 -0
  7. cotdata-0.1.0/src/cotdata/config.py +35 -0
  8. cotdata-0.1.0/src/cotdata/cot.py +62 -0
  9. cotdata-0.1.0/src/cotdata/prices.py +43 -0
  10. cotdata-0.1.0/src/cotdata/providers/__init__.py +2 -0
  11. cotdata-0.1.0/src/cotdata/providers/base.py +14 -0
  12. cotdata-0.1.0/src/cotdata/providers/cftc.py +129 -0
  13. cotdata-0.1.0/src/cotdata/providers/cftc_disagg.py +146 -0
  14. cotdata-0.1.0/src/cotdata/providers/cftc_tff.py +146 -0
  15. cotdata-0.1.0/src/cotdata/providers/databento.py +280 -0
  16. cotdata-0.1.0/src/cotdata/providers/norgate.py +78 -0
  17. cotdata-0.1.0/src/cotdata/registry.py +127 -0
  18. cotdata-0.1.0/src/cotdata/registry.yaml +105 -0
  19. cotdata-0.1.0/src/cotdata/store.py +96 -0
  20. cotdata-0.1.0/src/cotdata/update.py +42 -0
  21. cotdata-0.1.0/src/cotdata.egg-info/PKG-INFO +246 -0
  22. cotdata-0.1.0/src/cotdata.egg-info/SOURCES.txt +30 -0
  23. cotdata-0.1.0/src/cotdata.egg-info/dependency_links.txt +1 -0
  24. cotdata-0.1.0/src/cotdata.egg-info/entry_points.txt +2 -0
  25. cotdata-0.1.0/src/cotdata.egg-info/requires.txt +16 -0
  26. cotdata-0.1.0/src/cotdata.egg-info/top_level.txt +1 -0
  27. cotdata-0.1.0/tests/test_adjustment.py +111 -0
  28. cotdata-0.1.0/tests/test_cftc_disagg_fidelity.py +29 -0
  29. cotdata-0.1.0/tests/test_cftc_reconciliation.py +54 -0
  30. cotdata-0.1.0/tests/test_cftc_tff_reconciliation.py +64 -0
  31. cotdata-0.1.0/tests/test_registry.py +152 -0
  32. cotdata-0.1.0/tests/test_store.py +48 -0
cotdata-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Matt S.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
cotdata-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,246 @@
1
+ Metadata-Version: 2.4
2
+ Name: cotdata
3
+ Version: 0.1.0
4
+ Summary: Canonical data layer for COT/futures strategies — a producer/consumer split over a file-based store.
5
+ Author-email: "Matt S." <cotanalyzer@gmail.com>
6
+ Project-URL: Homepage, https://github.com/mspinola/cotdata
7
+ Project-URL: Repository, https://github.com/mspinola/cotdata
8
+ Project-URL: Issues, https://github.com/mspinola/cotdata/issues
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Operating System :: OS Independent
12
+ Requires-Python: >=3.9
13
+ Description-Content-Type: text/markdown
14
+ License-File: LICENSE
15
+ Requires-Dist: pandas>=1.5
16
+ Requires-Dist: pyarrow>=10
17
+ Requires-Dist: requests>=2.28
18
+ Requires-Dist: python-dateutil>=2.8
19
+ Requires-Dist: packaging
20
+ Requires-Dist: xlrd>=2.0
21
+ Requires-Dist: PyYAML>=6.0
22
+ Provides-Extra: norgate
23
+ Requires-Dist: norgatedata; extra == "norgate"
24
+ Provides-Extra: databento
25
+ Requires-Dist: databento>=0.30; extra == "databento"
26
+ Provides-Extra: dev
27
+ Requires-Dist: pytest>=7; extra == "dev"
28
+ Dynamic: license-file
29
+
30
+ # cotdata
31
+
32
+ [![PyPI version](https://img.shields.io/pypi/v/cotdata.svg)](https://pypi.org/project/cotdata/)
33
+ [![Python versions](https://img.shields.io/pypi/pyversions/cotdata.svg)](https://pypi.org/project/cotdata/)
34
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
35
+
36
+ Canonical data layer for the COT / futures-strategy stack. It exists so that quantitative analysis toolset never fetch data directly. They read a shared, file-based store through a stable API.
37
+
38
+ ```
39
+ PRODUCER (runs where each source is reachable)
40
+ Windows: Norgate export anywhere: CFTC COT download
41
+ │ writes │
42
+ ▼ ▼
43
+ ┌───────────────────────────────────────┐
44
+ │ CANONICAL STORE ($COTDATA_STORE) │
45
+ │ prices/*.parquet cot_legacy/*.parquet │ ← synced (rsync / Dropbox / S3)
46
+ │ cot_disagg/*.parquet manifest.json │
47
+ └────────────────────────────────────────────────┘
48
+ ▲ ▲ reads (offline, cross-platform)
49
+ ┌──────────────┴───┐ ┌────┴──────────────┐
50
+ │ COT analyzer │ │ analysis toolset │ both: import cotdata
51
+ └──────────────────┘ └───────────────────┘
52
+ ```
53
+
54
+ ## Installation
55
+
56
+ You can install `cotdata` directly from PyPI:
57
+ ```bash
58
+ pip install cotdata
59
+ ```
60
+
61
+ If you are running the **producer machine** on Windows to fetch Norgate data, install with the `norgate` extra:
62
+ ```bash
63
+ pip install "cotdata[norgate]"
64
+ ```
65
+
66
+ ## Development
67
+
68
+ When using Norgate Data one must run this on a Windows machine as Norgate Updater requires Windows.
69
+
70
+ The downstream analysis toolset can run on any machine (Windows, Mac, Linux) as they only need to read data from the canonical store.
71
+
72
+ One can create a unified workspace by cloning all repositories into the same parent directory, and installing them into a single virtual environment.
73
+
74
+ ```bash
75
+ uv venv # create .venv
76
+ uv pip install -e . # install cotdata & all deps
77
+ export COTDATA_STORE=/path/to/synced/store # the shared store
78
+ ```
79
+
80
+ **Producer machine (Windows, Norgate)** — only cotdata + the `norgate` extra:
81
+
82
+ ```powershell
83
+ uv venv --python 3.10 # Tested through norgate's supported python versions.
84
+ uv pip install -e ".[norgate]" # from the cotdata repo; pulls norgatedata
85
+ $env:COTDATA_STORE = "C:\path\to\store" # Path to output location
86
+ cotdata-update --prices --symbols ES NQ
87
+ ```
88
+
89
+ Use `uv run <cmd>` to run without activating, or activate with
90
+ `source .venv/bin/activate` (Mac) / `.venv\Scripts\activate` (Windows).
91
+
92
+ ## The contract
93
+
94
+ The **store is the API boundary** — not Python imports. Producers write Parquet +
95
+ `manifest.json`; consumers only read. Nobody touches a vendor SDK at app runtime.
96
+ Swapping a vendor is a producer-only change.
97
+
98
+ - `prices/{symbol}_{adjustment}.parquet` — Open/High/Low/Close/Volume/Open Interest,
99
+ tz-naive `Date` index. `adjustment` ∈ {`backadj`, `unadj`}. Close = exchange settlement.
100
+ - `cot_legacy/{code}.parquet` — weekly CFTC Legacy positioning.
101
+ - `cot_disagg/{code}.parquet` — weekly CFTC Disaggregated positioning.
102
+ - `cot_tff/{code}.parquet` — weekly CFTC Traders in Financial Futures positioning.
103
+ - `manifest.json` — per-table `last_date`, `n_rows`, `source`, `updated_at`, `schema_version`.
104
+
105
+ ## Consumer
106
+
107
+ ```python
108
+ import cotdata
109
+ df = cotdata.get_prices("ES", adjustment="backadj") # USE THIS FOR SIGNALS + STOPS
110
+ sz = cotdata.get_prices("ES", adjustment="unadj") # USE FOR POSITION SIZING / POINT VALUE
111
+ cot_legacy = cotdata.get_cot("ES", report="legacy") # USE FOR COMM/NON-COMM
112
+ cot_disagg = cotdata.get_cot("ES", report="disagg") # USE FOR TRADER COUNTS (MM/SD/OR)
113
+ cot_tff = cotdata.get_cot("ES", report="tff") # USE FOR TRADER COUNTS (FINANCIALS)
114
+ ```
115
+
116
+ Set `COTDATA_STORE` to the synced store directory.
117
+
118
+ **Predecessor Stitching & Scaling:** The `get_cot()` function doesn't just read a file; it dynamically stitches historical CFTC codes for contracts that migrated exchanges (like the Russell 2000) or rescales data for contracts that changed tick sizes (like Lumber). Downstream models see one clean, continuous asset.
119
+
120
+ ## Producer (run on the machine that can reach the source)
121
+
122
+ ```
123
+ COTDATA_STORE=/store cotdata-update --prices --symbols ES NQ # Norgate (Windows)
124
+ COTDATA_STORE=/store cotdata-update --cot-legacy # CFTC Legacy (cross-platform)
125
+ COTDATA_STORE=/store cotdata-update --cot-disagg # CFTC Disaggregated (cross-platform)
126
+ COTDATA_STORE=/store cotdata-update --cot-tff # CFTC Traders in Financial Futures (cross-platform)
127
+ COTDATA_STORE=/store cotdata-update --cot-all # Update all CFTC COT pipelines
128
+ ```
129
+
130
+ Schedule nightly (prices, after the Norgate Data Updater) and weekly (COT Friday releases).
131
+
132
+ ## Design rules
133
+
134
+ ### Why Back-Adjusted vs Unadjusted?
135
+
136
+ Futures contracts expire, forcing traders to "roll" into the next contract, which usually trades at a slightly different price. Simply stitching these contracts together creates artificial price gaps.
137
+
138
+ - **`backadj` (for signals & stops)**: Uses gap-free arithmetic rolls. This mathematically shifts historical prices backward to align with the new contract, preserving the *true shape* and percentage moves of the market. You must use this for technical indicators, trade signals, and stop-losses to avoid false triggers on rollover gaps.
139
+ - **`unadj` (for position sizing)**: Because back-adjustment shifts historical prices (sometimes into the negative), you cannot use it to calculate true dollar values. You must use `unadj` (raw, real-life) data for that exact day to calculate your true dollar risk and decide exactly how many contracts to buy.
140
+
141
+ ### Providers & Authentication
142
+
143
+ **Norgate Data (Primary)**:
144
+ There are no API keys to configure in Python. The `norgatedata` Python package communicates locally with the Norgate Data Updater application. You simply need to have the Norgate Data Updater installed, authenticated, and running in the background on your Windows machine.
145
+
146
+ **Databento (Dormant/Intraday)**:
147
+ Databento is kept as a dormant provider because it works well and can be leveraged for intraday data. If you wish to use it, you must provide your API key via the `DATABENTO_API_KEY` environment variable:
148
+ ```bash
149
+ export DATABENTO_API_KEY="your_api_key_here"
150
+ ```
151
+
152
+ ### Parameterizing the Asset List
153
+
154
+ The list of supported futures contracts (the symbol registry) is dynamically loaded from a YAML configuration file, making it easy to add new markets without touching any Python code.
155
+
156
+ - **Adding a Market**: Simply edit `cotdata/src/cotdata/registry.yaml` and add the new market under its respective Asset Class (e.g., Copper or Cocoa). The system natively handles metadata such as `is_equity` and complex `hist_codes` structures.
157
+ - **Environment Override**: The `COTDATA_REGISTRY` environment variable allows you to point to a centralized `registry.yaml` file. For instance, you could place your `registry.yaml` inside your synced `$COTDATA_STORE`. This ensures both your Windows producer and Mac consumer are always looking at the exact same asset definitions, without needing to `git pull` the Python repository.
158
+
159
+ ## Atomic Store
160
+
161
+ The store uses **atomic writes**. Consumers can safely query the store via `get_prices` or `get_cot` even while `cotdata-update` is actively downloading and writing new data.
162
+
163
+ ## Diagnostics
164
+
165
+ You can verify your Norgate subscription's data quality and system configuration using the included smoke test script. Run it on your Windows producer machine:
166
+ ```bash
167
+ python tests/test_adjustment.py
168
+ ```
169
+ This diagnostic script tests:
170
+ 1. **Local Communication**: Verifies that Python can successfully communicate with the Norgate Data Updater running in the background.
171
+ 2. **Subscription Access**: Validates that your Norgate subscription is active and has the required CME futures data package enabled.
172
+ 3. **Roll Gap Validation**: Mathematically proves whether your Norgate Data Updater is configured globally to return back-adjusted or unadjusted continuous contracts. It hunts for artificial calendar-spread gaps at contract roll dates to ensure you are receiving gap-free, continuous data, which is absolutely vital for accurate stop-loss modeling.
173
+
174
+ ## COT Formats Explained
175
+
176
+ The CFTC publishes positioning data in three distinct formats. `cotdata` manages all three to ensure complete market coverage and the deepest possible historical backtesting.
177
+
178
+ 1. **Legacy Format (1986–Present)**
179
+ - **Scope:** All markets.
180
+ - **Categories:** Divides traders broadly into **Commercial** (hedgers) and **Non-Commercial** (large speculators).
181
+ - **Use Case:** This is the original format. While its broad categories make it less precise for modern analysis, it is the only format that provides data prior to 2006, making it essential for long-term historical backtesting.
182
+
183
+ 2. **Disaggregated Format (DIS) (2006–Present)**
184
+ - **Scope:** Physical commodities only (Agriculture, Energy, Metals).
185
+ - **Categories:** Splits traders into four granular groups: **Producer/Merchant** (classic hedgers), **Swap Dealers** (financial intermediaries), **Managed Money** (hedge funds / CTAs), and **Other Reportables**.
186
+ - **Use Case:** Provides a much clearer view of the "Smart Money" (Managed Money) in commodity markets.
187
+
188
+ 3. **Traders in Financial Futures (TFF) (2006–Present)**
189
+ - **Scope:** Financial markets only (Equities, Rates, Currencies).
190
+ - **Categories:** The financial counterpart to Disaggregated. Splits traders into: **Dealer/Intermediary** (sell-side), **Asset Manager** (pension/mutual funds), **Leveraged Funds** (hedge funds / CTAs), and **Other Reportables**.
191
+ - **Use Case:** The definitive source for tracking speculative flow (Leveraged Funds) in financial markets.
192
+
193
+ ## Data Schemas
194
+
195
+ The canonical store uses standard Parquet files. When loaded into a pandas DataFrame (e.g., via `pd.read_parquet()`), they conform to the following schemas.
196
+
197
+ ### Price Data (`prices/{symbol}_{adjustment}.parquet`)
198
+ The primary source for price history (Norgate Data). Indexed by tz-naive `Date`.
199
+
200
+ | Column | Type | Description |
201
+ |--------|------|-------------|
202
+ | `Date` | DatetimeIndex | Trading day (tz-naive, normalized to midnight). |
203
+ | `Open` | float | Opening price. |
204
+ | `High` | float | High price. |
205
+ | `Low` | float | Low price. |
206
+ | `Close` | float | Exchange settlement close. |
207
+ | `Volume` | float | Trading volume. |
208
+ | `Open Interest` | float | Total open interest. |
209
+ | `Delivery Month` | float | Expiration month of the active contract (e.g. `202609`). Used to detect contract rolls. |
210
+
211
+ ### COT Legacy Data (`cot_legacy/{code}.parquet`)
212
+ The primary source for Legacy positioning data (CFTC Legacy Futures Report). **History starts in 1986.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
213
+
214
+ > [!NOTE]
215
+ > **Legacy Reports**: The Legacy reports are broken down by exchange. These reports have a futures only report and a combined futures and options report. Legacy reports break down the reportable open interest positions into two classifications: non-commercial and commercial traders. The `cotdata` pipeline strictly downloads the **Futures-only** reports (located at `https://www.cftc.gov/files/dea/history/dea_fut_xls_{YEAR}.zip`).
216
+
217
+ > [!NOTE]
218
+ > **Column Subset**: While the raw CFTC `.xls` files contain [well over 100 columns](https://www.cftc.gov/MarketReports/CommitmentsofTraders/HistoricalViewable/cotvariableslegacy.html) (including spreading, concentration ratios, etc.), the producer pipeline explicitly discards them. The parquet files only maintain the exact 10-column subset listed below to keep the file sizes extremely small and strictly focused on what the downstream models require. To include additional data points from the raw reports, simply add the exact CFTC column name to the `TARGET_COLS` list inside `src/cotdata/providers/cftc.py`.
219
+
220
+ | Column | Type | Description |
221
+ |--------|------|-------------|
222
+ | `Report_Date_as_MM_DD_YYYY` | DatetimeIndex | Reporting date (typically Tuesday). |
223
+ | `Market_and_Exchange_Names` | string | Name of the contract and exchange. |
224
+ | `CFTC_Contract_Market_Code` | string | 6-digit CFTC contract code. |
225
+ | `Open_Interest_All` | float | Total open interest for the contract. |
226
+ | `Comm_Positions_Long_All` | float | Commercial Long positions. |
227
+ | `Comm_Positions_Short_All` | float | Commercial Short positions. |
228
+ | `NonComm_Positions_Long_All` | float | Non-Commercial (Large Speculator) Long positions. |
229
+ | `NonComm_Positions_Short_All` | float | Non-Commercial (Large Speculator) Short positions. |
230
+ | `NonRept_Positions_Long_All` | float | Non-Reportable (Small Speculator) Long positions. |
231
+ | `NonRept_Positions_Short_All` | float | Non-Reportable (Small Speculator) Short positions. |
232
+
233
+ ### COT Disaggregated Data (`cot_disagg/{code}.parquet`)
234
+ The primary source for entity-specific positioning and trader counts (CFTC Disaggregated Futures-Only Report). **History starts in 2006.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
235
+
236
+ > [!NOTE]
237
+ > **Lossless Image**: Unlike the Legacy schema which filters down to 10 specific columns, the Disaggregated parquets are a **lossless image** of the source CFTC `txt` files. They contain all granular entity groups (Money Manager, Swap Dealer, Producer/Merchant, Other Reportable) and their respective `Traders_*` counts (e.g., `Traders_Tot_All`, `Traders_M_Money_Long_All`). This is the required store for computing Position Size and Clustering metrics.
238
+
239
+ ### COT Traders in Financial Futures (TFF) Data (`cot_tff/{code}.parquet`)
240
+ The primary source for entity-specific positioning and trader counts for Financial markets (CFTC Traders in Financial Futures Futures-Only Report). **History starts in 2006.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
241
+
242
+ > [!NOTE]
243
+ > **Financials Counterpart**: TFF is the exact counterpart to Disaggregated reports, used exclusively for financial markets (Equities, FX, Rates) which do not have Disaggregated reports.
244
+
245
+ > [!NOTE]
246
+ > **Lossless Image**: Like Disaggregated, TFF parquets are a **lossless image** of the source CFTC `txt` files. They contain the financial entity groups (`Dealer`, `Asset_Mgr`, `Lev_Money`, `Other_Rept`) and their respective `Traders_*` counts. This is the required store for computing Position Size and Clustering metrics for financial assets.
@@ -0,0 +1,217 @@
1
+ # cotdata
2
+
3
+ [![PyPI version](https://img.shields.io/pypi/v/cotdata.svg)](https://pypi.org/project/cotdata/)
4
+ [![Python versions](https://img.shields.io/pypi/pyversions/cotdata.svg)](https://pypi.org/project/cotdata/)
5
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
6
+
7
+ Canonical data layer for the COT / futures-strategy stack. It exists so that quantitative analysis toolset never fetch data directly. They read a shared, file-based store through a stable API.
8
+
9
+ ```
10
+ PRODUCER (runs where each source is reachable)
11
+ Windows: Norgate export anywhere: CFTC COT download
12
+ │ writes │
13
+ ▼ ▼
14
+ ┌───────────────────────────────────────┐
15
+ │ CANONICAL STORE ($COTDATA_STORE) │
16
+ │ prices/*.parquet cot_legacy/*.parquet │ ← synced (rsync / Dropbox / S3)
17
+ │ cot_disagg/*.parquet manifest.json │
18
+ └────────────────────────────────────────────────┘
19
+ ▲ ▲ reads (offline, cross-platform)
20
+ ┌──────────────┴───┐ ┌────┴──────────────┐
21
+ │ COT analyzer │ │ analysis toolset │ both: import cotdata
22
+ └──────────────────┘ └───────────────────┘
23
+ ```
24
+
25
+ ## Installation
26
+
27
+ You can install `cotdata` directly from PyPI:
28
+ ```bash
29
+ pip install cotdata
30
+ ```
31
+
32
+ If you are running the **producer machine** on Windows to fetch Norgate data, install with the `norgate` extra:
33
+ ```bash
34
+ pip install "cotdata[norgate]"
35
+ ```
36
+
37
+ ## Development
38
+
39
+ When using Norgate Data one must run this on a Windows machine as Norgate Updater requires Windows.
40
+
41
+ The downstream analysis toolset can run on any machine (Windows, Mac, Linux) as they only need to read data from the canonical store.
42
+
43
+ One can create a unified workspace by cloning all repositories into the same parent directory, and installing them into a single virtual environment.
44
+
45
+ ```bash
46
+ uv venv # create .venv
47
+ uv pip install -e . # install cotdata & all deps
48
+ export COTDATA_STORE=/path/to/synced/store # the shared store
49
+ ```
50
+
51
+ **Producer machine (Windows, Norgate)** — only cotdata + the `norgate` extra:
52
+
53
+ ```powershell
54
+ uv venv --python 3.10 # Tested through norgate's supported python versions.
55
+ uv pip install -e ".[norgate]" # from the cotdata repo; pulls norgatedata
56
+ $env:COTDATA_STORE = "C:\path\to\store" # Path to output location
57
+ cotdata-update --prices --symbols ES NQ
58
+ ```
59
+
60
+ Use `uv run <cmd>` to run without activating, or activate with
61
+ `source .venv/bin/activate` (Mac) / `.venv\Scripts\activate` (Windows).
62
+
63
+ ## The contract
64
+
65
+ The **store is the API boundary** — not Python imports. Producers write Parquet +
66
+ `manifest.json`; consumers only read. Nobody touches a vendor SDK at app runtime.
67
+ Swapping a vendor is a producer-only change.
68
+
69
+ - `prices/{symbol}_{adjustment}.parquet` — Open/High/Low/Close/Volume/Open Interest,
70
+ tz-naive `Date` index. `adjustment` ∈ {`backadj`, `unadj`}. Close = exchange settlement.
71
+ - `cot_legacy/{code}.parquet` — weekly CFTC Legacy positioning.
72
+ - `cot_disagg/{code}.parquet` — weekly CFTC Disaggregated positioning.
73
+ - `cot_tff/{code}.parquet` — weekly CFTC Traders in Financial Futures positioning.
74
+ - `manifest.json` — per-table `last_date`, `n_rows`, `source`, `updated_at`, `schema_version`.
75
+
76
+ ## Consumer
77
+
78
+ ```python
79
+ import cotdata
80
+ df = cotdata.get_prices("ES", adjustment="backadj") # USE THIS FOR SIGNALS + STOPS
81
+ sz = cotdata.get_prices("ES", adjustment="unadj") # USE FOR POSITION SIZING / POINT VALUE
82
+ cot_legacy = cotdata.get_cot("ES", report="legacy") # USE FOR COMM/NON-COMM
83
+ cot_disagg = cotdata.get_cot("ES", report="disagg") # USE FOR TRADER COUNTS (MM/SD/OR)
84
+ cot_tff = cotdata.get_cot("ES", report="tff") # USE FOR TRADER COUNTS (FINANCIALS)
85
+ ```
86
+
87
+ Set `COTDATA_STORE` to the synced store directory.
88
+
89
+ **Predecessor Stitching & Scaling:** The `get_cot()` function doesn't just read a file; it dynamically stitches historical CFTC codes for contracts that migrated exchanges (like the Russell 2000) or rescales data for contracts that changed tick sizes (like Lumber). Downstream models see one clean, continuous asset.
90
+
91
+ ## Producer (run on the machine that can reach the source)
92
+
93
+ ```
94
+ COTDATA_STORE=/store cotdata-update --prices --symbols ES NQ # Norgate (Windows)
95
+ COTDATA_STORE=/store cotdata-update --cot-legacy # CFTC Legacy (cross-platform)
96
+ COTDATA_STORE=/store cotdata-update --cot-disagg # CFTC Disaggregated (cross-platform)
97
+ COTDATA_STORE=/store cotdata-update --cot-tff # CFTC Traders in Financial Futures (cross-platform)
98
+ COTDATA_STORE=/store cotdata-update --cot-all # Update all CFTC COT pipelines
99
+ ```
100
+
101
+ Schedule nightly (prices, after the Norgate Data Updater) and weekly (COT Friday releases).
102
+
103
+ ## Design rules
104
+
105
+ ### Why Back-Adjusted vs Unadjusted?
106
+
107
+ Futures contracts expire, forcing traders to "roll" into the next contract, which usually trades at a slightly different price. Simply stitching these contracts together creates artificial price gaps.
108
+
109
+ - **`backadj` (for signals & stops)**: Uses gap-free arithmetic rolls. This mathematically shifts historical prices backward to align with the new contract, preserving the *true shape* and percentage moves of the market. You must use this for technical indicators, trade signals, and stop-losses to avoid false triggers on rollover gaps.
110
+ - **`unadj` (for position sizing)**: Because back-adjustment shifts historical prices (sometimes into the negative), you cannot use it to calculate true dollar values. You must use `unadj` (raw, real-life) data for that exact day to calculate your true dollar risk and decide exactly how many contracts to buy.
111
+
112
+ ### Providers & Authentication
113
+
114
+ **Norgate Data (Primary)**:
115
+ There are no API keys to configure in Python. The `norgatedata` Python package communicates locally with the Norgate Data Updater application. You simply need to have the Norgate Data Updater installed, authenticated, and running in the background on your Windows machine.
116
+
117
+ **Databento (Dormant/Intraday)**:
118
+ Databento is kept as a dormant provider because it works well and can be leveraged for intraday data. If you wish to use it, you must provide your API key via the `DATABENTO_API_KEY` environment variable:
119
+ ```bash
120
+ export DATABENTO_API_KEY="your_api_key_here"
121
+ ```
122
+
123
+ ### Parameterizing the Asset List
124
+
125
+ The list of supported futures contracts (the symbol registry) is dynamically loaded from a YAML configuration file, making it easy to add new markets without touching any Python code.
126
+
127
+ - **Adding a Market**: Simply edit `cotdata/src/cotdata/registry.yaml` and add the new market under its respective Asset Class (e.g., Copper or Cocoa). The system natively handles metadata such as `is_equity` and complex `hist_codes` structures.
128
+ - **Environment Override**: The `COTDATA_REGISTRY` environment variable allows you to point to a centralized `registry.yaml` file. For instance, you could place your `registry.yaml` inside your synced `$COTDATA_STORE`. This ensures both your Windows producer and Mac consumer are always looking at the exact same asset definitions, without needing to `git pull` the Python repository.
129
+
130
+ ## Atomic Store
131
+
132
+ The store uses **atomic writes**. Consumers can safely query the store via `get_prices` or `get_cot` even while `cotdata-update` is actively downloading and writing new data.
133
+
134
+ ## Diagnostics
135
+
136
+ You can verify your Norgate subscription's data quality and system configuration using the included smoke test script. Run it on your Windows producer machine:
137
+ ```bash
138
+ python tests/test_adjustment.py
139
+ ```
140
+ This diagnostic script tests:
141
+ 1. **Local Communication**: Verifies that Python can successfully communicate with the Norgate Data Updater running in the background.
142
+ 2. **Subscription Access**: Validates that your Norgate subscription is active and has the required CME futures data package enabled.
143
+ 3. **Roll Gap Validation**: Mathematically proves whether your Norgate Data Updater is configured globally to return back-adjusted or unadjusted continuous contracts. It hunts for artificial calendar-spread gaps at contract roll dates to ensure you are receiving gap-free, continuous data, which is absolutely vital for accurate stop-loss modeling.
144
+
145
+ ## COT Formats Explained
146
+
147
+ The CFTC publishes positioning data in three distinct formats. `cotdata` manages all three to ensure complete market coverage and the deepest possible historical backtesting.
148
+
149
+ 1. **Legacy Format (1986–Present)**
150
+ - **Scope:** All markets.
151
+ - **Categories:** Divides traders broadly into **Commercial** (hedgers) and **Non-Commercial** (large speculators).
152
+ - **Use Case:** This is the original format. While its broad categories make it less precise for modern analysis, it is the only format that provides data prior to 2006, making it essential for long-term historical backtesting.
153
+
154
+ 2. **Disaggregated Format (DIS) (2006–Present)**
155
+ - **Scope:** Physical commodities only (Agriculture, Energy, Metals).
156
+ - **Categories:** Splits traders into four granular groups: **Producer/Merchant** (classic hedgers), **Swap Dealers** (financial intermediaries), **Managed Money** (hedge funds / CTAs), and **Other Reportables**.
157
+ - **Use Case:** Provides a much clearer view of the "Smart Money" (Managed Money) in commodity markets.
158
+
159
+ 3. **Traders in Financial Futures (TFF) (2006–Present)**
160
+ - **Scope:** Financial markets only (Equities, Rates, Currencies).
161
+ - **Categories:** The financial counterpart to Disaggregated. Splits traders into: **Dealer/Intermediary** (sell-side), **Asset Manager** (pension/mutual funds), **Leveraged Funds** (hedge funds / CTAs), and **Other Reportables**.
162
+ - **Use Case:** The definitive source for tracking speculative flow (Leveraged Funds) in financial markets.
163
+
164
+ ## Data Schemas
165
+
166
+ The canonical store uses standard Parquet files. When loaded into a pandas DataFrame (e.g., via `pd.read_parquet()`), they conform to the following schemas.
167
+
168
+ ### Price Data (`prices/{symbol}_{adjustment}.parquet`)
169
+ The primary source for price history (Norgate Data). Indexed by tz-naive `Date`.
170
+
171
+ | Column | Type | Description |
172
+ |--------|------|-------------|
173
+ | `Date` | DatetimeIndex | Trading day (tz-naive, normalized to midnight). |
174
+ | `Open` | float | Opening price. |
175
+ | `High` | float | High price. |
176
+ | `Low` | float | Low price. |
177
+ | `Close` | float | Exchange settlement close. |
178
+ | `Volume` | float | Trading volume. |
179
+ | `Open Interest` | float | Total open interest. |
180
+ | `Delivery Month` | float | Expiration month of the active contract (e.g. `202609`). Used to detect contract rolls. |
181
+
182
+ ### COT Legacy Data (`cot_legacy/{code}.parquet`)
183
+ The primary source for Legacy positioning data (CFTC Legacy Futures Report). **History starts in 1986.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
184
+
185
+ > [!NOTE]
186
+ > **Legacy Reports**: The Legacy reports are broken down by exchange. These reports have a futures only report and a combined futures and options report. Legacy reports break down the reportable open interest positions into two classifications: non-commercial and commercial traders. The `cotdata` pipeline strictly downloads the **Futures-only** reports (located at `https://www.cftc.gov/files/dea/history/dea_fut_xls_{YEAR}.zip`).
187
+
188
+ > [!NOTE]
189
+ > **Column Subset**: While the raw CFTC `.xls` files contain [well over 100 columns](https://www.cftc.gov/MarketReports/CommitmentsofTraders/HistoricalViewable/cotvariableslegacy.html) (including spreading, concentration ratios, etc.), the producer pipeline explicitly discards them. The parquet files only maintain the exact 10-column subset listed below to keep the file sizes extremely small and strictly focused on what the downstream models require. To include additional data points from the raw reports, simply add the exact CFTC column name to the `TARGET_COLS` list inside `src/cotdata/providers/cftc.py`.
190
+
191
+ | Column | Type | Description |
192
+ |--------|------|-------------|
193
+ | `Report_Date_as_MM_DD_YYYY` | DatetimeIndex | Reporting date (typically Tuesday). |
194
+ | `Market_and_Exchange_Names` | string | Name of the contract and exchange. |
195
+ | `CFTC_Contract_Market_Code` | string | 6-digit CFTC contract code. |
196
+ | `Open_Interest_All` | float | Total open interest for the contract. |
197
+ | `Comm_Positions_Long_All` | float | Commercial Long positions. |
198
+ | `Comm_Positions_Short_All` | float | Commercial Short positions. |
199
+ | `NonComm_Positions_Long_All` | float | Non-Commercial (Large Speculator) Long positions. |
200
+ | `NonComm_Positions_Short_All` | float | Non-Commercial (Large Speculator) Short positions. |
201
+ | `NonRept_Positions_Long_All` | float | Non-Reportable (Small Speculator) Long positions. |
202
+ | `NonRept_Positions_Short_All` | float | Non-Reportable (Small Speculator) Short positions. |
203
+
204
+ ### COT Disaggregated Data (`cot_disagg/{code}.parquet`)
205
+ The primary source for entity-specific positioning and trader counts (CFTC Disaggregated Futures-Only Report). **History starts in 2006.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
206
+
207
+ > [!NOTE]
208
+ > **Lossless Image**: Unlike the Legacy schema which filters down to 10 specific columns, the Disaggregated parquets are a **lossless image** of the source CFTC `txt` files. They contain all granular entity groups (Money Manager, Swap Dealer, Producer/Merchant, Other Reportable) and their respective `Traders_*` counts (e.g., `Traders_Tot_All`, `Traders_M_Money_Long_All`). This is the required store for computing Position Size and Clustering metrics.
209
+
210
+ ### COT Traders in Financial Futures (TFF) Data (`cot_tff/{code}.parquet`)
211
+ The primary source for entity-specific positioning and trader counts for Financial markets (CFTC Traders in Financial Futures Futures-Only Report). **History starts in 2006.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
212
+
213
+ > [!NOTE]
214
+ > **Financials Counterpart**: TFF is the exact counterpart to Disaggregated reports, used exclusively for financial markets (Equities, FX, Rates) which do not have Disaggregated reports.
215
+
216
+ > [!NOTE]
217
+ > **Lossless Image**: Like Disaggregated, TFF parquets are a **lossless image** of the source CFTC `txt` files. They contain the financial entity groups (`Dealer`, `Asset_Mgr`, `Lev_Money`, `Other_Rept`) and their respective `Traders_*` counts. This is the required store for computing Position Size and Clustering metrics for financial assets.
@@ -0,0 +1,48 @@
1
+ [build-system]
2
+ requires = ["setuptools>=64", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "cotdata"
7
+ version = "0.1.0"
8
+ description = "Canonical data layer for COT/futures strategies — a producer/consumer split over a file-based store."
9
+ readme = "README.md"
10
+ authors = [
11
+ {name = "Matt S.", email = "cotanalyzer@gmail.com"}
12
+ ]
13
+ classifiers = [
14
+ "Programming Language :: Python :: 3",
15
+ "License :: OSI Approved :: MIT License",
16
+ "Operating System :: OS Independent",
17
+ ]
18
+ requires-python = ">=3.9"
19
+ dependencies = [
20
+ "pandas>=1.5",
21
+ "pyarrow>=10",
22
+ "requests>=2.28", # CFTC COT zips
23
+ "python-dateutil>=2.8",
24
+ "packaging", # required by norgatedata
25
+ "xlrd>=2.0", # CFTC dea_fut_xls are legacy .xls
26
+ "PyYAML>=6.0",
27
+ ]
28
+
29
+ [project.urls]
30
+ Homepage = "https://github.com/mspinola/cotdata"
31
+ Repository = "https://github.com/mspinola/cotdata"
32
+ Issues = "https://github.com/mspinola/cotdata/issues"
33
+
34
+ [project.optional-dependencies]
35
+ # Producer-only, Windows: the Norgate Data Updater must be installed & running.
36
+ norgate = ["norgatedata"]
37
+ # Dormant provider, retained for the intraday news-failure work + settlement cross-check.
38
+ databento = ["databento>=0.30"]
39
+ dev = ["pytest>=7"]
40
+
41
+ [project.scripts]
42
+ cotdata-update = "cotdata.update:main"
43
+
44
+ [tool.setuptools.packages.find]
45
+ where = ["src"]
46
+
47
+ [tool.setuptools.package-data]
48
+ cotdata = ["*.yaml"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,11 @@
1
+ """cotdata — canonical data layer (see README). Public consumer API."""
2
+ from .prices import get_prices, roll_dates
3
+ from .cot import get_cot
4
+ from .registry import symbol, all_symbols, REGISTRY, Symbol
5
+ from .store import load_manifest
6
+
7
+ __version__ = "0.1.0"
8
+ __all__ = [
9
+ "get_prices", "roll_dates", "get_cot",
10
+ "symbol", "all_symbols", "REGISTRY", "Symbol", "load_manifest",
11
+ ]
@@ -0,0 +1,35 @@
1
+ """Store location + schema version. The store is set via COTDATA_STORE."""
2
+ import os
3
+ from pathlib import Path
4
+
5
+ SCHEMA_VERSION = 1
6
+
7
+
8
+ def store_root() -> Path:
9
+ root = os.environ.get("COTDATA_STORE", "").strip()
10
+ if not root:
11
+ raise RuntimeError(
12
+ "COTDATA_STORE is not set. Point it at the shared data store "
13
+ "(the synced folder holding prices/, cot/, manifest.json)."
14
+ )
15
+ return Path(root)
16
+
17
+
18
+ def prices_dir() -> Path:
19
+ return store_root() / "prices"
20
+
21
+
22
+ def cot_legacy_dir() -> Path:
23
+ return store_root() / "cot_legacy"
24
+
25
+
26
+ def cot_disagg_dir() -> Path:
27
+ return store_root() / "cot_disagg"
28
+
29
+
30
+ def cot_tff_dir() -> Path:
31
+ return store_root() / "cot_tff"
32
+
33
+
34
+ def manifest_path() -> Path:
35
+ return store_root() / "manifest.json"
@@ -0,0 +1,62 @@
1
+ """Consumer COT API. Reads weekly CFTC positioning the producer wrote.
2
+
3
+ NOTE: the positioning-INDEX computation and the trading signals stay in
4
+ cot-analyzer (CotIndexer / metrics) — cotdata owns only the raw data. This
5
+ returns the cleaned weekly COT table; the indexer builds indices on top.
6
+ """
7
+ import pandas as pd
8
+
9
+ from . import store
10
+ from .registry import REGISTRY, hist_code_scales
11
+
12
+ _CODE_COL = "CFTC_Contract_Market_Code" # Legacy schema contract-code column
13
+
14
+
15
+ def get_cot(name: str, report: str = "legacy") -> pd.DataFrame:
16
+ """Read a symbol's weekly COT history with predecessor code stitching.
17
+
18
+ name: the internal pipeline symbol, e.g. 'ES' or 'GC'.
19
+ report: 'legacy' (default), 'disagg' (commodity futures only), or 'tff' (financial futures only).
20
+
21
+ Returns:
22
+ DataFrame indexed by date. The columns depend on the `report` requested:
23
+ - legacy: Open_Interest_All, NonComm_Positions_Long_All, Comm_Positions_Long_All, etc.
24
+ - disagg: Open_Interest_All, Prod_Merc_Positions_Long_All, M_Money_Positions_Long_All, etc.
25
+ - tff: Open_Interest_All, Dealer_Positions_Long_All, Lev_Money_Positions_Long_All, etc.
26
+ If no data exists, returns an empty DataFrame.
27
+ """
28
+ sym = REGISTRY.get(name)
29
+ if sym is None: # allow lookup by primary CFTC code, not just symbol
30
+ sym = next((s for s in REGISTRY.values() if s.cftc_code == name), None)
31
+
32
+ read_fn = {
33
+ "legacy": store.read_cot_legacy,
34
+ "disagg": store.read_cot_disagg,
35
+ "tff": store.read_cot_tff,
36
+ }.get(report)
37
+ if not read_fn:
38
+ raise ValueError(f"Unknown report type: {report}. Expected 'legacy', 'disagg', or 'tff'")
39
+
40
+ if sym is None or not sym.cftc_code:
41
+ return read_fn(name)
42
+ primary = read_fn(sym.cftc_code)
43
+ if not sym.hist_codes:
44
+ return primary
45
+ frames = [primary] # primary first → wins de-duplication on overlaps
46
+ for hc, scale in hist_code_scales(sym.hist_codes):
47
+ h = read_fn(hc)
48
+ if h.empty:
49
+ continue
50
+ h = h.copy()
51
+ if scale != 1.0: # bridge a contract-size change (position/OI counts)
52
+ num = h.select_dtypes("number").columns
53
+ h[num] = h[num] * scale
54
+ if _CODE_COL in h.columns: # present predecessor rows under the primary code
55
+ h[_CODE_COL] = sym.cftc_code
56
+ frames.append(h)
57
+ frames = [f for f in frames if not f.empty]
58
+ if not frames:
59
+ return primary
60
+ combined = pd.concat(frames)
61
+ combined = combined[~combined.index.duplicated(keep="first")].sort_index()
62
+ return combined