cotdata 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cotdata/__init__.py +11 -0
- cotdata/config.py +35 -0
- cotdata/cot.py +62 -0
- cotdata/prices.py +43 -0
- cotdata/providers/__init__.py +2 -0
- cotdata/providers/base.py +14 -0
- cotdata/providers/cftc.py +129 -0
- cotdata/providers/cftc_disagg.py +146 -0
- cotdata/providers/cftc_tff.py +146 -0
- cotdata/providers/databento.py +280 -0
- cotdata/providers/norgate.py +78 -0
- cotdata/registry.py +127 -0
- cotdata/registry.yaml +105 -0
- cotdata/store.py +96 -0
- cotdata/update.py +42 -0
- cotdata-0.1.0.dist-info/METADATA +246 -0
- cotdata-0.1.0.dist-info/RECORD +21 -0
- cotdata-0.1.0.dist-info/WHEEL +5 -0
- cotdata-0.1.0.dist-info/entry_points.txt +2 -0
- cotdata-0.1.0.dist-info/licenses/LICENSE +21 -0
- cotdata-0.1.0.dist-info/top_level.txt +1 -0
cotdata/store.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Canonical store I/O: atomic Parquet writes + a manifest. The store is the
|
|
2
|
+
contract between producers (write) and consumers (read)."""
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import tempfile
|
|
6
|
+
import datetime as dt
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from . import config
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _atomic_write_parquet(df: pd.DataFrame, path: Path) -> None:
|
|
15
|
+
"""Write to a temp file in the same dir, then os.replace — so a consumer
|
|
16
|
+
syncing/reading concurrently never sees a half-written parquet."""
|
|
17
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
18
|
+
fd, tmp = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp")
|
|
19
|
+
os.close(fd)
|
|
20
|
+
try:
|
|
21
|
+
df.to_parquet(tmp)
|
|
22
|
+
os.replace(tmp, path)
|
|
23
|
+
finally:
|
|
24
|
+
if os.path.exists(tmp):
|
|
25
|
+
os.remove(tmp)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# ── Prices ────────────────────────────────────────────────────────────────
|
|
29
|
+
def write_prices(symbol: str, adjustment: str, df: pd.DataFrame, source: str) -> None:
|
|
30
|
+
_atomic_write_parquet(df, config.prices_dir() / f"{symbol}_{adjustment}.parquet")
|
|
31
|
+
_touch_manifest("prices", f"{symbol}_{adjustment}", df, source)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def read_prices(symbol: str, adjustment: str) -> pd.DataFrame:
|
|
35
|
+
p = config.prices_dir() / f"{symbol}_{adjustment}.parquet"
|
|
36
|
+
return pd.read_parquet(p) if p.exists() else pd.DataFrame()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# ── COT Legacy ────────────────────────────────────────────────────────────
|
|
40
|
+
def write_cot_legacy(name: str, df: pd.DataFrame, source: str) -> None:
|
|
41
|
+
_atomic_write_parquet(df, config.cot_legacy_dir() / f"{name}.parquet")
|
|
42
|
+
_touch_manifest("cot_legacy", name, df, source)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def read_cot_legacy(name: str) -> pd.DataFrame:
|
|
46
|
+
p = config.cot_legacy_dir() / f"{name}.parquet"
|
|
47
|
+
return pd.read_parquet(p) if p.exists() else pd.DataFrame()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# ── COT Disaggregated ─────────────────────────────────────────────────────
|
|
51
|
+
def write_cot_disagg(name: str, df: pd.DataFrame, source: str) -> None:
|
|
52
|
+
_atomic_write_parquet(df, config.cot_disagg_dir() / f"{name}.parquet")
|
|
53
|
+
_touch_manifest("cot_disagg", name, df, source)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def read_cot_disagg(name: str) -> pd.DataFrame:
|
|
57
|
+
p = config.cot_disagg_dir() / f"{name}.parquet"
|
|
58
|
+
return pd.read_parquet(p) if p.exists() else pd.DataFrame()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# ── COT TFF (Traders in Financial Futures) ────────────────────────────────
|
|
62
|
+
def write_cot_tff(name: str, df: pd.DataFrame, source: str) -> None:
|
|
63
|
+
_atomic_write_parquet(df, config.cot_tff_dir() / f"{name}.parquet")
|
|
64
|
+
_touch_manifest("cot_tff", name, df, source)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def read_cot_tff(name: str) -> pd.DataFrame:
|
|
68
|
+
p = config.cot_tff_dir() / f"{name}.parquet"
|
|
69
|
+
return pd.read_parquet(p) if p.exists() else pd.DataFrame()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# ── Manifest ──────────────────────────────────────────────────────────────
|
|
74
|
+
def load_manifest() -> dict:
|
|
75
|
+
p = config.manifest_path()
|
|
76
|
+
if p.exists():
|
|
77
|
+
return json.loads(p.read_text())
|
|
78
|
+
return {"schema_version": config.SCHEMA_VERSION, "prices": {}, "cot_legacy": {}, "cot_disagg": {}, "cot_tff": {}}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _touch_manifest(kind: str, name: str, df: pd.DataFrame, source: str) -> None:
|
|
82
|
+
m = load_manifest()
|
|
83
|
+
last = None
|
|
84
|
+
if len(df) and isinstance(df.index, pd.DatetimeIndex):
|
|
85
|
+
last = str(df.index.max().date())
|
|
86
|
+
m.setdefault(kind, {})[name] = {
|
|
87
|
+
"last_date": last,
|
|
88
|
+
"n_rows": int(len(df)),
|
|
89
|
+
"source": source,
|
|
90
|
+
"updated_at": dt.datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
|
91
|
+
}
|
|
92
|
+
m["schema_version"] = config.SCHEMA_VERSION
|
|
93
|
+
tmp = config.manifest_path().with_suffix(".json.tmp")
|
|
94
|
+
tmp.parent.mkdir(parents=True, exist_ok=True)
|
|
95
|
+
tmp.write_text(json.dumps(m, indent=2, sort_keys=True))
|
|
96
|
+
os.replace(tmp, config.manifest_path())
|
cotdata/update.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Producer CLI: cotdata-update --prices --symbols ES NQ
|
|
2
|
+
cotdata-update --cot
|
|
3
|
+
Writes to $COTDATA_STORE. Schedule prices nightly (after the Norgate Data
|
|
4
|
+
Updater) and COT weekly (Friday, after the CFTC release)."""
|
|
5
|
+
import argparse
|
|
6
|
+
|
|
7
|
+
from . import config
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def main() -> None:
|
|
11
|
+
p = argparse.ArgumentParser(description="cotdata producer — fetch sources into the store.")
|
|
12
|
+
p.add_argument("--prices", action="store_true", help="Update Norgate price bars (Windows).")
|
|
13
|
+
p.add_argument("--cot-legacy", action="store_true", help="Update CFTC COT Legacy (cross-platform).")
|
|
14
|
+
p.add_argument("--cot-disagg", action="store_true", help="Update CFTC COT Disaggregated Futures-Only (cross-platform).")
|
|
15
|
+
p.add_argument("--cot-tff", action="store_true", help="Update Traders in Financial Futures (TFF) COT (cross-platform).")
|
|
16
|
+
p.add_argument("--cot-all", action="store_true", help="Update all COT pipelines (Legacy, Disagg, TFF).")
|
|
17
|
+
p.add_argument("--symbols", nargs="+", default=None, help="Internal symbols; default = all in registry.")
|
|
18
|
+
args = p.parse_args()
|
|
19
|
+
|
|
20
|
+
config.store_root() # fail fast if COTDATA_STORE unset
|
|
21
|
+
|
|
22
|
+
if not (args.prices or args.cot_legacy or args.cot_disagg or args.cot_tff or args.cot_all):
|
|
23
|
+
p.error("nothing to do — pass --prices, --cot-legacy, --cot-disagg, --cot-tff, or --cot-all")
|
|
24
|
+
|
|
25
|
+
if args.prices:
|
|
26
|
+
from .providers import norgate
|
|
27
|
+
norgate.update(symbols=args.symbols)
|
|
28
|
+
|
|
29
|
+
if args.cot_legacy or args.cot_all:
|
|
30
|
+
from .providers import cftc
|
|
31
|
+
cftc.update()
|
|
32
|
+
|
|
33
|
+
if args.cot_disagg or args.cot_all:
|
|
34
|
+
from .providers import cftc_disagg
|
|
35
|
+
cftc_disagg.update()
|
|
36
|
+
|
|
37
|
+
if args.cot_tff or args.cot_all:
|
|
38
|
+
from .providers import cftc_tff
|
|
39
|
+
cftc_tff.update()
|
|
40
|
+
|
|
41
|
+
if __name__ == "__main__":
|
|
42
|
+
main()
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cotdata
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Canonical data layer for COT/futures strategies — a producer/consumer split over a file-based store.
|
|
5
|
+
Author-email: "Matt S." <cotanalyzer@gmail.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/mspinola/cotdata
|
|
7
|
+
Project-URL: Repository, https://github.com/mspinola/cotdata
|
|
8
|
+
Project-URL: Issues, https://github.com/mspinola/cotdata/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.9
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: pandas>=1.5
|
|
16
|
+
Requires-Dist: pyarrow>=10
|
|
17
|
+
Requires-Dist: requests>=2.28
|
|
18
|
+
Requires-Dist: python-dateutil>=2.8
|
|
19
|
+
Requires-Dist: packaging
|
|
20
|
+
Requires-Dist: xlrd>=2.0
|
|
21
|
+
Requires-Dist: PyYAML>=6.0
|
|
22
|
+
Provides-Extra: norgate
|
|
23
|
+
Requires-Dist: norgatedata; extra == "norgate"
|
|
24
|
+
Provides-Extra: databento
|
|
25
|
+
Requires-Dist: databento>=0.30; extra == "databento"
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
# cotdata
|
|
31
|
+
|
|
32
|
+
[](https://pypi.org/project/cotdata/)
|
|
33
|
+
[](https://pypi.org/project/cotdata/)
|
|
34
|
+
[](https://opensource.org/licenses/MIT)
|
|
35
|
+
|
|
36
|
+
Canonical data layer for the COT / futures-strategy stack. It exists so that quantitative analysis toolset never fetch data directly. They read a shared, file-based store through a stable API.
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
PRODUCER (runs where each source is reachable)
|
|
40
|
+
Windows: Norgate export anywhere: CFTC COT download
|
|
41
|
+
│ writes │
|
|
42
|
+
▼ ▼
|
|
43
|
+
┌───────────────────────────────────────┐
|
|
44
|
+
│ CANONICAL STORE ($COTDATA_STORE) │
|
|
45
|
+
│ prices/*.parquet cot_legacy/*.parquet │ ← synced (rsync / Dropbox / S3)
|
|
46
|
+
│ cot_disagg/*.parquet manifest.json │
|
|
47
|
+
└────────────────────────────────────────────────┘
|
|
48
|
+
▲ ▲ reads (offline, cross-platform)
|
|
49
|
+
┌──────────────┴───┐ ┌────┴──────────────┐
|
|
50
|
+
│ COT analyzer │ │ analysis toolset │ both: import cotdata
|
|
51
|
+
└──────────────────┘ └───────────────────┘
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Installation
|
|
55
|
+
|
|
56
|
+
You can install `cotdata` directly from PyPI:
|
|
57
|
+
```bash
|
|
58
|
+
pip install cotdata
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
If you are running the **producer machine** on Windows to fetch Norgate data, install with the `norgate` extra:
|
|
62
|
+
```bash
|
|
63
|
+
pip install "cotdata[norgate]"
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Development
|
|
67
|
+
|
|
68
|
+
When using Norgate Data one must run this on a Windows machine as Norgate Updater requires Windows.
|
|
69
|
+
|
|
70
|
+
The downstream analysis toolset can run on any machine (Windows, Mac, Linux) as they only need to read data from the canonical store.
|
|
71
|
+
|
|
72
|
+
One can create a unified workspace by cloning all repositories into the same parent directory, and installing them into a single virtual environment.
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
uv venv # create .venv
|
|
76
|
+
uv pip install -e . # install cotdata & all deps
|
|
77
|
+
export COTDATA_STORE=/path/to/synced/store # the shared store
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
**Producer machine (Windows, Norgate)** — only cotdata + the `norgate` extra:
|
|
81
|
+
|
|
82
|
+
```powershell
|
|
83
|
+
uv venv --python 3.10 # Tested through norgate's supported python versions.
|
|
84
|
+
uv pip install -e ".[norgate]" # from the cotdata repo; pulls norgatedata
|
|
85
|
+
$env:COTDATA_STORE = "C:\path\to\store" # Path to output location
|
|
86
|
+
cotdata-update --prices --symbols ES NQ
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Use `uv run <cmd>` to run without activating, or activate with
|
|
90
|
+
`source .venv/bin/activate` (Mac) / `.venv\Scripts\activate` (Windows).
|
|
91
|
+
|
|
92
|
+
## The contract
|
|
93
|
+
|
|
94
|
+
The **store is the API boundary** — not Python imports. Producers write Parquet +
|
|
95
|
+
`manifest.json`; consumers only read. Nobody touches a vendor SDK at app runtime.
|
|
96
|
+
Swapping a vendor is a producer-only change.
|
|
97
|
+
|
|
98
|
+
- `prices/{symbol}_{adjustment}.parquet` — Open/High/Low/Close/Volume/Open Interest,
|
|
99
|
+
tz-naive `Date` index. `adjustment` ∈ {`backadj`, `unadj`}. Close = exchange settlement.
|
|
100
|
+
- `cot_legacy/{code}.parquet` — weekly CFTC Legacy positioning.
|
|
101
|
+
- `cot_disagg/{code}.parquet` — weekly CFTC Disaggregated positioning.
|
|
102
|
+
- `cot_tff/{code}.parquet` — weekly CFTC Traders in Financial Futures positioning.
|
|
103
|
+
- `manifest.json` — per-table `last_date`, `n_rows`, `source`, `updated_at`, `schema_version`.
|
|
104
|
+
|
|
105
|
+
## Consumer
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
import cotdata
|
|
109
|
+
df = cotdata.get_prices("ES", adjustment="backadj") # USE THIS FOR SIGNALS + STOPS
|
|
110
|
+
sz = cotdata.get_prices("ES", adjustment="unadj") # USE FOR POSITION SIZING / POINT VALUE
|
|
111
|
+
cot_legacy = cotdata.get_cot("ES", report="legacy") # USE FOR COMM/NON-COMM
|
|
112
|
+
cot_disagg = cotdata.get_cot("ES", report="disagg") # USE FOR TRADER COUNTS (MM/SD/OR)
|
|
113
|
+
cot_tff = cotdata.get_cot("ES", report="tff") # USE FOR TRADER COUNTS (FINANCIALS)
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Set `COTDATA_STORE` to the synced store directory.
|
|
117
|
+
|
|
118
|
+
**Predecessor Stitching & Scaling:** The `get_cot()` function doesn't just read a file; it dynamically stitches historical CFTC codes for contracts that migrated exchanges (like the Russell 2000) or rescales data for contracts that changed tick sizes (like Lumber). Downstream models see one clean, continuous asset.
|
|
119
|
+
|
|
120
|
+
## Producer (run on the machine that can reach the source)
|
|
121
|
+
|
|
122
|
+
```
|
|
123
|
+
COTDATA_STORE=/store cotdata-update --prices --symbols ES NQ # Norgate (Windows)
|
|
124
|
+
COTDATA_STORE=/store cotdata-update --cot-legacy # CFTC Legacy (cross-platform)
|
|
125
|
+
COTDATA_STORE=/store cotdata-update --cot-disagg # CFTC Disaggregated (cross-platform)
|
|
126
|
+
COTDATA_STORE=/store cotdata-update --cot-tff # CFTC Traders in Financial Futures (cross-platform)
|
|
127
|
+
COTDATA_STORE=/store cotdata-update --cot-all # Update all CFTC COT pipelines
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Schedule nightly (prices, after the Norgate Data Updater) and weekly (COT Friday releases).
|
|
131
|
+
|
|
132
|
+
## Design rules
|
|
133
|
+
|
|
134
|
+
### Why Back-Adjusted vs Unadjusted?
|
|
135
|
+
|
|
136
|
+
Futures contracts expire, forcing traders to "roll" into the next contract, which usually trades at a slightly different price. Simply stitching these contracts together creates artificial price gaps.
|
|
137
|
+
|
|
138
|
+
- **`backadj` (for signals & stops)**: Uses gap-free arithmetic rolls. This mathematically shifts historical prices backward to align with the new contract, preserving the *true shape* and percentage moves of the market. You must use this for technical indicators, trade signals, and stop-losses to avoid false triggers on rollover gaps.
|
|
139
|
+
- **`unadj` (for position sizing)**: Because back-adjustment shifts historical prices (sometimes into the negative), you cannot use it to calculate true dollar values. You must use `unadj` (raw, real-life) data for that exact day to calculate your true dollar risk and decide exactly how many contracts to buy.
|
|
140
|
+
|
|
141
|
+
### Providers & Authentication
|
|
142
|
+
|
|
143
|
+
**Norgate Data (Primary)**:
|
|
144
|
+
There are no API keys to configure in Python. The `norgatedata` Python package communicates locally with the Norgate Data Updater application. You simply need to have the Norgate Data Updater installed, authenticated, and running in the background on your Windows machine.
|
|
145
|
+
|
|
146
|
+
**Databento (Dormant/Intraday)**:
|
|
147
|
+
Databento is kept as a dormant provider because it works well and can be leveraged for intraday data. If you wish to use it, you must provide your API key via the `DATABENTO_API_KEY` environment variable:
|
|
148
|
+
```bash
|
|
149
|
+
export DATABENTO_API_KEY="your_api_key_here"
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
### Parameterizing the Asset List
|
|
153
|
+
|
|
154
|
+
The list of supported futures contracts (the symbol registry) is dynamically loaded from a YAML configuration file, making it easy to add new markets without touching any Python code.
|
|
155
|
+
|
|
156
|
+
- **Adding a Market**: Simply edit `cotdata/src/cotdata/registry.yaml` and add the new market under its respective Asset Class (e.g., Copper or Cocoa). The system natively handles metadata such as `is_equity` and complex `hist_codes` structures.
|
|
157
|
+
- **Environment Override**: The `COTDATA_REGISTRY` environment variable allows you to point to a centralized `registry.yaml` file. For instance, you could place your `registry.yaml` inside your synced `$COTDATA_STORE`. This ensures both your Windows producer and Mac consumer are always looking at the exact same asset definitions, without needing to `git pull` the Python repository.
|
|
158
|
+
|
|
159
|
+
## Atomic Store
|
|
160
|
+
|
|
161
|
+
The store uses **atomic writes**. Consumers can safely query the store via `get_prices` or `get_cot` even while `cotdata-update` is actively downloading and writing new data.
|
|
162
|
+
|
|
163
|
+
## Diagnostics
|
|
164
|
+
|
|
165
|
+
You can verify your Norgate subscription's data quality and system configuration using the included smoke test script. Run it on your Windows producer machine:
|
|
166
|
+
```bash
|
|
167
|
+
python tests/test_adjustment.py
|
|
168
|
+
```
|
|
169
|
+
This diagnostic script tests:
|
|
170
|
+
1. **Local Communication**: Verifies that Python can successfully communicate with the Norgate Data Updater running in the background.
|
|
171
|
+
2. **Subscription Access**: Validates that your Norgate subscription is active and has the required CME futures data package enabled.
|
|
172
|
+
3. **Roll Gap Validation**: Mathematically proves whether your Norgate Data Updater is configured globally to return back-adjusted or unadjusted continuous contracts. It hunts for artificial calendar-spread gaps at contract roll dates to ensure you are receiving gap-free, continuous data, which is absolutely vital for accurate stop-loss modeling.
|
|
173
|
+
|
|
174
|
+
## COT Formats Explained
|
|
175
|
+
|
|
176
|
+
The CFTC publishes positioning data in three distinct formats. `cotdata` manages all three to ensure complete market coverage and the deepest possible historical backtesting.
|
|
177
|
+
|
|
178
|
+
1. **Legacy Format (1986–Present)**
|
|
179
|
+
- **Scope:** All markets.
|
|
180
|
+
- **Categories:** Divides traders broadly into **Commercial** (hedgers) and **Non-Commercial** (large speculators).
|
|
181
|
+
- **Use Case:** This is the original format. While its broad categories make it less precise for modern analysis, it is the only format that provides data prior to 2006, making it essential for long-term historical backtesting.
|
|
182
|
+
|
|
183
|
+
2. **Disaggregated Format (DIS) (2006–Present)**
|
|
184
|
+
- **Scope:** Physical commodities only (Agriculture, Energy, Metals).
|
|
185
|
+
- **Categories:** Splits traders into four granular groups: **Producer/Merchant** (classic hedgers), **Swap Dealers** (financial intermediaries), **Managed Money** (hedge funds / CTAs), and **Other Reportables**.
|
|
186
|
+
- **Use Case:** Provides a much clearer view of the "Smart Money" (Managed Money) in commodity markets.
|
|
187
|
+
|
|
188
|
+
3. **Traders in Financial Futures (TFF) (2006–Present)**
|
|
189
|
+
- **Scope:** Financial markets only (Equities, Rates, Currencies).
|
|
190
|
+
- **Categories:** The financial counterpart to Disaggregated. Splits traders into: **Dealer/Intermediary** (sell-side), **Asset Manager** (pension/mutual funds), **Leveraged Funds** (hedge funds / CTAs), and **Other Reportables**.
|
|
191
|
+
- **Use Case:** The definitive source for tracking speculative flow (Leveraged Funds) in financial markets.
|
|
192
|
+
|
|
193
|
+
## Data Schemas
|
|
194
|
+
|
|
195
|
+
The canonical store uses standard Parquet files. When loaded into a pandas DataFrame (e.g., via `pd.read_parquet()`), they conform to the following schemas.
|
|
196
|
+
|
|
197
|
+
### Price Data (`prices/{symbol}_{adjustment}.parquet`)
|
|
198
|
+
The primary source for price history (Norgate Data). Indexed by tz-naive `Date`.
|
|
199
|
+
|
|
200
|
+
| Column | Type | Description |
|
|
201
|
+
|--------|------|-------------|
|
|
202
|
+
| `Date` | DatetimeIndex | Trading day (tz-naive, normalized to midnight). |
|
|
203
|
+
| `Open` | float | Opening price. |
|
|
204
|
+
| `High` | float | High price. |
|
|
205
|
+
| `Low` | float | Low price. |
|
|
206
|
+
| `Close` | float | Exchange settlement close. |
|
|
207
|
+
| `Volume` | float | Trading volume. |
|
|
208
|
+
| `Open Interest` | float | Total open interest. |
|
|
209
|
+
| `Delivery Month` | float | Expiration month of the active contract (e.g. `202609`). Used to detect contract rolls. |
|
|
210
|
+
|
|
211
|
+
### COT Legacy Data (`cot_legacy/{code}.parquet`)
|
|
212
|
+
The primary source for Legacy positioning data (CFTC Legacy Futures Report). **History starts in 1986.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
|
|
213
|
+
|
|
214
|
+
> [!NOTE]
|
|
215
|
+
> **Legacy Reports**: The Legacy reports are broken down by exchange. These reports have a futures only report and a combined futures and options report. Legacy reports break down the reportable open interest positions into two classifications: non-commercial and commercial traders. The `cotdata` pipeline strictly downloads the **Futures-only** reports (located at `https://www.cftc.gov/files/dea/history/dea_fut_xls_{YEAR}.zip`).
|
|
216
|
+
|
|
217
|
+
> [!NOTE]
|
|
218
|
+
> **Column Subset**: While the raw CFTC `.xls` files contain [well over 100 columns](https://www.cftc.gov/MarketReports/CommitmentsofTraders/HistoricalViewable/cotvariableslegacy.html) (including spreading, concentration ratios, etc.), the producer pipeline explicitly discards them. The parquet files only maintain the exact 10-column subset listed below to keep the file sizes extremely small and strictly focused on what the downstream models require. To include additional data points from the raw reports, simply add the exact CFTC column name to the `TARGET_COLS` list inside `src/cotdata/providers/cftc.py`.
|
|
219
|
+
|
|
220
|
+
| Column | Type | Description |
|
|
221
|
+
|--------|------|-------------|
|
|
222
|
+
| `Report_Date_as_MM_DD_YYYY` | DatetimeIndex | Reporting date (typically Tuesday). |
|
|
223
|
+
| `Market_and_Exchange_Names` | string | Name of the contract and exchange. |
|
|
224
|
+
| `CFTC_Contract_Market_Code` | string | 6-digit CFTC contract code. |
|
|
225
|
+
| `Open_Interest_All` | float | Total open interest for the contract. |
|
|
226
|
+
| `Comm_Positions_Long_All` | float | Commercial Long positions. |
|
|
227
|
+
| `Comm_Positions_Short_All` | float | Commercial Short positions. |
|
|
228
|
+
| `NonComm_Positions_Long_All` | float | Non-Commercial (Large Speculator) Long positions. |
|
|
229
|
+
| `NonComm_Positions_Short_All` | float | Non-Commercial (Large Speculator) Short positions. |
|
|
230
|
+
| `NonRept_Positions_Long_All` | float | Non-Reportable (Small Speculator) Long positions. |
|
|
231
|
+
| `NonRept_Positions_Short_All` | float | Non-Reportable (Small Speculator) Short positions. |
|
|
232
|
+
|
|
233
|
+
### COT Disaggregated Data (`cot_disagg/{code}.parquet`)
|
|
234
|
+
The primary source for entity-specific positioning and trader counts (CFTC Disaggregated Futures-Only Report). **History starts in 2006.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
|
|
235
|
+
|
|
236
|
+
> [!NOTE]
|
|
237
|
+
> **Lossless Image**: Unlike the Legacy schema which filters down to 10 specific columns, the Disaggregated parquets are a **lossless image** of the source CFTC `txt` files. They contain all granular entity groups (Money Manager, Swap Dealer, Producer/Merchant, Other Reportable) and their respective `Traders_*` counts (e.g., `Traders_Tot_All`, `Traders_M_Money_Long_All`). This is the required store for computing Position Size and Clustering metrics.
|
|
238
|
+
|
|
239
|
+
### COT Traders in Financial Futures (TFF) Data (`cot_tff/{code}.parquet`)
|
|
240
|
+
The primary source for entity-specific positioning and trader counts for Financial markets (CFTC Traders in Financial Futures Futures-Only Report). **History starts in 2006.** Indexed by tz-naive `Report_Date_as_MM_DD_YYYY`.
|
|
241
|
+
|
|
242
|
+
> [!NOTE]
|
|
243
|
+
> **Financials Counterpart**: TFF is the exact counterpart to Disaggregated reports, used exclusively for financial markets (Equities, FX, Rates) which do not have Disaggregated reports.
|
|
244
|
+
|
|
245
|
+
> [!NOTE]
|
|
246
|
+
> **Lossless Image**: Like Disaggregated, TFF parquets are a **lossless image** of the source CFTC `txt` files. They contain the financial entity groups (`Dealer`, `Asset_Mgr`, `Lev_Money`, `Other_Rept`) and their respective `Traders_*` counts. This is the required store for computing Position Size and Clustering metrics for financial assets.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
cotdata/__init__.py,sha256=QtIuvU_K5_fo46dPJzVbTsMQWUvzNMlrGngYjytVHGE,383
|
|
2
|
+
cotdata/config.py,sha256=6pDDy5HFjYQzLxWcxW0gHrZKvt8qBUV49SYop_zj6_Y,780
|
|
3
|
+
cotdata/cot.py,sha256=DDuQdtpLwFM92bxZ0E9o7OoJ8YBKlGi_F1qtjnRIC68,2640
|
|
4
|
+
cotdata/prices.py,sha256=AgnHN65VlHhSEkMM0MVv1CQcjlgDwvPepGilk2Cjn7k,1679
|
|
5
|
+
cotdata/registry.py,sha256=MhCvZV74Myq7YV9WD6qqE4ZWdo6oFGVQ27V0RHKQ34I,5601
|
|
6
|
+
cotdata/registry.yaml,sha256=qRREAoODAxPO6DfkmPGxOXk-DSW47wDWIdWmQW4O9TA,1434
|
|
7
|
+
cotdata/store.py,sha256=wondatMoOs69HxOli4mEys7w06pQV77fpyU9kZBaQ9Q,4158
|
|
8
|
+
cotdata/update.py,sha256=3aex4ZhexYccdkukUNp1mooM5HMyG0V7kgLoHiNwzIY,1837
|
|
9
|
+
cotdata/providers/__init__.py,sha256=rA1U_LpOZIv2AYjwOzonNiotp5xxqr7upjqchb82jI8,155
|
|
10
|
+
cotdata/providers/base.py,sha256=Z7RB78SQ8-uVJW-NYapSxOqZMhMukWRjGA2JdIyX9hc,468
|
|
11
|
+
cotdata/providers/cftc.py,sha256=kUMHpM4DXxSSfYFDHQPflSI5CGIztJUiHdlyqNC0Gks,4822
|
|
12
|
+
cotdata/providers/cftc_disagg.py,sha256=FPcnAkI3fpK3A75KlbPdz2zmEq-OF59-xOJJ1M2N7g4,5858
|
|
13
|
+
cotdata/providers/cftc_tff.py,sha256=JBnqyqa_Z3nteRN-mIjrT3qfAQce3lBziuD3-HsR7Mk,5784
|
|
14
|
+
cotdata/providers/databento.py,sha256=rdyaYYb3AJgMtLpIqLD0Bm0PPYORL4UJBl6m37kZuQw,13676
|
|
15
|
+
cotdata/providers/norgate.py,sha256=yzvsMN369W9r_9VDGz275a1cN2CqQ2FshMU5lMh1i54,3653
|
|
16
|
+
cotdata-0.1.0.dist-info/licenses/LICENSE,sha256=7UGGfJFjLux60Xp0IzQj66y2DJ5c9xTiFL_xSK2Tlgw,1064
|
|
17
|
+
cotdata-0.1.0.dist-info/METADATA,sha256=LOX-VhzCsejY16twS7YpWsd-asWDIlXs7BNja25S3QU,15903
|
|
18
|
+
cotdata-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
19
|
+
cotdata-0.1.0.dist-info/entry_points.txt,sha256=EDI4yu7fX63CkeXgP4P8f-fo4JvrngpMPVZnMFVsCms,55
|
|
20
|
+
cotdata-0.1.0.dist-info/top_level.txt,sha256=T9kcltz9Hf1hM-m9Jo8GJdMAFpqIjRGInyWVGuO8pnQ,8
|
|
21
|
+
cotdata-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Matt S.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cotdata
|