deeptick 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deeptick-0.1.0/.gitignore +68 -0
- deeptick-0.1.0/PKG-INFO +43 -0
- deeptick-0.1.0/README.md +19 -0
- deeptick-0.1.0/pyproject.toml +43 -0
- deeptick-0.1.0/src/deeptick/__init__.py +6 -0
- deeptick-0.1.0/src/deeptick/client.py +450 -0
- deeptick-0.1.0/src/deeptick/py.typed +1 -0
- deeptick-0.1.0/tests/test_client.py +216 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.egg-info/
|
|
6
|
+
dist/
|
|
7
|
+
build/
|
|
8
|
+
*.egg
|
|
9
|
+
|
|
10
|
+
# Virtual environments
|
|
11
|
+
.venv/
|
|
12
|
+
venv/
|
|
13
|
+
env/
|
|
14
|
+
|
|
15
|
+
# IDE
|
|
16
|
+
.idea/
|
|
17
|
+
.vscode/
|
|
18
|
+
*.swp
|
|
19
|
+
*.swo
|
|
20
|
+
.DS_Store
|
|
21
|
+
|
|
22
|
+
# Data (large Parquet files — don't commit)
|
|
23
|
+
data/
|
|
24
|
+
*.parquet
|
|
25
|
+
*.sqlite3
|
|
26
|
+
*.sqlite3-*
|
|
27
|
+
|
|
28
|
+
# Logs
|
|
29
|
+
*.log
|
|
30
|
+
|
|
31
|
+
# Environment
|
|
32
|
+
.env
|
|
33
|
+
.env.local
|
|
34
|
+
|
|
35
|
+
# JavaScript / Next.js
|
|
36
|
+
node_modules/
|
|
37
|
+
.next/
|
|
38
|
+
web/node_modules/
|
|
39
|
+
web/.next/
|
|
40
|
+
web/out/
|
|
41
|
+
web/.env*.local
|
|
42
|
+
*.tsbuildinfo
|
|
43
|
+
|
|
44
|
+
# Rust build output
|
|
45
|
+
client-rs/target/
|
|
46
|
+
|
|
47
|
+
# Local visual test artifacts
|
|
48
|
+
output/playwright/
|
|
49
|
+
|
|
50
|
+
# Terraform local working directories
|
|
51
|
+
.terraform/
|
|
52
|
+
**/.terraform/
|
|
53
|
+
*.tfstate
|
|
54
|
+
*.tfstate.*
|
|
55
|
+
|
|
56
|
+
# Operational artifacts with expiring signed URLs
|
|
57
|
+
reports/sample-links-*.md
|
|
58
|
+
reports/collector-symbol-status-*.json
|
|
59
|
+
reports/collector-symbol-status-*.md
|
|
60
|
+
reports/hyperliquid-12h-sample-*.md
|
|
61
|
+
reports/options-10m-samples-*.md
|
|
62
|
+
dist-sdk/
|
|
63
|
+
.feature-credentials.env
|
|
64
|
+
|
|
65
|
+
# Rust build output (sdk copy; client-rs/target already ignored above)
|
|
66
|
+
sdk/rust/target/
|
|
67
|
+
# Shadow-gate run outputs (host cron writes these; history lives in S3/logs)
|
|
68
|
+
reports/shadow-gate-*
|
deeptick-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: deeptick
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: DeepTick Python SDK for authenticated market data downloads
|
|
5
|
+
Project-URL: Homepage, https://deeptick.lacertalabs.xyz
|
|
6
|
+
Project-URL: Repository, https://github.com/deeptick/deeptick
|
|
7
|
+
Project-URL: Documentation, https://deeptick.lacertalabs.xyz/docs
|
|
8
|
+
Author: DeepTick
|
|
9
|
+
License: MIT
|
|
10
|
+
Keywords: crypto,deeptick,market-data,parquet,pyarrow
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Requires-Dist: pyarrow>=17.0
|
|
19
|
+
Requires-Dist: requests>=2.31
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
22
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# DeepTick Python SDK
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
from deeptick import CachedClient
|
|
29
|
+
|
|
30
|
+
client = CachedClient("dtk_your_key", "https://deeptick.dev")
|
|
31
|
+
trades = client.load_trades("lighter", "BTC-USD", "2026-07-08", "2026-07-08")
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The SDK uses the public DeepTick REST API under `/v1/*` and sends credentials
|
|
35
|
+
with the `X-API-Key` header.
|
|
36
|
+
|
|
37
|
+
Surface:
|
|
38
|
+
|
|
39
|
+
- `DeepTickClient(api_key, base_url="https://deeptick.dev")`
|
|
40
|
+
- `load_trades(exchange, symbol, start, end)` returns a `pyarrow.Table`
|
|
41
|
+
- `download(exchange, data_type, symbol, date, dest)` streams one partition to disk
|
|
42
|
+
- `entitlements()` and `catalog(exchange=None)` return API metadata
|
|
43
|
+
- `CachedClient` adds local Parquet caching under `~/.deeptick/cache`
|
deeptick-0.1.0/README.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# DeepTick Python SDK
|
|
2
|
+
|
|
3
|
+
```python
|
|
4
|
+
from deeptick import CachedClient
|
|
5
|
+
|
|
6
|
+
client = CachedClient("dtk_your_key", "https://deeptick.dev")
|
|
7
|
+
trades = client.load_trades("lighter", "BTC-USD", "2026-07-08", "2026-07-08")
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
The SDK uses the public DeepTick REST API under `/v1/*` and sends credentials
|
|
11
|
+
with the `X-API-Key` header.
|
|
12
|
+
|
|
13
|
+
Surface:
|
|
14
|
+
|
|
15
|
+
- `DeepTickClient(api_key, base_url="https://deeptick.dev")`
|
|
16
|
+
- `load_trades(exchange, symbol, start, end)` returns a `pyarrow.Table`
|
|
17
|
+
- `download(exchange, data_type, symbol, date, dest)` streams one partition to disk
|
|
18
|
+
- `entitlements()` and `catalog(exchange=None)` return API metadata
|
|
19
|
+
- `CachedClient` adds local Parquet caching under `~/.deeptick/cache`
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.25"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "deeptick"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "DeepTick Python SDK for authenticated market data downloads"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [{name = "DeepTick"}]
|
|
13
|
+
keywords = ["deeptick", "market-data", "crypto", "parquet", "pyarrow"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Financial and Insurance Industry",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
]
|
|
22
|
+
dependencies = [
|
|
23
|
+
"requests>=2.31",
|
|
24
|
+
"pyarrow>=17.0",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
dev = [
|
|
29
|
+
"pytest>=8.0",
|
|
30
|
+
"build>=1.2",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://deeptick.lacertalabs.xyz"
|
|
35
|
+
Repository = "https://github.com/deeptick/deeptick"
|
|
36
|
+
Documentation = "https://deeptick.lacertalabs.xyz/docs"
|
|
37
|
+
|
|
38
|
+
[tool.hatch.build.targets.wheel]
|
|
39
|
+
packages = ["src/deeptick"]
|
|
40
|
+
|
|
41
|
+
[tool.pytest.ini_options]
|
|
42
|
+
testpaths = ["tests"]
|
|
43
|
+
|
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import io
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
8
|
+
from datetime import date, datetime, timedelta
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Iterable
|
|
11
|
+
from urllib.parse import quote
|
|
12
|
+
|
|
13
|
+
import pyarrow as pa
|
|
14
|
+
import pyarrow.parquet as pq
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
DEFAULT_BASE_URL = "https://deeptick.dev"
|
|
18
|
+
USER_AGENT = "deeptick-python/0.1.0"
|
|
19
|
+
DEFAULT_CACHE_DIR = Path(os.path.expanduser("~/.deeptick/cache"))
|
|
20
|
+
DateLike = str | date | datetime
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class DeepTickAPIError(RuntimeError):
|
|
24
|
+
"""Raised when the DeepTick API returns an unexpected HTTP error."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _normalize_base_url(base_url: str | None) -> str:
|
|
28
|
+
value = (base_url or DEFAULT_BASE_URL).rstrip("/")
|
|
29
|
+
if value.endswith("/v1"):
|
|
30
|
+
value = value[:-3]
|
|
31
|
+
return value.rstrip("/")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _looks_like_url(value: str | None) -> bool:
|
|
35
|
+
return bool(value and value.startswith(("http://", "https://")))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _resolve_cached_client_args(
|
|
39
|
+
api_key: str | None,
|
|
40
|
+
base_url: str,
|
|
41
|
+
) -> tuple[str | None, str]:
|
|
42
|
+
if _looks_like_url(api_key):
|
|
43
|
+
if base_url == DEFAULT_BASE_URL:
|
|
44
|
+
return None, api_key or DEFAULT_BASE_URL
|
|
45
|
+
return base_url, api_key or DEFAULT_BASE_URL
|
|
46
|
+
return api_key, base_url
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _coerce_date(value: DateLike) -> date:
|
|
50
|
+
if isinstance(value, datetime):
|
|
51
|
+
return value.date()
|
|
52
|
+
if isinstance(value, date):
|
|
53
|
+
return value
|
|
54
|
+
try:
|
|
55
|
+
return datetime.strptime(value, "%Y-%m-%d").date()
|
|
56
|
+
except ValueError as exc:
|
|
57
|
+
raise ValueError(f"expected date as YYYY-MM-DD, got {value!r}") from exc
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _date_range(start: DateLike, end: DateLike | None) -> list[str]:
|
|
61
|
+
first = _coerce_date(start)
|
|
62
|
+
last = _coerce_date(end or start)
|
|
63
|
+
if last < first:
|
|
64
|
+
raise ValueError("end date must be on or after start date")
|
|
65
|
+
|
|
66
|
+
out: list[str] = []
|
|
67
|
+
current = first
|
|
68
|
+
while current <= last:
|
|
69
|
+
out.append(current.isoformat())
|
|
70
|
+
current += timedelta(days=1)
|
|
71
|
+
return out
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _columns_param(columns: Iterable[str] | None) -> str | None:
|
|
75
|
+
if columns is None:
|
|
76
|
+
return None
|
|
77
|
+
selected = [col for col in columns if col]
|
|
78
|
+
return ",".join(selected) if selected else None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _empty_table() -> pa.Table:
|
|
82
|
+
return pa.table({})
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _resolve_date_args(
|
|
86
|
+
start: DateLike | None,
|
|
87
|
+
end: DateLike | None,
|
|
88
|
+
*,
|
|
89
|
+
start_date: DateLike | None = None,
|
|
90
|
+
end_date: DateLike | None = None,
|
|
91
|
+
) -> tuple[DateLike, DateLike | None]:
|
|
92
|
+
if (
|
|
93
|
+
start is not None
|
|
94
|
+
and start_date is not None
|
|
95
|
+
and _coerce_date(start) != _coerce_date(start_date)
|
|
96
|
+
):
|
|
97
|
+
raise ValueError("pass either start or start_date, not both")
|
|
98
|
+
if (
|
|
99
|
+
end is not None
|
|
100
|
+
and end_date is not None
|
|
101
|
+
and _coerce_date(end) != _coerce_date(end_date)
|
|
102
|
+
):
|
|
103
|
+
raise ValueError("pass either end or end_date, not both")
|
|
104
|
+
|
|
105
|
+
resolved_start = start if start is not None else start_date
|
|
106
|
+
if resolved_start is None:
|
|
107
|
+
raise TypeError("missing required date: start")
|
|
108
|
+
return resolved_start, end if end is not None else end_date
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class DeepTickClient:
|
|
112
|
+
"""HTTP client for the DeepTick `/v1/*` REST API.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
api_key: API key sent in the `X-API-Key` header.
|
|
116
|
+
base_url: API origin, with or without a trailing `/v1`.
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
def __init__(
|
|
120
|
+
self,
|
|
121
|
+
api_key: str | None = None,
|
|
122
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
123
|
+
*,
|
|
124
|
+
session: requests.Session | None = None,
|
|
125
|
+
timeout: int = 120,
|
|
126
|
+
) -> None:
|
|
127
|
+
self.api_key = api_key or os.environ.get("DEEPTICK_API_KEY") or ""
|
|
128
|
+
self.base_url = _normalize_base_url(base_url)
|
|
129
|
+
self.session = session or requests.Session()
|
|
130
|
+
self.timeout = timeout
|
|
131
|
+
|
|
132
|
+
def catalog(self, exchange: str | None = None) -> dict[str, Any]:
|
|
133
|
+
"""Return the downloadable dataset catalog."""
|
|
134
|
+
params = {"exchange": exchange} if exchange else None
|
|
135
|
+
return self._json("/v1/catalog", params=params)
|
|
136
|
+
|
|
137
|
+
def entitlements(self) -> dict[str, Any]:
|
|
138
|
+
"""Return the presented API key's download entitlements."""
|
|
139
|
+
return self._json("/v1/entitlements")
|
|
140
|
+
|
|
141
|
+
def download(
|
|
142
|
+
self,
|
|
143
|
+
exchange: str,
|
|
144
|
+
data_type: str,
|
|
145
|
+
symbol: str,
|
|
146
|
+
date: DateLike,
|
|
147
|
+
dest: str | os.PathLike[str],
|
|
148
|
+
*,
|
|
149
|
+
columns: Iterable[str] | None = None,
|
|
150
|
+
format: str = "parquet",
|
|
151
|
+
) -> Path:
|
|
152
|
+
"""Download one daily partition to `dest` and return the final path."""
|
|
153
|
+
target = Path(dest)
|
|
154
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
155
|
+
tmp = target.with_name(f"{target.name}.part")
|
|
156
|
+
|
|
157
|
+
response = self._request(
|
|
158
|
+
"GET",
|
|
159
|
+
self._data_path(exchange, data_type, symbol, date),
|
|
160
|
+
params=self._data_params(format=format, columns=columns),
|
|
161
|
+
stream=True,
|
|
162
|
+
)
|
|
163
|
+
self._raise_for_status(response)
|
|
164
|
+
with tmp.open("wb") as fh:
|
|
165
|
+
for chunk in response.iter_content(chunk_size=256 * 1024):
|
|
166
|
+
if chunk:
|
|
167
|
+
fh.write(chunk)
|
|
168
|
+
tmp.replace(target)
|
|
169
|
+
return target
|
|
170
|
+
|
|
171
|
+
def load_trades(
|
|
172
|
+
self,
|
|
173
|
+
exchange: str,
|
|
174
|
+
symbol: str,
|
|
175
|
+
start: DateLike | None = None,
|
|
176
|
+
end: DateLike | None = None,
|
|
177
|
+
*,
|
|
178
|
+
start_date: DateLike | None = None,
|
|
179
|
+
end_date: DateLike | None = None,
|
|
180
|
+
columns: Iterable[str] | None = None,
|
|
181
|
+
) -> pa.Table:
|
|
182
|
+
"""Load trade data over an inclusive date range as a PyArrow table."""
|
|
183
|
+
return self.load(
|
|
184
|
+
"trades",
|
|
185
|
+
exchange,
|
|
186
|
+
symbol,
|
|
187
|
+
start,
|
|
188
|
+
end,
|
|
189
|
+
start_date=start_date,
|
|
190
|
+
end_date=end_date,
|
|
191
|
+
columns=columns,
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
def load(
|
|
195
|
+
self,
|
|
196
|
+
data_type: str,
|
|
197
|
+
exchange: str,
|
|
198
|
+
symbol: str,
|
|
199
|
+
start: DateLike | None = None,
|
|
200
|
+
end: DateLike | None = None,
|
|
201
|
+
*,
|
|
202
|
+
start_date: DateLike | None = None,
|
|
203
|
+
end_date: DateLike | None = None,
|
|
204
|
+
columns: Iterable[str] | None = None,
|
|
205
|
+
) -> pa.Table:
|
|
206
|
+
"""Load any DeepTick data type over an inclusive date range."""
|
|
207
|
+
start, end = _resolve_date_args(
|
|
208
|
+
start,
|
|
209
|
+
end,
|
|
210
|
+
start_date=start_date,
|
|
211
|
+
end_date=end_date,
|
|
212
|
+
)
|
|
213
|
+
tables = [
|
|
214
|
+
table
|
|
215
|
+
for day in _date_range(start, end)
|
|
216
|
+
if (table := self._load_day(exchange, data_type, symbol, day, columns=columns))
|
|
217
|
+
is not None
|
|
218
|
+
]
|
|
219
|
+
if not tables:
|
|
220
|
+
return _empty_table()
|
|
221
|
+
return pa.concat_tables(tables, promote_options="default")
|
|
222
|
+
|
|
223
|
+
def _load_day(
|
|
224
|
+
self,
|
|
225
|
+
exchange: str,
|
|
226
|
+
data_type: str,
|
|
227
|
+
symbol: str,
|
|
228
|
+
day: str,
|
|
229
|
+
*,
|
|
230
|
+
columns: Iterable[str] | None = None,
|
|
231
|
+
) -> pa.Table | None:
|
|
232
|
+
response = self._request(
|
|
233
|
+
"GET",
|
|
234
|
+
self._data_path(exchange, data_type, symbol, day),
|
|
235
|
+
params=self._data_params(format="parquet", columns=columns),
|
|
236
|
+
)
|
|
237
|
+
if response.status_code == 404:
|
|
238
|
+
return None
|
|
239
|
+
self._raise_for_status(response)
|
|
240
|
+
return pq.read_table(io.BytesIO(response.content))
|
|
241
|
+
|
|
242
|
+
def _json(self, path: str, *, params: dict[str, str] | None = None) -> dict[str, Any]:
|
|
243
|
+
response = self._request("GET", path, params=params)
|
|
244
|
+
self._raise_for_status(response)
|
|
245
|
+
data = response.json()
|
|
246
|
+
if not isinstance(data, dict):
|
|
247
|
+
raise DeepTickAPIError(f"expected object response from {path}")
|
|
248
|
+
return data
|
|
249
|
+
|
|
250
|
+
def _request(
|
|
251
|
+
self,
|
|
252
|
+
method: str,
|
|
253
|
+
path: str,
|
|
254
|
+
*,
|
|
255
|
+
params: dict[str, str] | None = None,
|
|
256
|
+
stream: bool = False,
|
|
257
|
+
) -> requests.Response:
|
|
258
|
+
return self.session.request(
|
|
259
|
+
method,
|
|
260
|
+
f"{self.base_url}{path}",
|
|
261
|
+
headers=self._headers(),
|
|
262
|
+
params=params,
|
|
263
|
+
timeout=self.timeout,
|
|
264
|
+
stream=stream,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
def _headers(self) -> dict[str, str]:
|
|
268
|
+
headers = {"User-Agent": USER_AGENT}
|
|
269
|
+
if self.api_key:
|
|
270
|
+
headers["X-API-Key"] = self.api_key
|
|
271
|
+
return headers
|
|
272
|
+
|
|
273
|
+
@staticmethod
|
|
274
|
+
def _raise_for_status(response: requests.Response) -> None:
|
|
275
|
+
try:
|
|
276
|
+
response.raise_for_status()
|
|
277
|
+
except requests.HTTPError as exc:
|
|
278
|
+
detail = response.text[:500] if response.text else response.reason
|
|
279
|
+
message = f"DeepTick API returned HTTP {response.status_code}: {detail}"
|
|
280
|
+
raise DeepTickAPIError(message) from exc
|
|
281
|
+
|
|
282
|
+
@staticmethod
|
|
283
|
+
def _data_path(
|
|
284
|
+
exchange: str,
|
|
285
|
+
data_type: str,
|
|
286
|
+
symbol: str,
|
|
287
|
+
day: DateLike,
|
|
288
|
+
) -> str:
|
|
289
|
+
return (
|
|
290
|
+
"/v1/data/"
|
|
291
|
+
f"{quote(exchange, safe='')}/"
|
|
292
|
+
f"{quote(data_type, safe='')}/"
|
|
293
|
+
f"{quote(symbol, safe='')}/"
|
|
294
|
+
f"{quote(_coerce_date(day).isoformat(), safe='')}"
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
@staticmethod
|
|
298
|
+
def _data_params(
|
|
299
|
+
*,
|
|
300
|
+
format: str = "parquet",
|
|
301
|
+
columns: Iterable[str] | None = None,
|
|
302
|
+
) -> dict[str, str]:
|
|
303
|
+
params = {"format": format}
|
|
304
|
+
if selected := _columns_param(columns):
|
|
305
|
+
params["columns"] = selected
|
|
306
|
+
return params
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
class CachedClient(DeepTickClient):
|
|
310
|
+
"""DeepTick client with local daily parquet caching.
|
|
311
|
+
|
|
312
|
+
The preferred positional order matches `DeepTickClient(api_key, base_url)`.
|
|
313
|
+
Existing site examples that pass `base_url` first or use keywords are also
|
|
314
|
+
accepted.
|
|
315
|
+
|
|
316
|
+
```python
|
|
317
|
+
from deeptick import CachedClient
|
|
318
|
+
client = CachedClient(api_key="dtk_your_key")
|
|
319
|
+
```
|
|
320
|
+
"""
|
|
321
|
+
|
|
322
|
+
def __init__(
|
|
323
|
+
self,
|
|
324
|
+
api_key: str | None = None,
|
|
325
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
326
|
+
*,
|
|
327
|
+
cache_dir: str | os.PathLike[str] = DEFAULT_CACHE_DIR,
|
|
328
|
+
max_workers: int = 8,
|
|
329
|
+
session: requests.Session | None = None,
|
|
330
|
+
timeout: int = 120,
|
|
331
|
+
) -> None:
|
|
332
|
+
api_key, base_url = _resolve_cached_client_args(api_key, base_url)
|
|
333
|
+
super().__init__(api_key=api_key, base_url=base_url, session=session, timeout=timeout)
|
|
334
|
+
self.cache_dir = Path(cache_dir)
|
|
335
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
336
|
+
self.max_workers = max_workers
|
|
337
|
+
self.cache_hits = 0
|
|
338
|
+
self.cache_misses = 0
|
|
339
|
+
self.bytes_downloaded = 0
|
|
340
|
+
self.bytes_from_cache = 0
|
|
341
|
+
|
|
342
|
+
def load(
|
|
343
|
+
self,
|
|
344
|
+
data_type: str,
|
|
345
|
+
exchange: str,
|
|
346
|
+
symbol: str,
|
|
347
|
+
start: DateLike | None = None,
|
|
348
|
+
end: DateLike | None = None,
|
|
349
|
+
*,
|
|
350
|
+
start_date: DateLike | None = None,
|
|
351
|
+
end_date: DateLike | None = None,
|
|
352
|
+
columns: Iterable[str] | None = None,
|
|
353
|
+
) -> pa.Table:
|
|
354
|
+
start, end = _resolve_date_args(start, end, start_date=start_date, end_date=end_date)
|
|
355
|
+
days = _date_range(start, end)
|
|
356
|
+
if len(days) == 1:
|
|
357
|
+
table = self._load_day_cached(exchange, data_type, symbol, days[0], columns=columns)
|
|
358
|
+
return table if table is not None else _empty_table()
|
|
359
|
+
|
|
360
|
+
tables: list[tuple[str, pa.Table]] = []
|
|
361
|
+
with ThreadPoolExecutor(max_workers=min(self.max_workers, len(days))) as pool:
|
|
362
|
+
futures = {
|
|
363
|
+
pool.submit(
|
|
364
|
+
self._load_day_cached,
|
|
365
|
+
exchange,
|
|
366
|
+
data_type,
|
|
367
|
+
symbol,
|
|
368
|
+
day,
|
|
369
|
+
columns=columns,
|
|
370
|
+
): day
|
|
371
|
+
for day in days
|
|
372
|
+
}
|
|
373
|
+
for future in as_completed(futures):
|
|
374
|
+
day = futures[future]
|
|
375
|
+
table = future.result()
|
|
376
|
+
if table is not None:
|
|
377
|
+
tables.append((day, table))
|
|
378
|
+
|
|
379
|
+
if not tables:
|
|
380
|
+
return _empty_table()
|
|
381
|
+
tables.sort(key=lambda item: item[0])
|
|
382
|
+
return pa.concat_tables([table for _, table in tables], promote_options="default")
|
|
383
|
+
|
|
384
|
+
def cache_stats(self) -> dict[str, Any]:
|
|
385
|
+
"""Return cache counters and current disk footprint."""
|
|
386
|
+
files = list(self.cache_dir.rglob("*.parquet"))
|
|
387
|
+
total_bytes = sum(path.stat().st_size for path in files)
|
|
388
|
+
total_requests = self.cache_hits + self.cache_misses
|
|
389
|
+
total_bytes_seen = self.bytes_from_cache + self.bytes_downloaded
|
|
390
|
+
return {
|
|
391
|
+
"cache_dir": str(self.cache_dir),
|
|
392
|
+
"cache_hits": self.cache_hits,
|
|
393
|
+
"cache_misses": self.cache_misses,
|
|
394
|
+
"hit_rate_pct": round(self.cache_hits / max(total_requests, 1) * 100, 1),
|
|
395
|
+
"cached_files": len(files),
|
|
396
|
+
"cache_size_mb": round(total_bytes / (1024 * 1024), 2),
|
|
397
|
+
"bytes_downloaded": self.bytes_downloaded,
|
|
398
|
+
"bytes_from_cache": self.bytes_from_cache,
|
|
399
|
+
"bandwidth_saved_pct": round(
|
|
400
|
+
self.bytes_from_cache / max(total_bytes_seen, 1) * 100,
|
|
401
|
+
1,
|
|
402
|
+
),
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
def clear_cache(self) -> None:
|
|
406
|
+
"""Remove all locally cached parquet files."""
|
|
407
|
+
shutil.rmtree(self.cache_dir, ignore_errors=True)
|
|
408
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
409
|
+
|
|
410
|
+
def _load_day_cached(
|
|
411
|
+
self,
|
|
412
|
+
exchange: str,
|
|
413
|
+
data_type: str,
|
|
414
|
+
symbol: str,
|
|
415
|
+
day: str,
|
|
416
|
+
*,
|
|
417
|
+
columns: Iterable[str] | None = None,
|
|
418
|
+
) -> pa.Table | None:
|
|
419
|
+
cache_path = self._cache_path(exchange, data_type, symbol, day, columns=columns)
|
|
420
|
+
if cache_path.exists() and cache_path.stat().st_size > 0:
|
|
421
|
+
self.cache_hits += 1
|
|
422
|
+
self.bytes_from_cache += cache_path.stat().st_size
|
|
423
|
+
return pq.read_table(cache_path)
|
|
424
|
+
|
|
425
|
+
self.cache_misses += 1
|
|
426
|
+
table = super()._load_day(exchange, data_type, symbol, day, columns=columns)
|
|
427
|
+
if table is None:
|
|
428
|
+
return None
|
|
429
|
+
|
|
430
|
+
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
|
431
|
+
tmp = cache_path.with_name(f"{cache_path.name}.part")
|
|
432
|
+
pq.write_table(table, tmp, compression="zstd")
|
|
433
|
+
tmp.replace(cache_path)
|
|
434
|
+
self.bytes_downloaded += cache_path.stat().st_size
|
|
435
|
+
return table
|
|
436
|
+
|
|
437
|
+
def _cache_path(
|
|
438
|
+
self,
|
|
439
|
+
exchange: str,
|
|
440
|
+
data_type: str,
|
|
441
|
+
symbol: str,
|
|
442
|
+
day: str,
|
|
443
|
+
*,
|
|
444
|
+
columns: Iterable[str] | None = None,
|
|
445
|
+
) -> Path:
|
|
446
|
+
safe_symbol = symbol.replace("/", "-").replace(":", "-")
|
|
447
|
+
suffix = ""
|
|
448
|
+
if selected := _columns_param(columns):
|
|
449
|
+
suffix = "_" + hashlib.md5(selected.encode("utf-8")).hexdigest()[:8]
|
|
450
|
+
return self.cache_dir / exchange / data_type / safe_symbol / f"{day}{suffix}.parquet"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import io
|
|
2
|
+
import json
|
|
3
|
+
|
|
4
|
+
import pyarrow as pa
|
|
5
|
+
import pyarrow.parquet as pq
|
|
6
|
+
import pytest
|
|
7
|
+
import requests
|
|
8
|
+
|
|
9
|
+
from deeptick import CachedClient, DeepTickAPIError, DeepTickClient
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class FakeResponse:
|
|
13
|
+
def __init__(self, status_code=200, content=b"", json_data=None, reason="OK"):
|
|
14
|
+
self.status_code = status_code
|
|
15
|
+
self.content = content
|
|
16
|
+
self._json_data = json_data
|
|
17
|
+
self.reason = reason
|
|
18
|
+
self.text = content.decode("utf-8", errors="replace") if content else ""
|
|
19
|
+
|
|
20
|
+
def json(self):
|
|
21
|
+
if self._json_data is not None:
|
|
22
|
+
return self._json_data
|
|
23
|
+
return json.loads(self.content)
|
|
24
|
+
|
|
25
|
+
def raise_for_status(self):
|
|
26
|
+
if self.status_code >= 400:
|
|
27
|
+
error = requests.HTTPError(f"{self.status_code} {self.reason}")
|
|
28
|
+
error.response = self
|
|
29
|
+
raise error
|
|
30
|
+
|
|
31
|
+
def iter_content(self, chunk_size=1):
|
|
32
|
+
for offset in range(0, len(self.content), chunk_size):
|
|
33
|
+
yield self.content[offset : offset + chunk_size]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class FakeSession:
|
|
37
|
+
def __init__(self, routes):
|
|
38
|
+
self.routes = routes
|
|
39
|
+
self.calls = []
|
|
40
|
+
|
|
41
|
+
def request(self, method, url, headers=None, params=None, timeout=None, stream=False):
|
|
42
|
+
self.calls.append(
|
|
43
|
+
{
|
|
44
|
+
"method": method,
|
|
45
|
+
"url": url,
|
|
46
|
+
"headers": headers or {},
|
|
47
|
+
"params": params,
|
|
48
|
+
"timeout": timeout,
|
|
49
|
+
"stream": stream,
|
|
50
|
+
}
|
|
51
|
+
)
|
|
52
|
+
key = (method, url, tuple(sorted((params or {}).items())))
|
|
53
|
+
try:
|
|
54
|
+
return self.routes[key]
|
|
55
|
+
except KeyError:
|
|
56
|
+
return FakeResponse(404, b"not found", reason="Not Found")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def parquet_bytes(values):
|
|
60
|
+
table = pa.table(values)
|
|
61
|
+
buf = io.BytesIO()
|
|
62
|
+
pq.write_table(table, buf, compression="zstd")
|
|
63
|
+
return buf.getvalue()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_metadata_requests_include_api_key():
|
|
67
|
+
session = FakeSession(
|
|
68
|
+
{
|
|
69
|
+
("GET", "https://example.test/v1/entitlements", ()): FakeResponse(
|
|
70
|
+
json_data={"plan": "pilot"}
|
|
71
|
+
),
|
|
72
|
+
(
|
|
73
|
+
"GET",
|
|
74
|
+
"https://example.test/v1/catalog",
|
|
75
|
+
(("exchange", "lighter"),),
|
|
76
|
+
): FakeResponse(json_data={"datasets": []}),
|
|
77
|
+
}
|
|
78
|
+
)
|
|
79
|
+
client = DeepTickClient("dtk_test", "https://example.test", session=session)
|
|
80
|
+
|
|
81
|
+
assert client.entitlements() == {"plan": "pilot"}
|
|
82
|
+
assert client.catalog("lighter") == {"datasets": []}
|
|
83
|
+
assert all(call["headers"]["X-API-Key"] == "dtk_test" for call in session.calls)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_load_trades_returns_concatenated_pyarrow_table():
|
|
87
|
+
day1 = parquet_bytes({"price": [1.0], "amount": [2.0]})
|
|
88
|
+
day2 = parquet_bytes({"price": [3.0], "amount": [4.0]})
|
|
89
|
+
session = FakeSession(
|
|
90
|
+
{
|
|
91
|
+
(
|
|
92
|
+
"GET",
|
|
93
|
+
"https://example.test/v1/data/lighter/trades/BTC-USD/2026-07-01",
|
|
94
|
+
(("format", "parquet"),),
|
|
95
|
+
): FakeResponse(content=day1),
|
|
96
|
+
(
|
|
97
|
+
"GET",
|
|
98
|
+
"https://example.test/v1/data/lighter/trades/BTC-USD/2026-07-02",
|
|
99
|
+
(("format", "parquet"),),
|
|
100
|
+
): FakeResponse(content=day2),
|
|
101
|
+
}
|
|
102
|
+
)
|
|
103
|
+
client = DeepTickClient("dtk_test", "https://example.test/v1", session=session)
|
|
104
|
+
|
|
105
|
+
table = client.load_trades("lighter", "BTC-USD", "2026-07-01", "2026-07-02")
|
|
106
|
+
|
|
107
|
+
assert table.num_rows == 2
|
|
108
|
+
assert table.column("price").to_pylist() == [1.0, 3.0]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_load_accepts_site_date_keyword_aliases():
|
|
112
|
+
payload = parquet_bytes({"price": [5.0], "amount": [1.25]})
|
|
113
|
+
session = FakeSession(
|
|
114
|
+
{
|
|
115
|
+
(
|
|
116
|
+
"GET",
|
|
117
|
+
"https://example.test/v1/data/lighter/trades/BTC-USD/2026-07-01",
|
|
118
|
+
(("format", "parquet"),),
|
|
119
|
+
): FakeResponse(content=payload),
|
|
120
|
+
}
|
|
121
|
+
)
|
|
122
|
+
client = DeepTickClient("dtk_test", "https://example.test", session=session)
|
|
123
|
+
|
|
124
|
+
table = client.load(
|
|
125
|
+
exchange="lighter",
|
|
126
|
+
data_type="trades",
|
|
127
|
+
symbol="BTC-USD",
|
|
128
|
+
start_date="2026-07-01",
|
|
129
|
+
end_date="2026-07-01",
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
assert table.num_rows == 1
|
|
133
|
+
assert table.column("amount").to_pylist() == [1.25]
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test_conflicting_date_aliases_are_rejected():
|
|
137
|
+
client = DeepTickClient("dtk_test", "https://example.test", session=FakeSession({}))
|
|
138
|
+
|
|
139
|
+
with pytest.raises(ValueError, match="start or start_date"):
|
|
140
|
+
client.load_trades(
|
|
141
|
+
"lighter",
|
|
142
|
+
"BTC-USD",
|
|
143
|
+
"2026-07-01",
|
|
144
|
+
start_date="2026-07-02",
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def test_download_streams_to_destination(tmp_path):
|
|
149
|
+
session = FakeSession(
|
|
150
|
+
{
|
|
151
|
+
(
|
|
152
|
+
"GET",
|
|
153
|
+
"https://example.test/v1/data/lighter/trades/BTC-USD/2026-07-01",
|
|
154
|
+
(("format", "parquet"),),
|
|
155
|
+
): FakeResponse(content=b"parquet-bytes"),
|
|
156
|
+
}
|
|
157
|
+
)
|
|
158
|
+
client = DeepTickClient("dtk_test", "https://example.test", session=session)
|
|
159
|
+
dest = tmp_path / "trades.parquet"
|
|
160
|
+
|
|
161
|
+
returned = client.download("lighter", "trades", "BTC-USD", "2026-07-01", dest)
|
|
162
|
+
|
|
163
|
+
assert returned == dest
|
|
164
|
+
assert dest.read_bytes() == b"parquet-bytes"
|
|
165
|
+
assert session.calls[0]["stream"] is True
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def test_cached_client_uses_disk_cache(tmp_path):
|
|
169
|
+
payload = parquet_bytes({"price": [10.0], "amount": [0.5]})
|
|
170
|
+
session = FakeSession(
|
|
171
|
+
{
|
|
172
|
+
(
|
|
173
|
+
"GET",
|
|
174
|
+
"https://example.test/v1/data/lighter/trades/BTC-USD/2026-07-01",
|
|
175
|
+
(("format", "parquet"),),
|
|
176
|
+
): FakeResponse(content=payload),
|
|
177
|
+
}
|
|
178
|
+
)
|
|
179
|
+
client = CachedClient("https://example.test", "dtk_test", cache_dir=tmp_path, session=session)
|
|
180
|
+
|
|
181
|
+
first = client.load_trades("lighter", "BTC-USD", "2026-07-01", "2026-07-01")
|
|
182
|
+
second = client.load_trades("lighter", "BTC-USD", "2026-07-01", "2026-07-01")
|
|
183
|
+
|
|
184
|
+
assert first.column("price").to_pylist() == [10.0]
|
|
185
|
+
assert second.column("price").to_pylist() == [10.0]
|
|
186
|
+
assert len(session.calls) == 1
|
|
187
|
+
assert client.cache_stats()["cache_hits"] == 1
|
|
188
|
+
assert client.cache_stats()["cache_misses"] == 1
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def test_cached_client_accepts_api_key_first_positional_args():
|
|
192
|
+
session = FakeSession(
|
|
193
|
+
{
|
|
194
|
+
("GET", "https://example.test/v1/entitlements", ()): FakeResponse(
|
|
195
|
+
json_data={"plan": "pilot"}
|
|
196
|
+
),
|
|
197
|
+
}
|
|
198
|
+
)
|
|
199
|
+
client = CachedClient("dtk_test", "https://example.test", session=session)
|
|
200
|
+
|
|
201
|
+
assert client.entitlements() == {"plan": "pilot"}
|
|
202
|
+
assert session.calls[0]["headers"]["X-API-Key"] == "dtk_test"
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def test_http_errors_raise_sdk_error():
|
|
206
|
+
session = FakeSession(
|
|
207
|
+
{
|
|
208
|
+
("GET", "https://example.test/v1/entitlements", ()): FakeResponse(
|
|
209
|
+
403, b"forbidden", reason="Forbidden"
|
|
210
|
+
),
|
|
211
|
+
}
|
|
212
|
+
)
|
|
213
|
+
client = DeepTickClient("bad", "https://example.test", session=session)
|
|
214
|
+
|
|
215
|
+
with pytest.raises(DeepTickAPIError):
|
|
216
|
+
client.entitlements()
|