everestapi 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {everestapi-0.2.2 → everestapi-0.2.4}/LICENSE +1 -1
- {everestapi-0.2.2/src/everestapi.egg-info → everestapi-0.2.4}/PKG-INFO +37 -5
- {everestapi-0.2.2 → everestapi-0.2.4}/README.md +32 -2
- {everestapi-0.2.2 → everestapi-0.2.4}/pyproject.toml +5 -2
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi/__init__.py +2 -2
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi/cli.py +3 -3
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi/client.py +259 -59
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi/mcp/server.py +287 -18
- everestapi-0.2.4/src/everestapi/plots.py +70 -0
- {everestapi-0.2.2 → everestapi-0.2.4/src/everestapi.egg-info}/PKG-INFO +37 -5
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi.egg-info/SOURCES.txt +5 -1
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi.egg-info/requires.txt +3 -0
- everestapi-0.2.4/tests/test_diagnostics.py +162 -0
- everestapi-0.2.4/tests/test_json_or_raise.py +59 -0
- {everestapi-0.2.2 → everestapi-0.2.4}/tests/test_mcp_and_models.py +53 -0
- everestapi-0.2.4/tests/test_prediction_range.py +57 -0
- {everestapi-0.2.2 → everestapi-0.2.4}/setup.cfg +0 -0
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi/__main__.py +0 -0
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi/mcp/__init__.py +0 -0
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi/mcp/__main__.py +0 -0
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi/types.py +0 -0
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi.egg-info/dependency_links.txt +0 -0
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi.egg-info/entry_points.txt +0 -0
- {everestapi-0.2.2 → everestapi-0.2.4}/src/everestapi.egg-info/top_level.txt +0 -0
- {everestapi-0.2.2 → everestapi-0.2.4}/tests/test_cli.py +0 -0
- {everestapi-0.2.2 → everestapi-0.2.4}/tests/test_client.py +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: everestapi
|
|
3
|
-
Version: 0.2.
|
|
4
|
-
Summary: Python SDK for the
|
|
5
|
-
Author-email:
|
|
3
|
+
Version: 0.2.4
|
|
4
|
+
Summary: Python SDK for the Everesteer prediction tournament platform
|
|
5
|
+
Author-email: Everesteer <support@everesteer.ai>
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Homepage, https://everesteer.ai
|
|
8
8
|
Project-URL: Documentation, https://docs.everesteer.ai
|
|
@@ -26,11 +26,13 @@ Requires-Dist: pytest>=8.0; extra == "dev"
|
|
|
26
26
|
Requires-Dist: pytest-httpx>=0.34.0; extra == "dev"
|
|
27
27
|
Provides-Extra: mcp
|
|
28
28
|
Requires-Dist: mcp>=1.0; extra == "mcp"
|
|
29
|
+
Provides-Extra: viz
|
|
30
|
+
Requires-Dist: plotnine>=0.13; extra == "viz"
|
|
29
31
|
Dynamic: license-file
|
|
30
32
|
|
|
31
33
|
# everestapi
|
|
32
34
|
|
|
33
|
-
Python SDK for the [
|
|
35
|
+
Python SDK for the [Everesteer](https://everesteer.ai) prediction tournament platform.
|
|
34
36
|
|
|
35
37
|
```bash
|
|
36
38
|
pip install everestapi
|
|
@@ -109,9 +111,39 @@ api.download_dataset(universe="futures", split="train")
|
|
|
109
111
|
api.download_benchmark(universe="futures", split="validation")
|
|
110
112
|
api.get_dataset_info(universe="futures")
|
|
111
113
|
api.get_diagnostics(model_id="my-model")
|
|
114
|
+
|
|
115
|
+
# Validation diagnostics: eiq_validation_example_preds.parquet is the upload template
|
|
116
|
+
# (same `id` set, your own `prediction` column).
|
|
117
|
+
api.download_dataset(universe="futures", split="validation_example_preds")
|
|
112
118
|
api.submit_validation_diagnostics(model_id="my-model", predictions=df)
|
|
113
119
|
```
|
|
114
120
|
|
|
121
|
+
### Plotting (optional `viz` extra)
|
|
122
|
+
|
|
123
|
+
The `viz` extra installs [plotnine](https://plotnine.org/) (a grammar-of-graphics /
|
|
124
|
+
ggplot2 port). Use it to chart **anything** the SDK returns — scores, leaderboards,
|
|
125
|
+
per-exped series, validation panels. `everestapi.plots.plot_corr_curve` is just a
|
|
126
|
+
worked example; for any other chart, build it with plotnine directly.
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
pip install 'everestapi[viz]'
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
# Convenience helper — cumulative-CORR curve to a PNG:
|
|
134
|
+
from everestapi.plots import plot_corr_curve
|
|
135
|
+
corr = api.get_model_per_exped_breakdown(model_id="my-model")
|
|
136
|
+
plot_corr_curve(corr, output_path="corr_curve.png")
|
|
137
|
+
|
|
138
|
+
# Any other chart — plotnine on SDK data (matplotlib Agg backend, headless-safe):
|
|
139
|
+
import matplotlib; matplotlib.use("Agg")
|
|
140
|
+
import pandas as pd, plotnine as p9
|
|
141
|
+
lb = api.get_leaderboard(period="30d")
|
|
142
|
+
df = pd.DataFrame(lb["entries"])
|
|
143
|
+
(p9.ggplot(df, p9.aes("model_name", "total_payout")) + p9.geom_col()
|
|
144
|
+
+ p9.coord_flip()).save("leaderboard.png", verbose=False)
|
|
145
|
+
```
|
|
146
|
+
|
|
115
147
|
### Serverless compute
|
|
116
148
|
|
|
117
149
|
```python
|
|
@@ -178,7 +210,7 @@ with EverestAPI(api_key="...") as api:
|
|
|
178
210
|
|
|
179
211
|
## Disclaimers
|
|
180
212
|
|
|
181
|
-
- **Not financial advice.**
|
|
213
|
+
- **Not financial advice.** Everesteer tournaments are prediction competitions. Nothing in this SDK or on the platform constitutes investment advice, a solicitation, or a recommendation to buy or sell any financial instrument.
|
|
182
214
|
- **Testnet / beta.** The staking system and compute platform are in beta. Smart contract addresses, API endpoints, and payout mechanics may change without notice.
|
|
183
215
|
- **API stability.** This SDK targets API v1. Breaking changes will be communicated via the platform changelog and will follow semver once the SDK reaches 1.0.
|
|
184
216
|
- **Data is obfuscated.** All features and instrument identifiers served by the API are obfuscated. Attempting to reverse-engineer or de-obfuscate data violates the platform terms of service.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# everestapi
|
|
2
2
|
|
|
3
|
-
Python SDK for the [
|
|
3
|
+
Python SDK for the [Everesteer](https://everesteer.ai) prediction tournament platform.
|
|
4
4
|
|
|
5
5
|
```bash
|
|
6
6
|
pip install everestapi
|
|
@@ -79,9 +79,39 @@ api.download_dataset(universe="futures", split="train")
|
|
|
79
79
|
api.download_benchmark(universe="futures", split="validation")
|
|
80
80
|
api.get_dataset_info(universe="futures")
|
|
81
81
|
api.get_diagnostics(model_id="my-model")
|
|
82
|
+
|
|
83
|
+
# Validation diagnostics: eiq_validation_example_preds.parquet is the upload template
|
|
84
|
+
# (same `id` set, your own `prediction` column).
|
|
85
|
+
api.download_dataset(universe="futures", split="validation_example_preds")
|
|
82
86
|
api.submit_validation_diagnostics(model_id="my-model", predictions=df)
|
|
83
87
|
```
|
|
84
88
|
|
|
89
|
+
### Plotting (optional `viz` extra)
|
|
90
|
+
|
|
91
|
+
The `viz` extra installs [plotnine](https://plotnine.org/) (a grammar-of-graphics /
|
|
92
|
+
ggplot2 port). Use it to chart **anything** the SDK returns — scores, leaderboards,
|
|
93
|
+
per-exped series, validation panels. `everestapi.plots.plot_corr_curve` is just a
|
|
94
|
+
worked example; for any other chart, build it with plotnine directly.
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
pip install 'everestapi[viz]'
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
# Convenience helper — cumulative-CORR curve to a PNG:
|
|
102
|
+
from everestapi.plots import plot_corr_curve
|
|
103
|
+
corr = api.get_model_per_exped_breakdown(model_id="my-model")
|
|
104
|
+
plot_corr_curve(corr, output_path="corr_curve.png")
|
|
105
|
+
|
|
106
|
+
# Any other chart — plotnine on SDK data (matplotlib Agg backend, headless-safe):
|
|
107
|
+
import matplotlib; matplotlib.use("Agg")
|
|
108
|
+
import pandas as pd, plotnine as p9
|
|
109
|
+
lb = api.get_leaderboard(period="30d")
|
|
110
|
+
df = pd.DataFrame(lb["entries"])
|
|
111
|
+
(p9.ggplot(df, p9.aes("model_name", "total_payout")) + p9.geom_col()
|
|
112
|
+
+ p9.coord_flip()).save("leaderboard.png", verbose=False)
|
|
113
|
+
```
|
|
114
|
+
|
|
85
115
|
### Serverless compute
|
|
86
116
|
|
|
87
117
|
```python
|
|
@@ -148,7 +178,7 @@ with EverestAPI(api_key="...") as api:
|
|
|
148
178
|
|
|
149
179
|
## Disclaimers
|
|
150
180
|
|
|
151
|
-
- **Not financial advice.**
|
|
181
|
+
- **Not financial advice.** Everesteer tournaments are prediction competitions. Nothing in this SDK or on the platform constitutes investment advice, a solicitation, or a recommendation to buy or sell any financial instrument.
|
|
152
182
|
- **Testnet / beta.** The staking system and compute platform are in beta. Smart contract addresses, API endpoints, and payout mechanics may change without notice.
|
|
153
183
|
- **API stability.** This SDK targets API v1. Breaking changes will be communicated via the platform changelog and will follow semver once the SDK reaches 1.0.
|
|
154
184
|
- **Data is obfuscated.** All features and instrument identifiers served by the API are obfuscated. Attempting to reverse-engineer or de-obfuscate data violates the platform terms of service.
|
|
@@ -5,11 +5,11 @@ build-backend = "setuptools.build_meta"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "everestapi"
|
|
7
7
|
dynamic = ["version"]
|
|
8
|
-
description = "Python SDK for the
|
|
8
|
+
description = "Python SDK for the Everesteer prediction tournament platform"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
11
11
|
requires-python = ">=3.10"
|
|
12
|
-
authors = [{ name = "
|
|
12
|
+
authors = [{ name = "Everesteer", email = "support@everesteer.ai" }]
|
|
13
13
|
keywords = ["quant", "tournament", "prediction", "staking", "machine-learning"]
|
|
14
14
|
classifiers = [
|
|
15
15
|
"Development Status :: 4 - Beta",
|
|
@@ -31,6 +31,9 @@ dev = ["pytest>=8.0", "pytest-httpx>=0.34.0"]
|
|
|
31
31
|
# MCP server (`python -m everestapi.mcp`). Optional — the server falls back to a
|
|
32
32
|
# built-in JSON-RPC stdio loop when the official `mcp` SDK isn't installed.
|
|
33
33
|
mcp = ["mcp>=1.0"]
|
|
34
|
+
# Plotting helpers (`everestapi.plots`). Heavy — pulls matplotlib/pandas/numpy/
|
|
35
|
+
# scipy/statsmodels/mizani. Optional: the SDK and MCP server never import it.
|
|
36
|
+
viz = ["plotnine>=0.13"]
|
|
34
37
|
|
|
35
38
|
[project.scripts]
|
|
36
39
|
everestapi = "everestapi.cli:cli"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
"""EverestAPI — Python SDK for the
|
|
1
|
+
"""EverestAPI — Python SDK for the Everesteer prediction tournament platform."""
|
|
2
2
|
|
|
3
|
-
__version__ = "0.2.
|
|
3
|
+
__version__ = "0.2.4"
|
|
4
4
|
|
|
5
5
|
from everestapi.client import EverestAPI, EverestError
|
|
6
6
|
from everestapi.types import (
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""EverestAPI CLI — command-line interface for the
|
|
1
|
+
"""EverestAPI CLI — command-line interface for the Everesteer tournament platform.
|
|
2
2
|
|
|
3
3
|
Usage:
|
|
4
4
|
everestapi health
|
|
@@ -32,7 +32,7 @@ def _print_json(data: dict) -> None:
|
|
|
32
32
|
@click.group()
|
|
33
33
|
@click.version_option(__version__, prog_name="everestapi")
|
|
34
34
|
def cli() -> None:
|
|
35
|
-
"""
|
|
35
|
+
"""Everesteer tournament CLI."""
|
|
36
36
|
|
|
37
37
|
|
|
38
38
|
@cli.command()
|
|
@@ -134,7 +134,7 @@ def submit(model: str, file_path: str, tournament: str, api_key: str | None) ->
|
|
|
134
134
|
sys.exit(1)
|
|
135
135
|
|
|
136
136
|
_print_json(result)
|
|
137
|
-
except (EverestError, KeyError, FileNotFoundError) as e:
|
|
137
|
+
except (EverestError, KeyError, FileNotFoundError, ValueError) as e:
|
|
138
138
|
click.echo(f"Error: {e}", err=True)
|
|
139
139
|
sys.exit(1)
|
|
140
140
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""EverestAPI — Python client for the
|
|
1
|
+
"""EverestAPI — Python client for the Everesteer prediction tournament platform.
|
|
2
2
|
|
|
3
3
|
Usage::
|
|
4
4
|
|
|
@@ -95,21 +95,96 @@ def _open_no_symlink(path: str):
|
|
|
95
95
|
|
|
96
96
|
|
|
97
97
|
class EverestError(Exception):
|
|
98
|
-
"""Error returned by the
|
|
98
|
+
"""Error returned by the Everesteer API."""
|
|
99
99
|
|
|
100
100
|
def __init__(self, status_code: int, detail: Any) -> None:
|
|
101
101
|
self.status_code = status_code
|
|
102
102
|
self.detail = detail
|
|
103
|
-
super().__init__(f"
|
|
103
|
+
super().__init__(f"Everesteer API error {status_code}: {detail}")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _raise_for_error(resp: httpx.Response) -> None:
|
|
107
|
+
"""Raise :class:`EverestError` for any >= 400 response.
|
|
108
|
+
|
|
109
|
+
Falls back to ``resp.text`` when the error body isn't JSON (e.g. an
|
|
110
|
+
S3/Cloudflare error page), so the caller gets the body instead of a
|
|
111
|
+
secondary JSON-decode failure.
|
|
112
|
+
"""
|
|
113
|
+
if resp.status_code >= 400:
|
|
114
|
+
try:
|
|
115
|
+
detail = resp.json()
|
|
116
|
+
except Exception:
|
|
117
|
+
detail = resp.text
|
|
118
|
+
raise EverestError(resp.status_code, detail)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _json_or_raise(resp: httpx.Response) -> dict:
|
|
122
|
+
"""Return parsed JSON, or raise a clear :class:`EverestError`.
|
|
123
|
+
|
|
124
|
+
Beyond the >= 400 check, this guards the *success* path: when an edge
|
|
125
|
+
proxy (e.g. Cloudflare Access) answers with a non-error status but an HTML
|
|
126
|
+
challenge page, a bare ``resp.json()`` blows up with a raw
|
|
127
|
+
``JSONDecodeError`` and a traceback. Detect that and raise an actionable
|
|
128
|
+
error pointing at the missing credentials instead.
|
|
129
|
+
"""
|
|
130
|
+
_raise_for_error(resp)
|
|
131
|
+
try:
|
|
132
|
+
return resp.json()
|
|
133
|
+
except Exception:
|
|
134
|
+
body = (resp.text or "").strip()
|
|
135
|
+
looks_like_cf = (
|
|
136
|
+
"<html" in body[:200].lower()
|
|
137
|
+
or "cloudflare" in body.lower()
|
|
138
|
+
or "AccessDenied" in body
|
|
139
|
+
)
|
|
140
|
+
hint = (
|
|
141
|
+
" — this looks like a Cloudflare Access challenge; set "
|
|
142
|
+
"CF_ACCESS_CLIENT_ID / CF_ACCESS_CLIENT_SECRET (and EIQ_API_KEY)"
|
|
143
|
+
if looks_like_cf
|
|
144
|
+
else ""
|
|
145
|
+
)
|
|
146
|
+
raise EverestError(
|
|
147
|
+
resp.status_code,
|
|
148
|
+
f"expected a JSON response but received non-JSON content{hint}: "
|
|
149
|
+
f"{body[:200]}",
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _check_prediction_range(
|
|
154
|
+
values: dict[str, float], *, lo: float = 0.0, hi: float = 1.0
|
|
155
|
+
) -> None:
|
|
156
|
+
"""Raise ``ValueError`` if any prediction is non-finite or outside ``[lo, hi]``.
|
|
157
|
+
|
|
158
|
+
Mirrors the server-side futures bound so a bad submission fails fast
|
|
159
|
+
locally with a clear message instead of a rejected round-trip.
|
|
160
|
+
"""
|
|
161
|
+
import math
|
|
162
|
+
|
|
163
|
+
bad: list[str] = []
|
|
164
|
+
for key, val in values.items():
|
|
165
|
+
try:
|
|
166
|
+
f = float(val)
|
|
167
|
+
except (TypeError, ValueError):
|
|
168
|
+
bad.append(f"{key}={val!r}")
|
|
169
|
+
continue
|
|
170
|
+
if not math.isfinite(f) or f < lo or f > hi:
|
|
171
|
+
bad.append(f"{key}={val}")
|
|
172
|
+
if bad:
|
|
173
|
+
preview = ", ".join(bad[:5])
|
|
174
|
+
more = f" (+{len(bad) - 5} more)" if len(bad) > 5 else ""
|
|
175
|
+
raise ValueError(
|
|
176
|
+
f"{len(bad)} prediction(s) must be finite and within "
|
|
177
|
+
f"[{lo}, {hi}]: {preview}{more}"
|
|
178
|
+
)
|
|
104
179
|
|
|
105
180
|
|
|
106
181
|
class EverestAPI:
|
|
107
|
-
"""Synchronous client for the
|
|
182
|
+
"""Synchronous client for the Everesteer tournament API.
|
|
108
183
|
|
|
109
184
|
Parameters
|
|
110
185
|
----------
|
|
111
186
|
api_key : str, optional
|
|
112
|
-
|
|
187
|
+
Everesteer API key. Falls back to ``EIQ_API_KEY`` (or legacy ``EVEREST_API_KEY``) env var.
|
|
113
188
|
base_url : str, optional
|
|
114
189
|
Base URL for the API. Defaults to ``https://everesteer.ai`` (apex
|
|
115
190
|
domain), overridable via ``EVEREST_API_URL`` env var.
|
|
@@ -186,22 +261,11 @@ class EverestAPI:
|
|
|
186
261
|
|
|
187
262
|
def _request(self, method: str, path: str, **kwargs: Any) -> dict:
|
|
188
263
|
resp = self._client.request(method, path, **kwargs)
|
|
189
|
-
|
|
190
|
-
try:
|
|
191
|
-
detail = resp.json()
|
|
192
|
-
except Exception:
|
|
193
|
-
detail = resp.text
|
|
194
|
-
raise EverestError(resp.status_code, detail)
|
|
195
|
-
return resp.json()
|
|
264
|
+
return _json_or_raise(resp)
|
|
196
265
|
|
|
197
266
|
def _download(self, path: str, output_path: str) -> str:
|
|
198
267
|
resp = self._client.request("GET", path)
|
|
199
|
-
|
|
200
|
-
try:
|
|
201
|
-
detail = resp.json()
|
|
202
|
-
except Exception:
|
|
203
|
-
detail = resp.text
|
|
204
|
-
raise EverestError(resp.status_code, detail)
|
|
268
|
+
_raise_for_error(resp)
|
|
205
269
|
safe_path = _safe_output_path(output_path)
|
|
206
270
|
with _open_no_symlink(safe_path) as f:
|
|
207
271
|
f.write(resp.content)
|
|
@@ -267,56 +331,182 @@ class EverestAPI:
|
|
|
267
331
|
files={"file": (fname, f, "application/octet-stream")},
|
|
268
332
|
params={"model_id": model_name, "tournament": tourn},
|
|
269
333
|
)
|
|
334
|
+
return _json_or_raise(resp)
|
|
335
|
+
|
|
336
|
+
def submit_validation_diagnostics(
|
|
337
|
+
self,
|
|
338
|
+
model_id: str,
|
|
339
|
+
predictions,
|
|
340
|
+
tournament: str = "futures",
|
|
341
|
+
target: str = "target_everest_20",
|
|
342
|
+
*,
|
|
343
|
+
wait: bool = False,
|
|
344
|
+
poll_interval: float = 2.0,
|
|
345
|
+
timeout: float = 600.0,
|
|
346
|
+
) -> dict:
|
|
347
|
+
"""POST /api/v1/diagnostics/upload (multipart) — score validation predictions.
|
|
348
|
+
|
|
349
|
+
``predictions`` is a pandas DataFrame (``id`` + ``prediction`` columns) or a
|
|
350
|
+
path to a ``.parquet`` / ``.csv`` file. Returns the 202 accept dict; with
|
|
351
|
+
``wait=True`` polls ``runs/{upload_id}`` until ``done`` (returns the run) and
|
|
352
|
+
raises :class:`EverestError` on ``failed`` or timeout. Display-only; results
|
|
353
|
+
also surface in the website Validation Diagnostics rail.
|
|
354
|
+
"""
|
|
355
|
+
import io
|
|
356
|
+
import time
|
|
357
|
+
|
|
358
|
+
if hasattr(predictions, "to_parquet"): # DataFrame
|
|
359
|
+
buf = io.BytesIO()
|
|
360
|
+
predictions.to_parquet(buf)
|
|
361
|
+
data = buf.getvalue()
|
|
362
|
+
fname = "predictions.parquet"
|
|
363
|
+
else: # path
|
|
364
|
+
with open(predictions, "rb") as f:
|
|
365
|
+
data = f.read()
|
|
366
|
+
fname = str(predictions).rsplit("/", 1)[-1].rsplit("\\", 1)[-1]
|
|
367
|
+
|
|
368
|
+
resp = self._client.post(
|
|
369
|
+
"/api/v1/diagnostics/upload",
|
|
370
|
+
files={"file": (fname, data, "application/octet-stream")},
|
|
371
|
+
data={"model_id": model_id, "tournament": tournament, "target": target},
|
|
372
|
+
)
|
|
270
373
|
if resp.status_code >= 400:
|
|
271
374
|
try:
|
|
272
375
|
detail = resp.json()
|
|
273
376
|
except Exception:
|
|
274
377
|
detail = resp.text
|
|
275
378
|
raise EverestError(resp.status_code, detail)
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
379
|
+
accepted = resp.json()
|
|
380
|
+
if not wait:
|
|
381
|
+
return accepted
|
|
382
|
+
deadline = time.time() + timeout
|
|
383
|
+
while time.time() < deadline:
|
|
384
|
+
run = self.get_diagnostics_run(accepted["upload_id"])
|
|
385
|
+
if run.get("status") == "done":
|
|
386
|
+
return run
|
|
387
|
+
if run.get("status") == "failed":
|
|
388
|
+
raise EverestError(500, run.get("error") or "diagnostics run failed")
|
|
389
|
+
time.sleep(poll_interval)
|
|
390
|
+
raise EverestError(504, "diagnostics run timed out")
|
|
391
|
+
|
|
392
|
+
def get_diagnostics_run(self, upload_id: str) -> dict:
|
|
393
|
+
"""GET /api/v1/diagnostics/runs/{upload_id} — status + metrics for one run."""
|
|
394
|
+
return self._request("GET", f"/api/v1/diagnostics/runs/{upload_id}")
|
|
395
|
+
|
|
396
|
+
def get_diagnostics_leaderboard(
|
|
279
397
|
self,
|
|
280
|
-
|
|
281
|
-
predictions,
|
|
398
|
+
view: str = "agents",
|
|
282
399
|
tournament: str = "futures",
|
|
283
|
-
|
|
400
|
+
limit: int = 100,
|
|
284
401
|
) -> dict:
|
|
285
|
-
"""
|
|
286
|
-
|
|
287
|
-
|
|
402
|
+
"""GET /api/v1/diagnostics/leaderboard — global CORR20 ranking.
|
|
403
|
+
|
|
404
|
+
``view="agents"`` ranks participant model runs; ``view="benchmarks"`` ranks
|
|
405
|
+
the official benchmark models.
|
|
406
|
+
"""
|
|
407
|
+
return self._request(
|
|
408
|
+
"GET",
|
|
409
|
+
"/api/v1/diagnostics/leaderboard",
|
|
410
|
+
params={"view": view, "tournament": tournament, "limit": limit},
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
@staticmethod
|
|
414
|
+
def _fmt(v) -> str:
|
|
415
|
+
"""Format a metric value for console display."""
|
|
416
|
+
return "—" if v is None else f"{v:.4f}"
|
|
417
|
+
|
|
418
|
+
@staticmethod
|
|
419
|
+
def format_diagnostics_table(status: dict) -> str:
|
|
420
|
+
"""Render a run status dict (from get_diagnostics_run / wait=True) as a table."""
|
|
421
|
+
rows = [
|
|
422
|
+
("CORR20", status.get("corr20")),
|
|
423
|
+
("BMC", status.get("bmc")),
|
|
424
|
+
("FNC", status.get("fnc")),
|
|
425
|
+
("Sharpe Ratio", status.get("sharpe_ratio")),
|
|
426
|
+
("Std Dev", status.get("std_dev")),
|
|
427
|
+
("Max Drawdown", status.get("max_drawdown")),
|
|
428
|
+
("Autocorr", status.get("autocorrelation")),
|
|
429
|
+
("Example-preds Corr", status.get("example_preds_corr")),
|
|
430
|
+
]
|
|
431
|
+
lines = [
|
|
432
|
+
f"Validation Diagnostics — {status.get('model_id', '')} [{status.get('status', '')}]",
|
|
433
|
+
f"{'Metric':<20} {'Value':>10}",
|
|
434
|
+
"-" * 31,
|
|
435
|
+
]
|
|
436
|
+
lines += [f"{k:<20} {EverestAPI._fmt(v):>10}" for k, v in rows]
|
|
437
|
+
return "\n".join(lines)
|
|
438
|
+
|
|
439
|
+
@staticmethod
|
|
440
|
+
def format_leaderboard_table(lb: dict) -> str:
|
|
441
|
+
"""Render a get_diagnostics_leaderboard dict as a table (agents or benchmarks view)."""
|
|
442
|
+
view = lb.get("view", "agents")
|
|
443
|
+
if view == "benchmarks":
|
|
444
|
+
hdr = f"{'#':>3} {'Label':<18} {'CORR20':>9} {'BMC':>9} {'FNC':>9} {'Sharpe':>9}"
|
|
445
|
+
lines = [
|
|
446
|
+
f"Diagnostics Leaderboard (benchmarks) — {lb.get('tournament', '')}",
|
|
447
|
+
hdr,
|
|
448
|
+
"-" * len(hdr),
|
|
449
|
+
]
|
|
450
|
+
for e in lb.get("entries", []):
|
|
451
|
+
mark = "*" if e.get("is_ensemble") else " "
|
|
452
|
+
lines.append(
|
|
453
|
+
f"{e.get('rank', '?'):>3}{mark} {e.get('label', ''):<18} "
|
|
454
|
+
f"{EverestAPI._fmt(e.get('corr20')):>9} {EverestAPI._fmt(e.get('bmc')):>9} "
|
|
455
|
+
f"{EverestAPI._fmt(e.get('fnc')):>9} {EverestAPI._fmt(e.get('sharpe_ratio')):>9}"
|
|
456
|
+
)
|
|
288
457
|
else:
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
458
|
+
hdr = f"{'#':>3} {'Model':<14} {'Agent':<12} {'CORR20':>9} {'BMC':>9} {'FNC':>9} {'Sharpe':>9}"
|
|
459
|
+
lines = [
|
|
460
|
+
f"Diagnostics Leaderboard (agents) — {lb.get('tournament', '')} "
|
|
461
|
+
f"(your best rank: {lb.get('your_best_rank')})",
|
|
462
|
+
hdr,
|
|
463
|
+
"-" * len(hdr),
|
|
464
|
+
]
|
|
465
|
+
for e in lb.get("entries", []):
|
|
466
|
+
mark = "*" if e.get("is_self") else " "
|
|
467
|
+
lines.append(
|
|
468
|
+
f"{e.get('rank', '?'):>3}{mark} {e.get('model_name', ''):<14} "
|
|
469
|
+
f"{e.get('agent_name', ''):<12} "
|
|
470
|
+
f"{EverestAPI._fmt(e.get('corr20')):>9} {EverestAPI._fmt(e.get('bmc')):>9} "
|
|
471
|
+
f"{EverestAPI._fmt(e.get('fnc')):>9} {EverestAPI._fmt(e.get('sharpe_ratio')):>9}"
|
|
472
|
+
)
|
|
473
|
+
return "\n".join(lines)
|
|
474
|
+
|
|
475
|
+
def get_validation_panel(
|
|
476
|
+
self, model_id: str, days: int = 365, source: str = "auto"
|
|
477
|
+
) -> dict:
|
|
478
|
+
"""GET /api/v1/models/{model_id}/diagnostics/validation — full validation metrics.
|
|
297
479
|
|
|
298
|
-
|
|
299
|
-
|
|
480
|
+
``source="auto"`` (default) returns the latest completed validation-diagnostics
|
|
481
|
+
UPLOAD run — the same panel the website tab shows — falling back to tournament
|
|
482
|
+
scored-round history only when the model has never uploaded. ``source="scored"``
|
|
483
|
+
forces the tournament scored-round time-series panel.
|
|
484
|
+
"""
|
|
300
485
|
return self._request(
|
|
301
486
|
"GET",
|
|
302
487
|
f"/api/v1/models/{model_id}/diagnostics/validation",
|
|
303
|
-
params={"days": days},
|
|
488
|
+
params={"days": days, "source": source},
|
|
304
489
|
)
|
|
305
490
|
|
|
306
|
-
def get_validation_diagnostics(
|
|
491
|
+
def get_validation_diagnostics(
|
|
492
|
+
self, model_id: str, days: int = 365, source: str = "auto"
|
|
493
|
+
) -> dict:
|
|
307
494
|
"""Alias for :meth:`get_validation_panel` — full validation diagnostics panel."""
|
|
308
|
-
return self.get_validation_panel(model_id=model_id, days=days)
|
|
495
|
+
return self.get_validation_panel(model_id=model_id, days=days, source=source)
|
|
309
496
|
|
|
310
497
|
def get_model_per_exped_breakdown(
|
|
311
498
|
self,
|
|
312
499
|
model_id: str,
|
|
313
500
|
days: int = 3650,
|
|
314
501
|
) -> list[float]:
|
|
315
|
-
"""Return the per-exped CORR series for a model (oldest→newest).
|
|
502
|
+
"""Return the per-exped tournament CORR series for a model (oldest→newest).
|
|
316
503
|
|
|
317
|
-
Thin convenience wrapper over :meth:`get_validation_diagnostics
|
|
504
|
+
Thin convenience wrapper over :meth:`get_validation_diagnostics` with
|
|
505
|
+
``source="scored"`` — the model's tournament scored-round history (daily
|
|
506
|
+
expeds, sqrt(252)-annualisable). For the validation-UPLOAD panel instead,
|
|
507
|
+
call :meth:`get_validation_diagnostics` with the default ``source="auto"``.
|
|
318
508
|
"""
|
|
319
|
-
resp = self.get_validation_diagnostics(model_id, days=days)
|
|
509
|
+
resp = self.get_validation_diagnostics(model_id, days=days, source="scored")
|
|
320
510
|
return list(resp.get("per_exped_corr", []))
|
|
321
511
|
|
|
322
512
|
def get_job_output(
|
|
@@ -442,10 +632,12 @@ class EverestAPI:
|
|
|
442
632
|
model_id : str
|
|
443
633
|
Model identifier.
|
|
444
634
|
predictions : dict[str, float]
|
|
445
|
-
{instrument_id: prediction} for the primary target.
|
|
635
|
+
{instrument_id: prediction} for the primary target. Each prediction
|
|
636
|
+
must be finite and within ``[0, 1]``.
|
|
446
637
|
exped : str, optional
|
|
447
638
|
Exped identifier. Defaults to current round.
|
|
448
639
|
"""
|
|
640
|
+
_check_prediction_range(predictions)
|
|
449
641
|
body = {
|
|
450
642
|
"model_id": model_id,
|
|
451
643
|
"exped": exped or "current",
|
|
@@ -463,9 +655,14 @@ class EverestAPI:
|
|
|
463
655
|
) -> dict:
|
|
464
656
|
"""POST /api/v1/futures/submit — submit per-target predictions (legacy v1 format).
|
|
465
657
|
|
|
658
|
+
Each per-target prediction must be finite and within ``[0, 1]``.
|
|
659
|
+
|
|
466
660
|
.. deprecated::
|
|
467
661
|
Use :meth:`submit_futures_predictions` (v2) instead.
|
|
468
662
|
"""
|
|
663
|
+
_check_prediction_range(
|
|
664
|
+
{f"{iid}:{tgt}": v for iid, tv in predictions.items() for tgt, v in tv.items()}
|
|
665
|
+
)
|
|
469
666
|
preds_list = [{"instrument_id": k, "targets": v} for k, v in sorted(predictions.items())]
|
|
470
667
|
body = {"model_id": model_id, "exped": exped, "predictions": preds_list}
|
|
471
668
|
return self._request("POST", "/api/v1/futures/submit", json=body)
|
|
@@ -639,13 +836,7 @@ class EverestAPI:
|
|
|
639
836
|
timeout=self._client.timeout,
|
|
640
837
|
auth=self.basic_auth,
|
|
641
838
|
)
|
|
642
|
-
|
|
643
|
-
try:
|
|
644
|
-
detail = resp.json()
|
|
645
|
-
except Exception:
|
|
646
|
-
detail = resp.text
|
|
647
|
-
raise EverestError(resp.status_code, detail)
|
|
648
|
-
return resp.json()
|
|
839
|
+
return _json_or_raise(resp)
|
|
649
840
|
|
|
650
841
|
def create_model(
|
|
651
842
|
self,
|
|
@@ -710,19 +901,28 @@ class EverestAPI:
|
|
|
710
901
|
# -- model uploads ----------------------------------------------------
|
|
711
902
|
|
|
712
903
|
def upload_model(self, model_id: str, file_path: str) -> dict:
|
|
713
|
-
"""POST /api/v1/models/{model_id}/upload — upload a .pkl model file.
|
|
904
|
+
"""POST /api/v1/models/{model_id}/upload — upload a .pkl model file.
|
|
905
|
+
|
|
906
|
+
The platform runs the pickle in a sandbox that calls
|
|
907
|
+
``model.predict(features)`` each round, so the file MUST unpickle to one of:
|
|
908
|
+
|
|
909
|
+
* a fitted estimator with a ``.predict(X)`` method — sklearn / LightGBM /
|
|
910
|
+
XGBoost, saved with ``pickle.dump(model, f)``; or
|
|
911
|
+
* a stacked ensemble as a ``dict`` with a ``"meta"`` key, e.g.
|
|
912
|
+
``{"base_lgbm": est, "base_ridge": est, "meta": est}`` (every value must
|
|
913
|
+
have ``.predict``).
|
|
914
|
+
|
|
915
|
+
A bare ``dict`` without ``"meta"``, or a joblib/torch file renamed ``.pkl``,
|
|
916
|
+
fails the sandbox run. The upload itself succeeds (the run is async); poll
|
|
917
|
+
:meth:`get_upload_status` — on failure its ``error_message`` carries the
|
|
918
|
+
exact reason (e.g. "Model type dict has no predict() method").
|
|
919
|
+
"""
|
|
714
920
|
with open(file_path, "rb") as f:
|
|
715
921
|
resp = self._client.post(
|
|
716
922
|
f"/api/v1/models/{model_id}/upload",
|
|
717
923
|
files={"file": (f.name, f, "application/octet-stream")},
|
|
718
924
|
)
|
|
719
|
-
|
|
720
|
-
try:
|
|
721
|
-
detail = resp.json()
|
|
722
|
-
except Exception:
|
|
723
|
-
detail = resp.text
|
|
724
|
-
raise EverestError(resp.status_code, detail)
|
|
725
|
-
return resp.json()
|
|
925
|
+
return _json_or_raise(resp)
|
|
726
926
|
|
|
727
927
|
def get_upload_status(self, model_id: str) -> dict:
|
|
728
928
|
"""GET /api/v1/models/{model_id}/upload/status"""
|