everestapi 0.2.7__tar.gz → 0.2.8__tar.gz

{everestapi-0.2.7/src/everestapi.egg-info → everestapi-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: everestapi
-Version: 0.2.7
+Version: 0.2.8
 Summary: Python SDK for the Everesteer prediction tournament platform
 Author-email: Everesteer <support@everesteer.ai>
 License-Expression: MIT
@@ -116,16 +116,23 @@ everestapi submit --model my-model --file predictions.parquet
 
 ### Data & diagnostics
 
+The hackathon is a display-only diagnostics event. **Tune and self-score offline
+on the labeled validation set** (features + `target_*` columns), then **predict on
+the blind `eiq_live_2026` set and submit** — the leaderboard ranks your
+out-of-sample 2026 CORR on `target_everest_20`. In-sample fit is not rewarded. The
+labeled `eiq_live_2026` answers are held out server-side and are never downloadable.
+
 ```python
-api.download_dataset(universe="futures", split="train")
-api.download_benchmark(universe="futures", split="validation")
-api.get_dataset_info(universe="futures")
-api.get_diagnostics(model_id="my-model")
+# Labeled practice set — tune + self-score offline with everestapi.scoring:
+api.download_dataset(universe="futures", split="validation")
 
-# Validation diagnostics: eiq_validation_example_preds.parquet is the upload template
-# (same `id` set, your own `prediction` column).
-api.download_dataset(universe="futures", split="validation_example_preds")
+# Blind scored set (columns: exped, exped_date, instrument, id — no targets).
+# Predict on it, then submit; it is also the upload id template.
+api.download_dataset(universe="futures", split="live")
 api.submit_validation_diagnostics(model_id="my-model", predictions=df)
+
+api.get_dataset_info(universe="futures")
+api.get_diagnostics(model_id="my-model")
 ```
 
 ### Plotting (optional `viz` extra)

{everestapi-0.2.7 → everestapi-0.2.8}/README.md

@@ -77,16 +77,23 @@ everestapi submit --model my-model --file predictions.parquet
 
 ### Data & diagnostics
 
+The hackathon is a display-only diagnostics event. **Tune and self-score offline
+on the labeled validation set** (features + `target_*` columns), then **predict on
+the blind `eiq_live_2026` set and submit** — the leaderboard ranks your
+out-of-sample 2026 CORR on `target_everest_20`. In-sample fit is not rewarded. The
+labeled `eiq_live_2026` answers are held out server-side and are never downloadable.
+
 ```python
-api.download_dataset(universe="futures", split="train")
-api.download_benchmark(universe="futures", split="validation")
-api.get_dataset_info(universe="futures")
-api.get_diagnostics(model_id="my-model")
+# Labeled practice set — tune + self-score offline with everestapi.scoring:
+api.download_dataset(universe="futures", split="validation")
 
-# Validation diagnostics: eiq_validation_example_preds.parquet is the upload template
-# (same `id` set, your own `prediction` column).
-api.download_dataset(universe="futures", split="validation_example_preds")
+# Blind scored set (columns: exped, exped_date, instrument, id — no targets).
+# Predict on it, then submit; it is also the upload id template.
+api.download_dataset(universe="futures", split="live")
 api.submit_validation_diagnostics(model_id="my-model", predictions=df)
+
+api.get_dataset_info(universe="futures")
+api.get_diagnostics(model_id="my-model")
 ```
 
 ### Plotting (optional `viz` extra)

{everestapi-0.2.7 → everestapi-0.2.8}/src/everestapi/__init__.py

@@ -1,6 +1,6 @@
 """EverestAPI — Python SDK for the Everesteer prediction tournament platform."""
 
-__version__ = "0.2.7"
+__version__ = "0.2.8"
 
 from everestapi.client import EverestAPI, EverestError
 from everestapi.types import (

{everestapi-0.2.7 → everestapi-0.2.8}/src/everestapi/client.py

@@ -366,10 +366,14 @@ class EverestAPI:
         poll_interval: float = 2.0,
         timeout: float = 600.0,
     ) -> dict:
-        """POST /api/v1/diagnostics/upload (multipart) — score validation predictions.
+        """POST /api/v1/diagnostics/upload (multipart) — score out-of-sample predictions.
 
         ``predictions`` is a pandas DataFrame (``id`` + ``prediction`` columns) or a
-        path to a ``.parquet`` / ``.csv`` file. Returns the 202 accept dict; with
+        path to a ``.parquet`` / ``.csv`` file, generated on the blind eiq_live_2026
+        set (``download_dataset(split="live")``). The platform scores it server-side
+        against the held-out labeled eiq_live_2026 answers — those answers are never
+        downloadable. Tune and self-score offline on the labeled validation set first
+        (see :mod:`everestapi.scoring`). Returns the 202 accept dict; with
         ``wait=True`` polls ``runs/{upload_id}`` until ``done`` (returns the run) and
         raises :class:`EverestError` on ``failed`` or timeout. Display-only; results
         also surface in the website Validation Diagnostics rail.
@@ -590,11 +594,18 @@ class EverestAPI:
         404 the current version is resolved and the download retried once.
 
         Splits: ``train`` / ``validation`` (features + targets), ``live``
-        (current round features only).
+        (features only, no targets). In hackathon mode, ``validation`` is the
+        LABELED practice set — features + all ``target_*`` columns — that you tune
+        and self-score on offline (see :mod:`everestapi.scoring`), and ``live`` is
+        the BLIND eiq_live_2026 out-of-sample set (columns exactly ``exped``,
+        ``exped_date``, ``instrument``, ``id`` — no targets) that you predict on and
+        submit. The labeled eiq_live_2026 answers are held out server-side and never
+        downloadable.
 
         Futures is served by the futures endpoint (``version`` is a no-op):
-        it returns the real bregen tree to full-scope keys and the
-        target-stripped hackathon tree to hackathon-scoped keys (EVE-931).
+        it returns the real bregen tree to full-scope keys and the hackathon tree
+        (labeled ``validation`` practice set + blind ``live`` eiq_live_2026 scored
+        set) to hackathon-scoped keys.
         """
         if output_path is None:
             output_path = f"{universe}_{split}.parquet"
@@ -933,7 +944,8 @@ class EverestAPI:
 
         With a hackathon-scoped key there is no live round — the response is a
         diagnostics-mode payload (mode='diagnostics_hackathon') directing you to
-        download_dataset + submit_validation_diagnostics.
+        tune offline on the labeled validation set, then predict on the blind
+        eiq_live_2026 set and submit_validation_diagnostics.
         """
         return self._request(
             "GET",
@@ -951,7 +963,10 @@ class EverestAPI:
         """GET /api/v1/get_started — mode-aware orientation.
 
         Returns what to do next given this key's scope: the display-only
-        diagnostics-hackathon loop, or the live futures tournament flow.
+        diagnostics-hackathon loop (tune and self-score offline on the labeled
+        validation set, then predict on the blind eiq_live_2026 set and submit —
+        ranked on out-of-sample 2026 CORR on target_everest_20), or the live
+        futures tournament flow.
         """
         return self._request("GET", "/api/v1/get_started")
 

{everestapi-0.2.7 → everestapi-0.2.8}/src/everestapi/mcp/server.py

@@ -238,7 +238,7 @@ TOOLS = [
     },
     {
         "name": "download_dataset",
-        "description": "Download a dataset split (train/validation/live) as a parquet file. Returns the local file path.",
+        "description": "Download a dataset split (train/validation/live) as a parquet file. Returns the local file path. In hackathon mode: 'validation' is the LABELED practice set (features + all target_* columns) for offline tuning and self-scoring; 'live' is the BLIND eiq_live_2026 out-of-sample set (columns exactly exped, exped_date, instrument, id — NO targets) that you predict on and submit. The labeled 2026 answers are held out server-side and never downloadable.",
         "inputSchema": {
             "type": "object",
             "properties": {
@@ -365,7 +365,7 @@ TOOLS = [
     },
     {
         "name": "get_current_round",
-        "description": "Get the current active round for a tournament (defaults to 'futures'; equities is unlaunched). With a hackathon key there is no live round — the response is a diagnostics-mode payload directing you to download_dataset + submit_validation_diagnostics.",
+        "description": "Get the current active round for a tournament (defaults to 'futures'; equities is unlaunched). With a hackathon key there is no live round — the response is a diagnostics-mode payload directing you to tune offline on the labeled validation set, then download the blind eiq_live_2026 set, predict, and submit_validation_diagnostics.",
         "inputSchema": {
             "type": "object",
             "properties": {
@@ -380,7 +380,7 @@ TOOLS = [
     },
     {
         "name": "get_started",
-        "description": "Mode-aware orientation: returns what to do next given your API key's scope. A hackathon key gets the display-only diagnostics loop (objective: maximize out-of-sample CORR on target_everest_20); a full key gets the live futures tournament flow. Start here.",
+        "description": "Mode-aware orientation: returns what to do next given your API key's scope. A hackathon key gets the display-only diagnostics loop (tune and self-score offline on the labeled validation set, then predict on the blind eiq_live_2026 set and submit; objective: maximize out-of-sample 2026 CORR on target_everest_20, in-sample fit is not rewarded); a full key gets the live futures tournament flow. Start here.",
         "inputSchema": {
             "type": "object",
             "properties": {},
@@ -546,13 +546,15 @@ TOOLS = [
     {
         "name": "submit_validation_diagnostics",
         "description": (
-            "Upload a NEW validation predictions file (parquet/CSV with id+prediction columns, "
-            "≤100 MB) and score it against eiq_validation.parquet. The required schema mirrors "
-            "eiq_validation_example_preds.parquet — fetch it with "
-            "download_dataset(split='validation_example_preds') and use it as the upload template "
-            "(same id set, your own prediction column). Use this only when you have fresh "
-            "predictions to score — to read the model's EXISTING latest result without re-scoring "
-            "(instant, no wait), call run_validation_diagnostics(model_id) instead. "
+            "Upload a NEW predictions file (parquet/CSV with id+prediction columns, "
+            "≤100 MB) generated on the blind eiq_live_2026 out-of-sample set; the platform "
+            "scores it server-side against the held-out labeled eiq_live_2026 answers (never "
+            "downloadable). The required id set is the blind eiq_live_2026 set — fetch it with "
+            "download_dataset(split='live') and use it as the upload template "
+            "(its id column, your own prediction column). Tune and self-score offline on the "
+            "labeled validation set first (see the scoring helper). Use this only when you have "
+            "fresh predictions to score — to read the model's EXISTING latest result without "
+            "re-scoring (instant, no wait), call run_validation_diagnostics(model_id) instead. "
             "The platform scores a 9-metric panel (CORR20, BMC, FNC, Sharpe, std dev, "
             "feature-exposure, Max Drawdown, autocorrelation, example-preds-corr) asynchronously "
             "and surfaces the run in the website Validation Diagnostics rail. By default "
@@ -594,9 +596,10 @@ TOOLS = [
     {
         "name": "get_diagnostics_leaderboard",
         "description": (
-            "Get the validation diagnostics leaderboard — global ranking of uploaded validation "
-            "runs by CORR20, BMC, FNC, and Sharpe. view='agents' (default) ranks participant model "
-            "runs and marks your own entries; view='benchmarks' ranks the official platform benchmarks."
+            "Get the validation diagnostics leaderboard — global ranking of agents by their "
+            "out-of-sample eiq_live_2026 CORR on target_everest_20 (in-sample fit is not rewarded). "
+            "view='agents' (default) ranks participant model runs and marks your own entries; "
+            "view='benchmarks' ranks the official platform benchmarks."
         ),
         "inputSchema": {
             "type": "object",
@@ -894,6 +897,12 @@ TOOLSETS: dict[str, set[str]] = {
         "get_scores",
         "get_leaderboard",
         "get_capabilities",
+        # EVE-967: the diagnostics upload + leaderboard are the entire hackathon
+        # flow get_started steers to; advertise them by default (core) so a
+        # hackathon key discovers them without setting EIQ_MCP_TOOLSETS. The rest
+        # of the diagnostics lifecycle stays in "diagnostics".
+        "submit_validation_diagnostics",
+        "get_diagnostics_leaderboard",
     },
     "data": {
         "get_universe",
@@ -911,9 +920,9 @@ TOOLSETS: dict[str, set[str]] = {
     },
     "submit": {"submit_predictions", "upload_model", "get_upload_status"},
     "diagnostics": {
+        # submit_validation_diagnostics + get_diagnostics_leaderboard promoted to
+        # "core" (EVE-967) — every name still lives in exactly one group.
         "run_validation_diagnostics",
-        "submit_validation_diagnostics",
-        "get_diagnostics_leaderboard",
     },
     "compute": {
         "quick_train",
@@ -1275,9 +1284,14 @@ def _error_payload(exc: Exception) -> str:
 
     if isinstance(exc, EverestError):
         code = exc.status_code
+        # FastAPI wraps errors as {"detail": {"code": ..., "message": ...}} —
+        # pull the machine code so 403 can distinguish a scope restriction
+        # (not available in diagnostics mode) from a genuine auth failure.
+        inner = exc.detail.get("detail") if isinstance(exc.detail, dict) else None
+        err_code = inner.get("code") if isinstance(inner, dict) else None
+
         hints = {
             401: "Auth failed — set EIQ_API_KEY to your Everesteer API key.",
-            403: "Auth failed — set EIQ_API_KEY to your Everesteer API key.",
             404: "Not found — check the model_id / upload_id exists.",
             409: "A diagnostics run is already in flight for this model — read it with "
             "run_validation_diagnostics(model_id) or wait for it to finish.",
@@ -1285,10 +1299,19 @@ def _error_payload(exc: Exception) -> str:
             504: "Still computing — scoring can take 15-20 min; poll again shortly.",
         }
         hint = hints.get(code)
+        if code == 403:
+            if err_code in ("scope_mismatch", "scope_restricted"):
+                hint = (
+                    "Not available in diagnostics (hackathon) mode — this is a "
+                    "live-tournament / full-scope action. Use the diagnostics flow: "
+                    "submit_validation_diagnostics + get_diagnostics_leaderboard."
+                )
+            else:
+                hint = "Auth failed — set EIQ_API_KEY to your Everesteer API key."
         if hint is None and 400 <= code < 500:
-            hint = "Bad request — file needs id+prediction columns and an existing model_id."
+            hint = "Bad request — check the arguments match the tool schema."
         elif hint is None and code >= 500:
-            hint = "Server error — retry; if it persists the run may have failed (e.g. predictions didn't overlap the validation universe)."
+            hint = "Server error — retry; if it persists the request may have failed."
         return json.dumps({"error": exc.detail, "status": code, "hint": hint})
     return json.dumps({"error": str(exc)})
 
@@ -1402,9 +1425,13 @@ _MCP_INSTRUCTIONS = (
     "Everesteer tournament MCP server. Call get_started first — it is mode-aware and tells you "
     "what to do next based on your API key's scope.\n\n"
     "Hackathon mode (a hackathon-scoped key): there is no live tournament round and no "
-    "staking or payout — this is a display-only diagnostics event. Your objective is to "
-    "maximize OUT-OF-SAMPLE CORR on target_everest_20; in-sample fit is not rewarded. Flow: "
-    "download_dataset -> score offline -> submit_validation_diagnostics -> "
+    "staking or payout — this is a display-only diagnostics event. Tune and self-score "
+    "offline on the LABELED validation set (features + target_* columns, via the scoring "
+    "helper), then predict on the BLIND eiq_live_2026 set (columns exped, exped_date, "
+    "instrument, id — no targets) and submit; the leaderboard ranks your OUT-OF-SAMPLE 2026 "
+    "CORR on target_everest_20. In-sample fit is not rewarded. Flow: "
+    "download_dataset(split='validation') -> tune + self-score offline -> "
+    "download_dataset(split='live') -> predict -> submit_validation_diagnostics -> "
     "get_diagnostics_leaderboard. Round/universe/feature/benchmark reads return empty "
     "diagnostics-mode payloads, not live data.\n\n"
     "Tournament mode (a full-scope key): submit daily futures predictions; payout = "

{everestapi-0.2.7 → everestapi-0.2.8}/src/everestapi/scoring.py

@@ -27,6 +27,14 @@ Quickstart::
         for e in val.exped.unique()
     ]
     print("mean CORR20:", sum(corrs) / len(corrs))
+
+In hackathon mode the validation set ships LABELED (features + ``target_*``
+columns), so this helper is the offline self-scoring path: use it to tune your
+model before you submit. It scores in-sample and is for tuning only — the
+official hackathon leaderboard scores your submitted predictions on the BLIND
+``eiq_live_2026`` out-of-sample set server-side (its labeled answers are held
+out and never downloadable), ranking your 2026 CORR on ``target_everest_20``.
+In-sample fit is not rewarded.
 """
 
 from __future__ import annotations

{everestapi-0.2.7 → everestapi-0.2.8/src/everestapi.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: everestapi
-Version: 0.2.7
+Version: 0.2.8
 Summary: Python SDK for the Everesteer prediction tournament platform
 Author-email: Everesteer <support@everesteer.ai>
 License-Expression: MIT
@@ -116,16 +116,23 @@ everestapi submit --model my-model --file predictions.parquet
 
 ### Data & diagnostics
 
+The hackathon is a display-only diagnostics event. **Tune and self-score offline
+on the labeled validation set** (features + `target_*` columns), then **predict on
+the blind `eiq_live_2026` set and submit** — the leaderboard ranks your
+out-of-sample 2026 CORR on `target_everest_20`. In-sample fit is not rewarded. The
+labeled `eiq_live_2026` answers are held out server-side and are never downloadable.
+
 ```python
-api.download_dataset(universe="futures", split="train")
-api.download_benchmark(universe="futures", split="validation")
-api.get_dataset_info(universe="futures")
-api.get_diagnostics(model_id="my-model")
+# Labeled practice set — tune + self-score offline with everestapi.scoring:
+api.download_dataset(universe="futures", split="validation")
 
-# Validation diagnostics: eiq_validation_example_preds.parquet is the upload template
-# (same `id` set, your own `prediction` column).
-api.download_dataset(universe="futures", split="validation_example_preds")
+# Blind scored set (columns: exped, exped_date, instrument, id — no targets).
+# Predict on it, then submit; it is also the upload id template.
+api.download_dataset(universe="futures", split="live")
 api.submit_validation_diagnostics(model_id="my-model", predictions=df)
+
+api.get_dataset_info(universe="futures")
+api.get_diagnostics(model_id="my-model")
 ```
 
 ### Plotting (optional `viz` extra)

{everestapi-0.2.7 → everestapi-0.2.8}/src/everestapi.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ tests/test_diagnostics.py
 tests/test_eve953_mcp_progress.py
 tests/test_eve957_mcp_annotations_resources.py
 tests/test_eve959_toolsets.py
+tests/test_eve967_mcp_discoverability.py
 tests/test_json_or_raise.py
 tests/test_mcp_and_models.py
 tests/test_prediction_range.py

everestapi-0.2.8/tests/test_eve967_mcp_discoverability.py

@@ -0,0 +1,40 @@
+"""EVE-967 — the diagnostics upload + leaderboard tools are discoverable on the
+default (core) toolset, and MCP error hints don't mislead:
+- a 403 scope restriction points at the diagnostics flow, not "rotate your key";
+- the generic 400 fallback is tool-neutral (no diagnostics-upload bleed).
+"""
+
+from __future__ import annotations
+
+import json
+
+import everestapi.mcp.server as S
+from everestapi.client import EverestError
+
+
+def test_diagnostics_tools_advertised_by_default(monkeypatch):
+    monkeypatch.delenv("EIQ_MCP_TOOLSETS", raising=False)
+    enabled = S._enabled_tool_names()
+    assert "submit_validation_diagnostics" in enabled
+    assert "get_diagnostics_leaderboard" in enabled
+
+
+def test_403_scope_mismatch_hint_points_to_diagnostics():
+    exc = EverestError(403, {"detail": {"code": "scope_mismatch", "message": "x"}})
+    payload = json.loads(S._error_payload(exc))
+    assert payload["status"] == 403
+    assert "diagnostics" in payload["hint"].lower()
+    assert "set EIQ_API_KEY" not in payload["hint"]
+
+
+def test_403_plain_auth_still_says_set_key():
+    exc = EverestError(403, "forbidden")  # no machine code → generic auth hint
+    payload = json.loads(S._error_payload(exc))
+    assert "EIQ_API_KEY" in payload["hint"]
+
+
+def test_generic_400_hint_is_tool_neutral():
+    exc = EverestError(400, {"detail": "bad"})
+    payload = json.loads(S._error_payload(exc))
+    assert "id+prediction" not in payload["hint"]
+    assert "schema" in payload["hint"].lower()