hkjc-bench 0.1.0__tar.gz

hkjc_bench-0.1.0/MANIFEST.in

@@ -0,0 +1,3 @@
+exclude README.md
+prune data
+prune examples

hkjc_bench-0.1.0/PKG-INFO

@@ -0,0 +1,38 @@
+Metadata-Version: 2.4
+Name: hkjc-bench
+Version: 0.1.0
+Summary: Compare your HKJC model with the predictor's best model on the same races, via its model test API
+Project-URL: Homepage, https://hashedcookies.com
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=2.0
+
+# hkjc-bench
+
+Compare your Hong Kong Jockey Club race model with the [HKJC predictor](https://hashedcookies.com)'s best model
+(Model C), the current model and the betting market, on exactly the same races, without anyone's model code
+changing hands.
+
+You need an account on the predictor and an API key (dashboard → Profile → Model test API).
+
+```python
+from hkjc_bench import Client
+
+c = Client("hkjc_...")                  # or set HKJC_API_KEY
+c.status()                              # what the benchmark covers
+races = c.races(since="2024-09-01")     # races you can be scored on
+report = c.score(preds, basis="form")   # preds: DataFrame of race_key, horse_no, p_win for every runner
+print(report.summary())
+report.by_season()
+```
+
+- **basis="form"** compares with the models' form-only chances (for a model that doesn't use the odds);
+  **"blended"** with their final chances, which include the odds.
+- The benchmark is every race since 2010, each predicted by a version of the model trained only on earlier races;
+  scores are log-loss on the winner (lower is better), with the chance your model is really better.
+- Your predictions are scored in memory on the server and not stored.
+- `c.predictions(race_keys)` returns Model C's own chances only if an admin has allowed it for your key.
+
+Requires Python 3.10+ and pandas.

hkjc_bench-0.1.0/PYPI.md

@@ -0,0 +1,27 @@
+# hkjc-bench
+
+Compare your Hong Kong Jockey Club race model with the [HKJC predictor](https://hashedcookies.com)'s best model
+(Model C), the current model and the betting market, on exactly the same races, without anyone's model code
+changing hands.
+
+You need an account on the predictor and an API key (dashboard → Profile → Model test API).
+
+```python
+from hkjc_bench import Client
+
+c = Client("hkjc_...")                  # or set HKJC_API_KEY
+c.status()                              # what the benchmark covers
+races = c.races(since="2024-09-01")     # races you can be scored on
+report = c.score(preds, basis="form")   # preds: DataFrame of race_key, horse_no, p_win for every runner
+print(report.summary())
+report.by_season()
+```
+
+- **basis="form"** compares with the models' form-only chances (for a model that doesn't use the odds);
+  **"blended"** with their final chances, which include the odds.
+- The benchmark is every race since 2010, each predicted by a version of the model trained only on earlier races;
+  scores are log-loss on the winner (lower is better), with the chance your model is really better.
+- Your predictions are scored in memory on the server and not stored.
+- `c.predictions(race_keys)` returns Model C's own chances only if an admin has allowed it for your key.
+
+Requires Python 3.10+ and pandas.

hkjc_bench-0.1.0/hkjc_bench/__init__.py

@@ -0,0 +1,94 @@
+"""hkjc-bench: compare your model with the HKJC predictor's best model (Model C) on exactly the same races,
+without either side sharing model code.
+
+    from hkjc_bench import Client
+    c = Client("hkjc_...")                 # or set HKJC_API_KEY; make a key in the dashboard's Profile tab
+    c.status()                             # what the benchmark covers
+    races = c.races(since="2024-09-01")    # the races you can be scored on
+    report = c.score(preds, basis="form")  # preds: DataFrame with race_key, horse_no, p_win
+    print(report.summary())
+
+basis="form" compares with the models' form-only chances (right for a model that doesn't use the betting odds,
+like the kit's examples); basis="blended" with their final chances, which include the odds. Give chances for
+every runner in a race; races with missing runners are skipped. Your predictions are scored in memory on the
+server and not stored.
+"""
+import json
+import os
+import urllib.error
+import urllib.request
+
+import pandas as pd
+
+__version__ = "0.1.0"
+DEFAULT_URL = "https://hashedcookies.com/api/v1"
+
+
+class APIError(RuntimeError):
+    pass
+
+
+class Report(dict):
+    """The server's comparison, with helpers."""
+
+    def by_season(self) -> pd.DataFrame:
+        return pd.DataFrame(self.get("by_season", []))
+
+    def summary(self) -> str:
+        if not self.get("races"):
+            return f"No complete benchmark races in the upload (skipped: {self.get('skipped')})."
+        ll = self["logloss"]
+        better = self["chance_better_than_model_c"]
+        verdict = ("better than Model C" if self["vs_model_c"] < 0 and better >= 0.9 else
+                   "possibly better than Model C (not proven)" if self["vs_model_c"] < 0 else "not better than Model C")
+        return (f"{self['races']:,} races ({self['basis']} chances), log-loss on the winner, lower is better:\n"
+                f"  yours          {ll['yours']:.4f}\n  Model C        {ll['model_c']:.4f}\n"
+                f"  current model  {ll['current']:.4f}\n  market         {ll['market']:.4f}  (final odds)\n"
+                f"Yours vs Model C: {self['vs_model_c']:+.4f} -> {verdict} "
+                f"({better:.0%} chance it's really better; treat under 90% as not proven).")
+
+
+class Client:
+    def __init__(self, api_key: str | None = None, url: str | None = None, timeout: float = 300):
+        self.key = api_key or os.environ.get("HKJC_API_KEY")
+        if not self.key:
+            raise APIError("No API key: pass Client(key) or set HKJC_API_KEY (create one in the dashboard's Profile tab).")
+        self.url = (url or os.environ.get("HKJC_API_URL") or DEFAULT_URL).rstrip("/")
+        self.timeout = timeout
+
+    def _call(self, path: str, body: dict | None = None) -> dict:
+        req = urllib.request.Request(self.url + path, data=None if body is None else json.dumps(body).encode(),
+                                     headers={"Authorization": f"Bearer {self.key}", "Content-Type": "application/json",
+                                              "User-Agent": f"hkjc-bench/{__version__}"})
+        try:
+            with urllib.request.urlopen(req, timeout=self.timeout) as r:
+                return json.loads(r.read())
+        except urllib.error.HTTPError as e:
+            try:
+                msg = json.loads(e.read()).get("error", e.reason)
+            except Exception:
+                msg = e.reason
+            raise APIError(f"{e.code}: {msg}") from None
+
+    def status(self) -> dict:
+        return self._call("/status")
+
+    def races(self, since: str | None = None, until: str | None = None) -> pd.DataFrame:
+        q = "&".join(f"{k}={v}" for k, v in (("since", since), ("until", until)) if v)
+        return pd.DataFrame(self._call("/races" + (f"?{q}" if q else ""))["races"])
+
+    def score(self, predictions: pd.DataFrame, basis: str = "form") -> Report:
+        """predictions: race_key, horse_no and p_win (or a score column you pass as p_win) for every runner."""
+        cols = {"race_key", "horse_no", "p_win"}
+        missing = cols - set(predictions.columns)
+        if missing:
+            raise ValueError(f"predictions need columns {sorted(cols)}; missing {sorted(missing)}")
+        rows = predictions[list(cols)].astype({"race_key": str}).to_dict("records")
+        return Report(self._call("/score", {"basis": basis, "predictions": rows}))
+
+    def predictions(self, race_keys: list[str]) -> pd.DataFrame:
+        """Model C's out-of-sample win chances (only for keys an admin has allowed; at most 200 races a call)."""
+        out = []
+        for i in range(0, len(race_keys), 200):
+            out += self._call("/predictions?race_keys=" + ",".join(race_keys[i:i + 200]))["predictions"]
+        return pd.DataFrame(out)

hkjc_bench-0.1.0/hkjc_bench.egg-info/PKG-INFO

@@ -0,0 +1,38 @@
+Metadata-Version: 2.4
+Name: hkjc-bench
+Version: 0.1.0
+Summary: Compare your HKJC model with the predictor's best model on the same races, via its model test API
+Project-URL: Homepage, https://hashedcookies.com
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=2.0
+
+# hkjc-bench
+
+Compare your Hong Kong Jockey Club race model with the [HKJC predictor](https://hashedcookies.com)'s best model
+(Model C), the current model and the betting market, on exactly the same races, without anyone's model code
+changing hands.
+
+You need an account on the predictor and an API key (dashboard → Profile → Model test API).
+
+```python
+from hkjc_bench import Client
+
+c = Client("hkjc_...")                  # or set HKJC_API_KEY
+c.status()                              # what the benchmark covers
+races = c.races(since="2024-09-01")     # races you can be scored on
+report = c.score(preds, basis="form")   # preds: DataFrame of race_key, horse_no, p_win for every runner
+print(report.summary())
+report.by_season()
+```
+
+- **basis="form"** compares with the models' form-only chances (for a model that doesn't use the odds);
+  **"blended"** with their final chances, which include the odds.
+- The benchmark is every race since 2010, each predicted by a version of the model trained only on earlier races;
+  scores are log-loss on the winner (lower is better), with the chance your model is really better.
+- Your predictions are scored in memory on the server and not stored.
+- `c.predictions(race_keys)` returns Model C's own chances only if an admin has allowed it for your key.
+
+Requires Python 3.10+ and pandas.

hkjc_bench-0.1.0/hkjc_bench.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+MANIFEST.in
+PYPI.md
+pyproject.toml
+hkjc_bench/__init__.py
+hkjc_bench.egg-info/PKG-INFO
+hkjc_bench.egg-info/SOURCES.txt
+hkjc_bench.egg-info/dependency_links.txt
+hkjc_bench.egg-info/requires.txt
+hkjc_bench.egg-info/top_level.txt
+tests/test_bench.py

hkjc_bench-0.1.0/hkjc_bench.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

hkjc_bench-0.1.0/hkjc_bench.egg-info/requires.txt

@@ -0,0 +1 @@
+pandas>=2.0

hkjc_bench-0.1.0/hkjc_bench.egg-info/top_level.txt

@@ -0,0 +1 @@
+hkjc_bench

hkjc_bench-0.1.0/pyproject.toml

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hkjc-bench"
+version = "0.1.0"
+description = "Compare your HKJC model with the predictor's best model on the same races, via its model test API"
+readme = "PYPI.md"
+requires-python = ">=3.10"
+classifiers = ["Programming Language :: Python :: 3", "Operating System :: OS Independent"]
+dependencies = ["pandas>=2.0"]
+
+[project.urls]
+Homepage = "https://hashedcookies.com"
+
+[tool.setuptools]
+packages = ["hkjc_bench"]

hkjc_bench-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

hkjc_bench-0.1.0/tests/test_bench.py

@@ -0,0 +1,55 @@
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+import pandas as pd
+import pytest
+
+from hkjc_bench import APIError, Client, Report
+
+
+class _Stub(BaseHTTPRequestHandler):
+    def log_message(self, *a):
+        pass
+
+    def _send(self, code, body):
+        data = json.dumps(body).encode()
+        self.send_response(code)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_GET(self):
+        if self.headers.get("Authorization") != "Bearer hkjc_good":
+            return self._send(401, {"error": "missing or invalid API key"})
+        self._send(200, {"races": [{"race_key": "2026-10-01_ST_01", "runners": 2}]})
+
+    def do_POST(self):
+        body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+        assert body["basis"] == "form" and len(body["predictions"]) == 2
+        self._send(200, {"races": 1, "basis": "form", "skipped": {}, "vs_model_c": -0.01,
+                         "chance_better_than_model_c": 0.95,
+                         "logloss": {"yours": 2.0, "model_c": 2.01, "current": 2.02, "market": 1.99},
+                         "by_season": [{"season": 2025, "yours": 2.0, "races": 1}]})
+
+
+@pytest.fixture()
+def url():
+    s = HTTPServer(("127.0.0.1", 0), _Stub)
+    threading.Thread(target=s.serve_forever, daemon=True).start()
+    yield f"http://127.0.0.1:{s.server_port}"
+    s.shutdown()
+
+
+def test_score_and_races(url):
+    c = Client("hkjc_good", url)
+    assert c.races().iloc[0]["race_key"] == "2026-10-01_ST_01"
+    r = c.score(pd.DataFrame({"race_key": ["a", "a"], "horse_no": [1, 2], "p_win": [0.6, 0.4]}))
+    assert isinstance(r, Report) and "better than Model C" in r.summary() and len(r.by_season()) == 1
+
+
+def test_errors_are_clear(url):
+    with pytest.raises(APIError, match="401"):
+        Client("hkjc_bad", url).races()
+    with pytest.raises(ValueError, match="missing"):
+        Client("hkjc_good", url).score(pd.DataFrame({"race_key": ["a"]}))