sportsball 0.3.31__tar.gz → 0.3.33__tar.gz

{sportsball-0.3.31/sportsball.egg-info → sportsball-0.3.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sportsball
-Version: 0.3.31
+Version: 0.3.33
 Summary: A library for pulling in and normalising sports stats.
 Home-page: https://github.com/8W9aG/sportsball
 Author: Will Sackfield

{sportsball-0.3.31 → sportsball-0.3.33}/setup.py

@@ -26,7 +26,7 @@ def install_requires() -> typing.List[str]:
 
 setup(
     name='sportsball',
-    version='0.3.31',
+    version='0.3.33',
     description='A library for pulling in and normalising sports stats.',
     long_description=long_description,
     long_description_content_type='text/markdown',

{sportsball-0.3.31 → sportsball-0.3.33}/sportsball/__init__.py

@@ -1,3 +1,3 @@
 """The main module for sportsball."""
 
-__VERSION__ = "0.3.31"
+__VERSION__ = "0.3.33"

{sportsball-0.3.31 → sportsball-0.3.33}/sportsball/data/address_model.py

@@ -11,7 +11,9 @@ class AddressModel(BaseModel):
 
     city: str = Field(..., json_schema_extra={TYPE_KEY: FieldType.CATEGORICAL})
     state: str = Field(..., json_schema_extra={TYPE_KEY: FieldType.CATEGORICAL})
-    zipcode: str = Field(..., json_schema_extra={TYPE_KEY: FieldType.CATEGORICAL})
+    zipcode: str | None = Field(
+        ..., json_schema_extra={TYPE_KEY: FieldType.CATEGORICAL}
+    )
     latitude: float | None
     longitude: float | None
     housenumber: str | None = Field(

{sportsball-0.3.31 → sportsball-0.3.33}/sportsball/data/google/google_address_model.py

@@ -7828,6 +7828,142 @@ _CACHED_GEOCODES: dict[str, Any] = {
         country="USA",
     ),
     "Island Federal Credit Union Arena, New York, NY, USA": STONY_BROOK_ARENA,
+    "Subiaco Oval, Perth, Australia": DOMAIN_STADIUM,
+    "University of Tasmania Stadium, Launceston, Australia": UTAS_STADIUM,
+    "Mars Stadium, Ballarat, Australia": MARS_STADIUM,
+    "Savannah, Georgia": SportsballGeocodeTuple(
+        city="Savannah",
+        state="GA",
+        postal="",
+        lat=32.081111,
+        lng=-81.091111,
+        housenumber="",
+        country="USA",
+    ),
+    "Blundstone Arena, Hobart, Australia": BLUNDSTONE_ARENA,
+    "Akron, Ohio": SportsballGeocodeTuple(
+        city="Akron",
+        state="OH",
+        postal="",
+        lat=41.080556,
+        lng=-81.522222,
+        housenumber="",
+        country="USA",
+    ),
+    "Michelob Ultra Arena, Paradise, Nevada": SportsballGeocodeTuple(
+        city="Paradise",
+        state="NV",
+        postal="",
+        lat=36.090678,
+        lng=-115.178981,
+        housenumber="3950",
+        country="USA",
+    ),
+    "Sioux Falls, South Dakota": SportsballGeocodeTuple(
+        city="Sioux Falls",
+        state="SD",
+        postal="",
+        lat=43.536389,
+        lng=-96.731667,
+        housenumber="",
+        country="USA",
+    ),
+    "TIO Traeger Park, Alice Springs, Australia": TIO_STADIUM,
+    "Springfield, Massachusetts": SportsballGeocodeTuple(
+        city="Springfield",
+        state="MA",
+        postal="",
+        lat=42.101389,
+        lng=-72.590278,
+        housenumber="",
+        country="USA",
+    ),
+    "Elmont, New York": SportsballGeocodeTuple(
+        city="Elmont",
+        state="NY",
+        postal="",
+        lat=40.701667,
+        lng=-73.7025,
+        housenumber="",
+        country="USA",
+    ),
+    "Manuka Oval, Canberra, Australia": MANUKA_OVAL,
+    "Tulsa, Oklahoma": SportsballGeocodeTuple(
+        city="Tulsa",
+        state="OK",
+        postal="",
+        lat=36.131389,
+        lng=-95.937222,
+        housenumber="",
+        country="USA",
+    ),
+    "Sunrise, Florida": SportsballGeocodeTuple(
+        city="Sunrise",
+        state="FL",
+        postal="",
+        lat=26.157222,
+        lng=-80.286111,
+        housenumber="",
+        country="USA",
+    ),
+    "Paradise, Nevada": SportsballGeocodeTuple(
+        city="Paradise",
+        state="NV",
+        postal="",
+        lat=36.081944,
+        lng=-115.124722,
+        housenumber="",
+        country="USA",
+    ),
+    "Niceville, Florida": SportsballGeocodeTuple(
+        city="Niceville",
+        state="FL",
+        postal="",
+        lat=30.516111,
+        lng=-86.471667,
+        housenumber="",
+        country="USA",
+    ),
+    "Jiangwan Stadium, Shanghai, China": JIANGWAN_STADIUM,
+    "Ondrej Nepela Arena, Bratislava, Slovakia": SportsballGeocodeTuple(
+        city="Bratislava",
+        state="",
+        postal="",
+        lat=48.16,
+        lng=17.135833,
+        housenumber="9",
+        country="Slovakia",
+    ),
+    "Cazaly's Stadium, Cairns, Australia": CAZALYS_STADIUM,
+    "San Juan Capistrano, California": SportsballGeocodeTuple(
+        city="San Juan Capistrano",
+        state="CA",
+        postal="",
+        lat=33.499444,
+        lng=-117.661667,
+        housenumber="",
+        country="USA",
+    ),
+    "Giants Stadium, East Rutherford, NJ, USA": METLIFE_STADIUM,
+    "TIO Stadium, Darwin, Australia": TIO_STADIUM,
+    "Fort Myers, Florida": SportsballGeocodeTuple(
+        city="Fort Myers",
+        state="FL",
+        postal="",
+        lat=26.616667,
+        lng=-81.833333,
+        housenumber="",
+        country="USA",
+    ),
+    "Thunderdome Stadium, Nonthaburi, Thailand": SportsballGeocodeTuple(
+        city="Nonthaburi",
+        state="",
+        postal="",
+        lat=13.917989,
+        lng=100.547411,
+        housenumber="",
+        country="Thailand",
+    ),
 }
 
 

{sportsball-0.3.31 → sportsball-0.3.33}/sportsball/data/oddsportal/decrypt.py

@@ -3,9 +3,11 @@
 # pylint: disable=too-many-locals
 import base64
 import json
+import logging
 import urllib.parse
 from typing import Any
 
+import requests
 import requests_cache
 from bs4 import BeautifulSoup
 from cryptography.hazmat.backends import default_backend
@@ -73,8 +75,18 @@ def fetch_data(
         url,
         headers=headers,
     )
-    response.raise_for_status()
-    decoded_data = base64.b64decode(response.content).decode()
+
+    try:
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as exc:
+        logging.error("HTTP error: %s for URL: %s", str(exc), referer_url)
+        raise exc
+
+    try:
+        decoded_data = base64.b64decode(response.content).decode()
+    except UnicodeDecodeError as exc:
+        logging.error("Error base64 decoding payload: %s", response.content)
+        raise exc
     encrypted, key = decoded_data.split(":")
     encrypted_bytes = base64.urlsafe_b64decode(encrypted)
     key_bytes = bytes.fromhex(key)

{sportsball-0.3.31 → sportsball-0.3.33}/sportsball/data/oddsportal/oddsportal_game_model.py

@@ -3,6 +3,7 @@
 # pylint: disable=too-many-locals,too-many-statements,line-too-long
 import datetime
 import json
+import logging
 import urllib.parse
 
 import pytest_is_running
@@ -22,8 +23,13 @@ def _create_oddsportal_game_model(
     session: requests_cache.CachedSession,
     url: str,
     league: League,
-) -> GameModel:
-    response = session.get(url, headers={X_NO_WAYBACK: "1"})
+) -> GameModel | None:
+    response = session.get(
+        url,
+        headers={
+            X_NO_WAYBACK: "1",
+        },
+    )
     response.raise_for_status()
 
     soup = BeautifulSoup(response.text, "html.parser")
@@ -32,7 +38,8 @@ def _create_oddsportal_game_model(
         soup_x = BeautifulSoup(response.text, "xml")
         event_tag = soup_x.find("Event")
         if not isinstance(event_tag, Tag):
-            raise ValueError(f"event_tag is not a tag for URL: {url}.")
+            logging.error("event_tag is not a tag for URL: %s.", url)
+            return None
         event = json.loads(str(event_tag[":data"]))
     else:
         event = json.loads(str(event_tag["data"]))
@@ -72,8 +79,8 @@ def _create_oddsportal_game_model(
             event_body["venueCountry"],
         )
 
-    url = f"https://www.oddsportal.com/match-event/{version_id}-{sport_id}-{unique_id}-{default_bet_id}-{default_scope_id}-{xhash}.dat"
-    parsed_data = fetch_data(url, session, url, soup)
+    dat_url = f"https://www.oddsportal.com/match-event/{version_id}-{sport_id}-{unique_id}-{default_bet_id}-{default_scope_id}-{xhash}.dat"
+    parsed_data = fetch_data(dat_url, session, url, soup)
 
     return GameModel(
         dt=dt,
@@ -121,7 +128,7 @@ def _cached_create_oddsportal_game_model(
     session: requests_cache.CachedSession,
     url: str,
     league: League,
-) -> GameModel:
+) -> GameModel | None:
     return _create_oddsportal_game_model(session, url, league)
 
 
@@ -130,7 +137,7 @@ def create_oddsportal_game_model(
     url: str,
     league: League,
     is_next: bool,
-) -> GameModel:
+) -> GameModel | None:
     """Create a OddsPortal game model."""
     if not pytest_is_running.is_running() and not is_next:
         return _cached_create_oddsportal_game_model(session, url, league)

{sportsball-0.3.31 → sportsball-0.3.33}/sportsball/data/oddsportal/oddsportal_league_model.py

@@ -31,6 +31,17 @@ USA = "usa"
 NCAA = "ncaa"
 
 
+def _find_ids(text: str) -> tuple[str, str]:
+    sentinel = "var pageOutrightsVar = '"
+    sanitised_text = text[text.find(sentinel) + len(sentinel) :]
+    sanitised_text = sanitised_text[: sanitised_text.find("'")]
+    try:
+        page_outrights = json.loads(sanitised_text)
+    except json.decoder.JSONDecodeError as exc:
+        raise exc
+    return str(page_outrights["sid"]), page_outrights["id"]
+
+
 def _process_results_pages(
     url: str,
     session: requests_cache.CachedSession,
@@ -40,28 +51,29 @@ def _process_results_pages(
     response: requests.Response,
 ) -> Iterator[GameModel]:
     # Fetch first page
-    sanitised_text = response.text[response.text.find("var pageOutrightsVar = '") :]
-    sanitised_text = sanitised_text[: sanitised_text.find("'")]
-    page_outrights = json.loads(sanitised_text)
-    sports_id = page_outrights["sid"]
-    oddsportal_id = page_outrights["id"]
+    sports_id, oddsportal_id = _find_ids(response.text)
 
     current_page = 1
     total_pages = None
     while (current_page == 1 and total_pages is None) or (
-        current_page <= 0 if total_pages is None else total_pages
+        current_page <= (0 if total_pages is None else total_pages)
     ):
         dat_url = f"https://www.oddsportal.com/ajax-sport-country-tournament-archive_/{sports_id}/{oddsportal_id}/X134529032X0X0X0X0X0X0X0X0X0X0X0X0X0X0X0X0X0X512X32X0X0X0X0X0X0X131072X0X2048/1/-5/page/{current_page}//"
         parsed_data = fetch_data(dat_url, session, url, soup)
         d = parsed_data["d"]
-        for row in d["rows"]:
+        if d.get("total") == 0:
+            return
+        for row in d.get("rows", []):
             game_model = create_oddsportal_game_model(
                 session, urllib.parse.urljoin(url, row["url"]), league, False
             )
+            if game_model is None:
+                continue
             pbar.update(1)
             pbar.set_description(f"OddsPortal {game_model.dt}")
             yield game_model
-        total_pages = d["pagination"]["pages"]
+        pagination = d["pagination"]
+        total_pages = pagination["pages"]
         current_page += 1
 
 
@@ -87,7 +99,7 @@ class OddsPortalLeagueModel(LeagueModel):
     def _find_next(self, pbar: tqdm.tqdm) -> Iterator[GameModel]:
         base_url = "https://www.oddsportal.com/" + self._path
         with self.session.cache_disabled():
-            response = self.session.get(base_url)
+            response = self.session.get(base_url, headers={X_NO_WAYBACK: "1"})
         response.raise_for_status()
         data = extruct.extract(response.text, base_url=base_url)
         for jsonld in data["json-ld"]:
@@ -100,6 +112,8 @@ class OddsPortalLeagueModel(LeagueModel):
                     self.league,
                     True,
                 )
+                if game_model is None:
+                    continue
                 pbar.update(1)
                 pbar.set_description(
                     f"OddsPortal {game_model.year} - {game_model.season_type} - {game_model.dt}"
@@ -119,13 +133,13 @@ class OddsPortalLeagueModel(LeagueModel):
             url = queued_urls.pop()
             if url in seen_urls:
                 continue
-            logging.debug("processing url: %s", url)
             seen_urls.add(url)
 
-            response = self.session.get(url, headers={X_NO_WAYBACK: "1"})
+            with self.session.cache_disabled():
+                response = self.session.get(url, headers={X_NO_WAYBACK: "1"})
             response.raise_for_status()
 
-            soup = BeautifulSoup(response.text, "xml")
+            soup = BeautifulSoup(response.text, "html.parser")
 
             # Find next URLs
             for option in soup.find_all("option"):

{sportsball-0.3.31 → sportsball-0.3.33}/sportsball/data/oddsportal/oddsportal_team_model.py

@@ -2,6 +2,7 @@
 
 # pylint: disable=too-many-arguments,duplicate-code,line-too-long,too-many-locals
 import datetime
+import logging
 from typing import Any
 
 import pytest_is_running
@@ -27,29 +28,34 @@ def _create_oddsportal_team_model(
     team_idx: int,
     parsed_data: dict[str, Any],
 ) -> TeamModel:
-    try:
-        odds_data = parsed_data["d"]["oddsdata"]["back"][
-            f"E-{default_bet_id}-{default_scope_id}-0-0-0"
-        ]
-    except KeyError:
-        back = parsed_data["d"]["oddsdata"]["back"]
-        odds_data = back[sorted(list(back.keys()))[0]]
-    try:
-        outcome_id = odds_data["outcomeId"][team_idx]
-    except KeyError:
-        outcome_id = odds_data["outcomeId"][str(team_idx)]
-    history = odds_data["history"][outcome_id]
+    back = parsed_data["d"]["oddsdata"]["back"]
     odds_models = []
-    for bookie_id, bookie_name in bookie_names.items():
-        for odds, _, timestamp in history.get(bookie_id, []):
-            odds_models.append(
-                create_oddsportal_odds_model(
-                    odds,
-                    datetime.datetime.fromtimestamp(timestamp),
-                    bookie_name,
-                    bookie_id,  # type: ignore
+
+    if not isinstance(back, list):
+        try:
+            odds_data = back[f"E-{default_bet_id}-{default_scope_id}-0-0-0"]
+        except KeyError:
+            back = parsed_data["d"]["oddsdata"]["back"]
+            odds_data = back[sorted(list(back.keys()))[0]]
+        except TypeError as exc:
+            logging.error("Failed to parse the following odds")
+            raise exc
+        try:
+            outcome_id = odds_data["outcomeId"][team_idx]
+        except KeyError:
+            outcome_id = odds_data["outcomeId"][str(team_idx)]
+        history = odds_data["history"][outcome_id]
+
+        for bookie_id, bookie_name in bookie_names.items():
+            for odds, _, timestamp in history.get(bookie_id, []):
+                odds_models.append(
+                    create_oddsportal_odds_model(
+                        odds,
+                        datetime.datetime.fromtimestamp(timestamp),
+                        bookie_name,
+                        bookie_id,  # type: ignore
+                    )
                 )
-            )
 
     return TeamModel(
         identifier=team_name,

{sportsball-0.3.31 → sportsball-0.3.33/sportsball.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sportsball
-Version: 0.3.31
+Version: 0.3.33
 Summary: A library for pulling in and normalising sports stats.
 Home-page: https://github.com/8W9aG/sportsball
 Author: Will Sackfield

{sportsball-0.3.31 → sportsball-0.3.33}/sportsball.egg-info/SOURCES.txt

@@ -191,6 +191,7 @@ tests/data/nfl/sportsdb/__init__.py
 tests/data/nfl/sportsdb/nfl_sportsdb_league_model_test.py
 tests/data/oddsportal/__init__.py
 tests/data/oddsportal/oddsportal_game_model_test.py
+tests/data/oddsportal/oddsportal_league_model_test.py
 tests/data/sportsdb/__init__.py
 tests/data/sportsdb/sportsdb_game_model_test.py
 tests/data/sportsdb/sportsdb_league_model_test.py

sportsball-0.3.33/tests/data/oddsportal/oddsportal_league_model_test.py

@@ -0,0 +1,19 @@
+"""Tests for the oddsportal league model class."""
+import os
+import unittest
+
+import requests_cache
+from sportsball.data.oddsportal.oddsportal_league_model import _find_ids
+
+
+class TestOddsPortalLeagueModel(unittest.TestCase):
+
+    def setUp(self):
+        self.session = requests_cache.CachedSession(backend="memory")
+        self.dir = os.path.dirname(__file__)
+
+    def test_current_html_ids(self):
+        with open(os.path.join(self.dir, "results.html"), "r") as handle:
+            sports_id, oddsportal_id = _find_ids(handle.read())
+            self.assertEqual(sports_id, "18")
+            self.assertEqual(oddsportal_id, "6y3HOkJ7")