sportsball 0.3.29__tar.gz → 0.3.31__tar.gz

{sportsball-0.3.29/sportsball.egg-info → sportsball-0.3.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sportsball
-Version: 0.3.29
+Version: 0.3.31
 Summary: A library for pulling in and normalising sports stats.
 Home-page: https://github.com/8W9aG/sportsball
 Author: Will Sackfield

{sportsball-0.3.29 → sportsball-0.3.31}/setup.py

@@ -26,7 +26,7 @@ def install_requires() -> typing.List[str]:
 
 setup(
     name='sportsball',
-    version='0.3.29',
+    version='0.3.31',
     description='A library for pulling in and normalising sports stats.',
     long_description=long_description,
     long_description_content_type='text/markdown',

{sportsball-0.3.29 → sportsball-0.3.31}/sportsball/__init__.py

@@ -1,3 +1,3 @@
 """The main module for sportsball."""
 
-__VERSION__ = "0.3.29"
+__VERSION__ = "0.3.31"

{sportsball-0.3.29 → sportsball-0.3.31}/sportsball/data/afl/afltables/afl_afltables_league_model.py

@@ -145,7 +145,9 @@ class AFLAFLTablesLeagueModel(LeagueModel):
 
     @property
     def games(self) -> Iterator[GameModel]:
-        response = self.session.get(_SEASON_URL)
+        with self.session.cache_disabled():
+            response = self.session.get(_SEASON_URL)
+        response.raise_for_status()
         soup = BeautifulSoup(response.text, "html.parser")
         with tqdm.tqdm(position=self.position) as pbar:
             for table in soup.find_all("table"):

{sportsball-0.3.29 → sportsball-0.3.31}/sportsball/data/afl/combined/afl_combined_league_model.py

@@ -71,6 +71,22 @@ AFL_TEAM_IDENTITY_MAP = {
     "North Melbourne": KANGAROOS,
     "Melbourne": MELBOURNE,
     "Collingwood": COLLINGWOOD,
+    # OddsPortal
+    "Collingwood Magpies": COLLINGWOOD,
+    "Port Adelaide Power": PORT_ADELAIDE,
+    "Richmond Tigers": RICHMOND,
+    "Carlton Blues": CARLTON,
+    "Hawthorn Hawks": HAWTHORN,
+    "Essendon Bombers": ESSENDON,
+    "Sydney Swans": SWANS,
+    "Adelaide Crows": ADELAIDE,
+    "St Kilda Saints": STKILDA,
+    "Brisbane Lions": BRISBANE_LIONS,
+    "Melbourne Demons": MELBOURNE,
+    "Geelong Cats": GEELONG,
+    "Fremantle Dockers": FREMANTLE,
+    "West Coast Eagles": WESTCOAST,
+    "Gold Coast Suns": GOLDCOAST,
 }
 BRUNSWICK_ST = "brunswick_st"
 VICTORIA_PARK = "victoria_park"
@@ -199,6 +215,8 @@ AFL_VENUE_IDENTITY_MAP = {
     "Westpac Stadium": WELLINGTON,
     "AAMI Stadium": FOOTBALL_PARK,
     "Blacktown Park": BLACKTOWN,
+    # OddsPortal
+    "Melbourne Cricket Ground": MCG,
 }
 
 

{sportsball-0.3.29 → sportsball-0.3.31}/sportsball/data/google/google_address_model.py

@@ -1441,6 +1441,15 @@ TED_CONSTANT_CONVOCATION_CENTER = SportsballGeocodeTuple(
     housenumber="4320",
     country="USA",
 )
+STONY_BROOK_ARENA = SportsballGeocodeTuple(
+    city="Stony Brook",
+    state="NY",
+    postal="11790",
+    lat=40.91741,
+    lng=-73.125955,
+    housenumber="100",
+    country="USA",
+)
 _CACHED_GEOCODES: dict[str, Any] = {
     "S.C.G. - Australia": SCG,
     "Victoria Park - Australia": SportsballGeocodeTuple(
@@ -5554,15 +5563,7 @@ _CACHED_GEOCODES: dict[str, Any] = {
         housenumber="4000",
         country="USA",
     ),
-    "Island Federal Credit Union, Stony Brook, New York": SportsballGeocodeTuple(
-        city="Stony Brook",
-        state="NY",
-        postal="11790",
-        lat=40.91741,
-        lng=-73.125955,
-        housenumber="100",
-        country="USA",
-    ),
+    "Island Federal Credit Union, Stony Brook, New York": STONY_BROOK_ARENA,
     "Cajundome, Lafayette, Louisiana": SportsballGeocodeTuple(
         city="Lafayette",
         state="LA",
@@ -7588,15 +7589,7 @@ _CACHED_GEOCODES: dict[str, Any] = {
         housenumber="11200",
         country="USA",
     ),
-    "Island Federal Credit Union Arena, Stony Brook, New York": SportsballGeocodeTuple(
-        city="Stony Brook",
-        state="NY",
-        postal="11790",
-        lat=40.91741,
-        lng=-73.125955,
-        housenumber="100",
-        country="USA",
-    ),
+    "Island Federal Credit Union Arena, Stony Brook, New York": STONY_BROOK_ARENA,
     "Phoenix, Arizona": SportsballGeocodeTuple(
         city="Phoenix",
         state="AZ",
@@ -7825,6 +7818,16 @@ _CACHED_GEOCODES: dict[str, Any] = {
         housenumber="35",
         country="USA",
     ),
+    "Corpus Christi, Texas": SportsballGeocodeTuple(
+        city="Corpus Christi",
+        state="TX",
+        postal="",
+        lat=27.742778,
+        lng=-97.401944,
+        housenumber="",
+        country="USA",
+    ),
+    "Island Federal Credit Union Arena, New York, NY, USA": STONY_BROOK_ARENA,
 }
 
 

sportsball-0.3.31/sportsball/data/oddsportal/decrypt.py

@@ -0,0 +1,98 @@
+"""A function for decrypting an odds portal dat file."""
+
+# pylint: disable=too-many-locals
+import base64
+import json
+import urllib.parse
+from typing import Any
+
+import requests_cache
+from bs4 import BeautifulSoup
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.hazmat.primitives.hashes import SHA256
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+
+from ...proxy_session import X_NO_WAYBACK
+
+
+def _find_decryption_data(
+    session: requests_cache.CachedSession,
+    soup: BeautifulSoup,
+    referer_url: str,
+    user_agent: str | None = None,
+) -> tuple[bytes, bytes]:
+    salt: bytes | None = None
+    password: bytes | None = None
+    for script in soup.find_all("script"):
+        src = script.get("src")
+        if src is None:
+            continue
+        if "/app.js" in src:
+            src_url = urllib.parse.urljoin(referer_url, src)
+            headers = {X_NO_WAYBACK: "1"}
+            if user_agent is not None:
+                headers["User-Agent"] = user_agent
+            src_response = session.get(src_url, headers=headers)
+            src_response.raise_for_status()
+            variables = src_response.text
+            sentinel = 'break}return e.next=9,g(r.data,"'
+            variables = variables[variables.find(sentinel) + len(sentinel) :]
+            variables = variables[
+                : variables.find('");case 9:return s=e.sent,l=JSON.parse(s),e.abrupt')
+            ]
+            password_str, salt_str = variables.split('","')
+            salt = str.encode(salt_str)
+            password = str.encode(password_str)
+            break
+    if salt is None:
+        raise ValueError("salt is null.")
+    if password is None:
+        raise ValueError("password is null.")
+    return salt, password
+
+
+def fetch_data(
+    url: str,
+    session: requests_cache.CachedSession,
+    referer_url: str,
+    soup: BeautifulSoup,
+    user_agent: str | None = None,
+) -> dict[str, Any]:
+    """Fetch the data from the URL and decrypt it."""
+    salt, password = _find_decryption_data(
+        session, soup, referer_url, user_agent=user_agent
+    )
+    headers = {
+        "X-Requested-With": "XMLHttpRequest",
+        "Referer": referer_url,
+    }
+    if user_agent is not None:
+        headers["User-Agent"] = user_agent
+    response = session.get(
+        url,
+        headers=headers,
+    )
+    response.raise_for_status()
+    decoded_data = base64.b64decode(response.content).decode()
+    encrypted, key = decoded_data.split(":")
+    encrypted_bytes = base64.urlsafe_b64decode(encrypted)
+    key_bytes = bytes.fromhex(key)
+    kdf = PBKDF2HMAC(
+        algorithm=SHA256(),
+        length=32,
+        salt=salt,
+        iterations=1000,
+        backend=default_backend(),
+    )
+    aes_key = kdf.derive(password)
+    cipher = Cipher(
+        algorithms.AES(aes_key), modes.CBC(key_bytes), backend=default_backend()
+    )
+    decryptor = cipher.decryptor()
+    decrypted_bytes = decryptor.update(encrypted_bytes) + decryptor.finalize()
+    decrypted_data = decrypted_bytes.decode("utf-8")
+    end_of_json = decrypted_data.rfind("}")
+    if end_of_json != -1:
+        decrypted_data = decrypted_data[: end_of_json + 1]
+    return json.loads(decrypted_data)

{sportsball-0.3.29 → sportsball-0.3.31}/sportsball/data/oddsportal/oddsportal_game_model.py

@@ -1,6 +1,6 @@
 """OddsPortal game model."""
 
-# pylint: disable=too-many-locals,too-many-statements
+# pylint: disable=too-many-locals,too-many-statements,line-too-long
 import datetime
 import json
 import urllib.parse
@@ -13,6 +13,7 @@ from ...cache import MEMORY
 from ...proxy_session import X_NO_WAYBACK
 from ..game_model import GameModel
 from ..league import League
+from .decrypt import fetch_data
 from .oddsportal_team_model import create_oddsportal_team_model
 from .oddsportal_venue_model import create_oddsportal_venue_model
 
@@ -49,31 +50,6 @@ def _create_oddsportal_game_model(
     default_scope_id = str(event_data["defaultScopeId"])
     xhash = urllib.parse.unquote(event_data["xhash"])
 
-    salt: bytes | None = None
-    password: bytes | None = None
-    for script in soup.find_all("script"):
-        src = script.get("src")
-        if src is None:
-            continue
-        if "/app.js" in src:
-            src_url = urllib.parse.urljoin(url, src)
-            src_response = session.get(src_url, headers={X_NO_WAYBACK: "1"})
-            src_response.raise_for_status()
-            variables = src_response.text
-            sentinel = 'break}return e.next=9,g(r.data,"'
-            variables = variables[variables.find(sentinel) + len(sentinel) :]
-            variables = variables[
-                : variables.find('");case 9:return s=e.sent,l=JSON.parse(s),e.abrupt')
-            ]
-            password_str, salt_str = variables.split('","')
-            salt = str.encode(salt_str)
-            password = str.encode(password_str)
-            break
-    if salt is None:
-        raise ValueError("salt is null.")
-    if password is None:
-        raise ValueError("password is null.")
-
     bookie_names = event_body["providersNames"]
     home_points = None
     if event_body["homeResult"] != "":
@@ -96,6 +72,9 @@ def _create_oddsportal_game_model(
             event_body["venueCountry"],
         )
 
+    url = f"https://www.oddsportal.com/match-event/{version_id}-{sport_id}-{unique_id}-{default_bet_id}-{default_scope_id}-{xhash}.dat"
+    parsed_data = fetch_data(url, session, url, soup)
+
     return GameModel(
         dt=dt,
         week=None,
@@ -108,16 +87,11 @@ def _create_oddsportal_game_model(
                 event_data["home"],
                 league,
                 home_points,
-                version_id,
-                sport_id,
-                unique_id,
                 default_bet_id,
                 default_scope_id,
-                xhash,
-                salt,
-                password,
                 bookie_names,
                 0,
+                parsed_data,
             ),
             create_oddsportal_team_model(
                 session,
@@ -125,16 +99,11 @@ def _create_oddsportal_game_model(
                 event_data["away"],
                 league,
                 away_points,
-                version_id,
-                sport_id,
-                unique_id,
                 default_bet_id,
                 default_scope_id,
-                xhash,
-                salt,
-                password,
                 bookie_names,
                 1,
+                parsed_data,
             ),
         ],
         end_dt=end_dt,

{sportsball-0.3.29 → sportsball-0.3.31}/sportsball/data/oddsportal/oddsportal_league_model.py

@@ -1,6 +1,6 @@
 """Odds Portal league model."""
 
-# pylint: disable=too-many-locals,too-many-branches,too-many-statements,protected-access
+# pylint: disable=too-many-locals,too-many-branches,too-many-statements,protected-access,too-many-arguments,line-too-long
 import http
 import json
 import logging
@@ -9,12 +9,15 @@ from typing import Iterator
 
 import extruct  # type: ignore
 import requests
+import requests_cache
 import tqdm
-from bs4 import BeautifulSoup, Tag
+from bs4 import BeautifulSoup
 
+from ...proxy_session import X_NO_WAYBACK
 from ..game_model import GameModel
 from ..league import League
 from ..league_model import LeagueModel
+from .decrypt import fetch_data
 from .oddsportal_game_model import create_oddsportal_game_model
 
 # Sports
@@ -28,6 +31,40 @@ USA = "usa"
 NCAA = "ncaa"
 
 
+def _process_results_pages(
+    url: str,
+    session: requests_cache.CachedSession,
+    soup: BeautifulSoup,
+    league: League,
+    pbar: tqdm.tqdm,
+    response: requests.Response,
+) -> Iterator[GameModel]:
+    # Fetch first page
+    sanitised_text = response.text[response.text.find("var pageOutrightsVar = '") :]
+    sanitised_text = sanitised_text[: sanitised_text.find("'")]
+    page_outrights = json.loads(sanitised_text)
+    sports_id = page_outrights["sid"]
+    oddsportal_id = page_outrights["id"]
+
+    current_page = 1
+    total_pages = None
+    while (current_page == 1 and total_pages is None) or (
+        current_page <= 0 if total_pages is None else total_pages
+    ):
+        dat_url = f"https://www.oddsportal.com/ajax-sport-country-tournament-archive_/{sports_id}/{oddsportal_id}/X134529032X0X0X0X0X0X0X0X0X0X0X0X0X0X0X0X0X0X512X32X0X0X0X0X0X0X131072X0X2048/1/-5/page/{current_page}//"
+        parsed_data = fetch_data(dat_url, session, url, soup)
+        d = parsed_data["d"]
+        for row in d["rows"]:
+            game_model = create_oddsportal_game_model(
+                session, urllib.parse.urljoin(url, row["url"]), league, False
+            )
+            pbar.update(1)
+            pbar.set_description(f"OddsPortal {game_model.dt}")
+            yield game_model
+        total_pages = d["pagination"]["pages"]
+        current_page += 1
+
+
 class OddsPortalLeagueModel(LeagueModel):
     """Odds Portal implementation of the league model."""
 
@@ -85,14 +122,10 @@ class OddsPortalLeagueModel(LeagueModel):
             logging.debug("processing url: %s", url)
             seen_urls.add(url)
 
-            if url.endswith(standard_suffix):
-                with self.session.cache_disabled():
-                    response = self.session.get(url)
-            else:
-                response = self.session.get(url)
+            response = self.session.get(url, headers={X_NO_WAYBACK: "1"})
             response.raise_for_status()
 
-            soup = BeautifulSoup(response.text, "html.parser")
+            soup = BeautifulSoup(response.text, "xml")
 
             # Find next URLs
             for option in soup.find_all("option"):
@@ -100,69 +133,17 @@ class OddsPortalLeagueModel(LeagueModel):
                 if next_url.endswith("/results/") and self._path[:-1] in next_url:
                     queued_urls.add(next_url)
 
-            # Paginate through results
-            while True:
-                game_urls = set()
-
-                # Check the react tags
-                tournament_component = soup.find("tournament-component")
-                if isinstance(tournament_component, Tag):
-                    matches = json.loads(str(tournament_component[":sport-data"]))
-                    component = matches.get(
-                        "tournamentGamesComponent", matches.get("d")
-                    )
-                    if component.get("total") != 0:
-                        game_urls = {
-                            urllib.parse.urljoin(url, x["url"])
-                            for x in component.get("rows", [])
-                        }
-                if not game_urls:
-                    for a in soup.find_all("a", href=True):
-                        game_url = urllib.parse.urljoin(url, a.get("href"))
-                        if (
-                            game_url.endswith("results/")
-                            or game_url.endswith("standings/")
-                            or game_url.endswith(self._path)
-                            or game_url.endswith("outrights/")
-                        ):
-                            continue
-                        if self._path[:-1] in game_url:
-                            game_urls.add(game_url)
-
-                for game_url in game_urls:
-                    if "#" in game_url:
-                        continue
-                    if (
-                        game_url.replace("results/", "") == url
-                        or game_url.replace("results/", "") in queued_urls
-                    ):
-                        continue
-                    if game_url.endswith("/archive/"):
-                        continue
-                    if len(game_url.split("/")) != 8:
-                        continue
-                    game_model = create_oddsportal_game_model(
-                        self.session,
-                        game_url,
-                        self.league,
-                        False,
-                    )
-                    pbar.update(1)
-                    pbar.set_description(f"OddsPortal {game_model.dt}")
-                    yield game_model
-
-                next_a = soup.find("a", text="Next", href=True)
-                if next_a is None:
-                    break
-                if not isinstance(next_a, Tag):
-                    raise ValueError("next_a is not a tag.")
-                next_url = urllib.parse.urljoin(url, str(next_a.get("href")))
-                response = self.session.get(next_url)
-                response.raise_for_status()
-                soup = BeautifulSoup(response.text, "html.parser")
+            yield from _process_results_pages(
+                url,
+                self.session,
+                soup,
+                self.league,
+                pbar,
+                response,
+            )
 
     @property
     def games(self) -> Iterator[GameModel]:
         with tqdm.tqdm(position=self.position) as pbar:
-            # yield from self._find_next(pbar)
+            yield from self._find_next(pbar)
             yield from self._find_previous(pbar)

{sportsball-0.3.29 → sportsball-0.3.31}/sportsball/data/oddsportal/oddsportal_team_model.py

@@ -1,16 +1,11 @@
 """Odds Portal team model."""
 
 # pylint: disable=too-many-arguments,duplicate-code,line-too-long,too-many-locals
-import base64
 import datetime
-import json
+from typing import Any
 
 import pytest_is_running
 import requests_cache
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
-from cryptography.hazmat.primitives.hashes import SHA256
-from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
 
 from ...cache import MEMORY
 from ..google.google_news_model import create_google_news_models
@@ -26,46 +21,12 @@ def _create_oddsportal_team_model(
     team_name: str,
     league: League,
     points: float | None,
-    version_id: str,
-    sport_id: str,
-    unique_id: str,
     default_bet_id: str,
     default_scope_id: str,
-    xhash: str,
-    salt: bytes,
-    password: bytes,
     bookie_names: dict[str, str],
     team_idx: int,
+    parsed_data: dict[str, Any],
 ) -> TeamModel:
-    response = session.get(
-        f"https://www.oddsportal.com/match-event/{version_id}-{sport_id}-{unique_id}-{default_bet_id}-{default_scope_id}-{xhash}.dat",
-        headers={
-            "X-Requested-With": "XMLHttpRequest",
-        },
-    )
-    response.raise_for_status()
-    decoded_data = base64.b64decode(response.content).decode()
-    encrypted, key = decoded_data.split(":")
-    encrypted_bytes = base64.urlsafe_b64decode(encrypted)
-    key_bytes = bytes.fromhex(key)
-    kdf = PBKDF2HMAC(
-        algorithm=SHA256(),
-        length=32,
-        salt=salt,
-        iterations=1000,
-        backend=default_backend(),
-    )
-    aes_key = kdf.derive(password)
-    cipher = Cipher(
-        algorithms.AES(aes_key), modes.CBC(key_bytes), backend=default_backend()
-    )
-    decryptor = cipher.decryptor()
-    decrypted_bytes = decryptor.update(encrypted_bytes) + decryptor.finalize()
-    decrypted_data = decrypted_bytes.decode("utf-8")
-    end_of_json = decrypted_data.rfind("}")
-    if end_of_json != -1:
-        decrypted_data = decrypted_data[: end_of_json + 1]
-    parsed_data = json.loads(decrypted_data)
     try:
         odds_data = parsed_data["d"]["oddsdata"]["back"][
             f"E-{default_bet_id}-{default_scope_id}-0-0-0"
@@ -111,16 +72,11 @@ def _cached_create_oddsportal_team_model(
     team_name: str,
     league: League,
     points: float | None,
-    version_id: str,
-    sport_id: str,
-    unique_id: str,
     default_bet_id: str,
     default_scope_id: str,
-    xhash: str,
-    salt: bytes,
-    password: bytes,
     bookie_names: dict[str, str],
     team_idx: int,
+    parsed_data: dict[str, Any],
 ) -> TeamModel:
     return _create_oddsportal_team_model(
         session,
@@ -128,16 +84,11 @@ def _cached_create_oddsportal_team_model(
         team_name,
         league,
         points,
-        version_id,
-        sport_id,
-        unique_id,
         default_bet_id,
         default_scope_id,
-        xhash,
-        salt,
-        password,
         bookie_names,
         team_idx,
+        parsed_data,
     )
 
 
@@ -147,16 +98,11 @@ def create_oddsportal_team_model(
     team_name: str,
     league: League,
     points: float | None,
-    version_id: str,
-    sport_id: str,
-    unique_id: str,
     default_bet_id: str,
     default_scope_id: str,
-    xhash: str,
-    salt: bytes,
-    password: bytes,
     bookie_names: dict[str, str],
     team_idx: int,
+    parsed_data: dict[str, Any],
 ) -> TeamModel:
     """Create a team model based off the odds portal response."""
     if not pytest_is_running.is_running() and dt < datetime.datetime.now().replace(
@@ -168,16 +114,11 @@ def create_oddsportal_team_model(
             team_name,
             league,
             points,
-            version_id,
-            sport_id,
-            unique_id,
             default_bet_id,
             default_scope_id,
-            xhash,
-            salt,
-            password,
             bookie_names,
             team_idx,
+            parsed_data,
         )
     with session.cache_disabled():
         return _create_oddsportal_team_model(
@@ -186,14 +127,9 @@ def create_oddsportal_team_model(
             team_name,
             league,
             points,
-            version_id,
-            sport_id,
-            unique_id,
             default_bet_id,
             default_scope_id,
-            xhash,
-            salt,
-            password,
             bookie_names,
             team_idx,
+            parsed_data,
         )

{sportsball-0.3.29 → sportsball-0.3.31/sportsball.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sportsball
-Version: 0.3.29
+Version: 0.3.31
 Summary: A library for pulling in and normalising sports stats.
 Home-page: https://github.com/8W9aG/sportsball
 Author: Will Sackfield

{sportsball-0.3.29 → sportsball-0.3.31}/sportsball.egg-info/SOURCES.txt

@@ -120,6 +120,7 @@ sportsball/data/nfl/oddsportal/nfl_oddsportal_league_model.py
 sportsball/data/nfl/sportsdb/__init__.py
 sportsball/data/nfl/sportsdb/nfl_sportsdb_league_model.py
 sportsball/data/oddsportal/__init__.py
+sportsball/data/oddsportal/decrypt.py
 sportsball/data/oddsportal/oddsportal_bookie_model.py
 sportsball/data/oddsportal/oddsportal_game_model.py
 sportsball/data/oddsportal/oddsportal_league_model.py