sportsball 0.3.19__tar.gz → 0.3.22__tar.gz

{sportsball-0.3.19/sportsball.egg-info → sportsball-0.3.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sportsball
-Version: 0.3.19
+Version: 0.3.22
 Summary: A library for pulling in and normalising sports stats.
 Home-page: https://github.com/8W9aG/sportsball
 Author: Will Sackfield
@@ -41,6 +41,8 @@ Requires-Dist: tenacity>=9.0.0
 Requires-Dist: random_user_agent>=1.0.1
 Requires-Dist: wayback>=0.4.5
 Requires-Dist: cryptography>=44.0.0
+Requires-Dist: feedparser>=6.0.11
+Requires-Dist: dateparser>=1.2.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -97,6 +99,8 @@ Python 3.11.6:
 - [random_user_agent](https://github.com/Luqman-Ud-Din/random_user_agent)
 - [wayback](https://github.com/edgi-govdata-archiving/wayback)
 - [cryptography](https://cryptography.io/en/latest/)
+- [feedparser](https://github.com/kurtmckee/feedparser)
+- [dateparser](https://dateparser.readthedocs.io/en/latest/)
 
 ## Raison D'être :thought_balloon:
 

{sportsball-0.3.19 → sportsball-0.3.22}/README.md

@@ -43,6 +43,8 @@ Python 3.11.6:
 - [random_user_agent](https://github.com/Luqman-Ud-Din/random_user_agent)
 - [wayback](https://github.com/edgi-govdata-archiving/wayback)
 - [cryptography](https://cryptography.io/en/latest/)
+- [feedparser](https://github.com/kurtmckee/feedparser)
+- [dateparser](https://dateparser.readthedocs.io/en/latest/)
 
 ## Raison D'être :thought_balloon:
 

{sportsball-0.3.19 → sportsball-0.3.22}/requirements.txt

@@ -18,7 +18,6 @@ nba_api>=1.7.0
 timezonefinder>=6.5.7
 pydantic>=2.10.4
 flatten_json>=0.1.14
--e git+https://github.com/8W9aG/pygooglenews@d40ad6f01cb661bbb1c4b4022896f93c83c88364#egg=pygooglenews
 extruct>=0.18.0
 wikipedia-api>=0.8.1
 tweepy>=4.15.0
@@ -29,3 +28,5 @@ tenacity>=9.0.0
 random_user_agent>=1.0.1
 wayback>=0.4.5
 cryptography>=44.0.0
+feedparser>=6.0.11
+dateparser>=1.2.0

{sportsball-0.3.19 → sportsball-0.3.22}/setup.py

@@ -13,17 +13,20 @@ def install_requires() -> typing.List[str]:
     with open(
         Path(__file__).absolute().parent.joinpath('requirements.txt'), "r"
     ) as requirments_txt_handle:
-        requires = [
-            x
-            for x in requirments_txt_handle
-            if not x.startswith(".") and not x.startswith("-e")
-        ]
+        for require in requirments_txt_handle:
+            if not require.startswith(".") and not require.startswith("-e"):
+                requires.append(require)
+            else:
+                require_file = require.split()[-1]
+                require_file = require_file.replace("git+", "")
+                package_name = require_file.split("#egg=")[-1]
+                requires.append(package_name + " @ " + require_file)
     return requires
 
 
 setup(
     name='sportsball',
-    version='0.3.19',
+    version='0.3.22',
     description='A library for pulling in and normalising sports stats.',
     long_description=long_description,
     long_description_content_type='text/markdown',

{sportsball-0.3.19 → sportsball-0.3.22}/sportsball/__init__.py

@@ -1,3 +1,3 @@
 """The main module for sportsball."""
 
-__VERSION__ = "0.3.19"
+__VERSION__ = "0.3.22"

{sportsball-0.3.19 → sportsball-0.3.22}/sportsball/data/combined/combined_league_model.py

@@ -3,6 +3,7 @@
 # pylint: disable=raise-missing-from
 import logging
 import multiprocessing
+import traceback
 from multiprocessing import Pool
 from typing import Any, Iterator
 
@@ -21,6 +22,7 @@ def _produce_league_games(league_model: LeagueModel) -> list[dict[str, Any]]:
         return [x.model_dump() for x in league_model.games]
     except Exception as exc:
         logging.error("Exception: %s", str(exc))
+        logging.error(traceback.format_exc())
         raise ValueError(str(exc))
 
 

{sportsball-0.3.19 → sportsball-0.3.22}/sportsball/data/google/google_address_model.py

@@ -1108,6 +1108,15 @@ TEXAS_AM_COMMERCE_FIELD_HOUSE = SportsballGeocodeTuple(
     housenumber="2600",
     country="USA",
 )
+WISDOM_GYM = SportsballGeocodeTuple(
+    city="Stephenville",
+    state="TX",
+    postal="",
+    lat=32.215438,
+    lng=-98.220617,
+    housenumber="",
+    country="USA",
+)
 _CACHED_GEOCODES: dict[str, Any] = {
     "S.C.G. - Australia": SCG,
     "Victoria Park - Australia": SportsballGeocodeTuple(
@@ -3528,15 +3537,7 @@ _CACHED_GEOCODES: dict[str, Any] = {
         housenumber="8600",
         country="USA",
     ),
-    "Wisdom Gym, Stephenville, Texas": SportsballGeocodeTuple(
-        city="Stephenville",
-        state="TX",
-        postal="",
-        lat=32.215438,
-        lng=-98.220617,
-        housenumber="",
-        country="USA",
-    ),
+    "Wisdom Gym, Stephenville, Texas": WISDOM_GYM,
     "Skyhawk Arena, Martin, Tennessee": SportsballGeocodeTuple(
         city="Martin",
         state="TN",
@@ -7066,6 +7067,34 @@ _CACHED_GEOCODES: dict[str, Any] = {
         housenumber="",
         country="USA",
     ),
+    "Dallas, Texas": SportsballGeocodeTuple(
+        city="Dallas",
+        state="TX",
+        postal="",
+        lat=32.779167,
+        lng=-96.808889,
+        housenumber="",
+        country="USA",
+    ),
+    "The Gabba, Brisbane, Australia": GABBA,
+    "Sydney Cricket Ground, Sydney, Australia": SCG,
+    "People First Stadium, Gold Coast, Australia": PEOPLE_FIRST_STADIUM,
+    "Engie Stadium, Sydney, Australia": ENGIE_STADIUM,
+    "Melbourne Cricket Ground, Melbourne, Australia": MCG,
+    "GMHBA Stadium, Geelong, Australia": GMHBA_STADIUM,
+    "Marvel Stadium, Melbourne, Australia": MARVEL_STADIUM,
+    "Adelaide Oval, Adelaide, Australia": ADELAIDE_OVAL,
+    "Optus Stadium, Perth, Australia": OPTUS_STADIUM,
+    "Wisdom Gymnasium, Stephenville, Texas": WISDOM_GYM,
+    "Charlotte, North Carolina": SportsballGeocodeTuple(
+        city="Charlotte",
+        state="NC",
+        postal="",
+        lat=35.227222,
+        lng=-80.843056,
+        housenumber="",
+        country="USA",
+    ),
 }
 
 

{sportsball-0.3.19 → sportsball-0.3.22}/sportsball/data/google/google_news_model.py

@@ -7,9 +7,9 @@ import pytest_is_running
 import requests_cache
 from bs4 import BeautifulSoup
 from dateutil import parser
-from pygooglenews import GoogleNews  # type: ignore
 
 from ...cache import MEMORY
+from ...vendor.pygooglenews import GoogleNews  # type: ignore
 from ..league import League, long_name
 from ..news_model import NewsModel
 

sportsball-0.3.22/sportsball/data/ncaaf/__init__.py

@@ -0,0 +1,5 @@
+"""The ncaaf data sportsball module."""
+
+# ruff: noqa: F401
+from .combined.ncaaf_combined_league_model import \
+    NCAAFCombinedLeagueModel as NCAAFLeagueModel

sportsball-0.3.22/sportsball/data/ncaaf/combined/ncaaf_combined_league_model.py

@@ -0,0 +1,34 @@
+"""NCAAF combined league model."""
+
+import requests_cache
+
+from ...combined.combined_league_model import CombinedLeagueModel
+from ...league import League
+from ..espn.ncaaf_espn_league_model import NCAAFESPNLeagueModel
+from ..oddsportal.ncaaf_oddsportal_league_model import \
+    NCAAFOddsPortalLeagueModel
+
+NCAAF_TEAM_IDENTITY_MAP: dict[str, str] = {}
+NCAAF_VENUE_IDENTITY_MAP: dict[str, str] = {}
+
+
+class NCAAFCombinedLeagueModel(CombinedLeagueModel):
+    """NCAAF combined implementation of the league model."""
+
+    def __init__(self, session: requests_cache.CachedSession) -> None:
+        super().__init__(
+            session,
+            League.NCAAF,
+            [
+                NCAAFESPNLeagueModel(session),
+                NCAAFOddsPortalLeagueModel(session),
+            ],
+        )
+
+    @classmethod
+    def team_identity_map(cls) -> dict[str, str]:
+        return NCAAF_TEAM_IDENTITY_MAP
+
+    @classmethod
+    def venue_identity_map(cls) -> dict[str, str]:
+        return NCAAF_VENUE_IDENTITY_MAP

sportsball-0.3.22/sportsball/data/ncaaf/oddsportal/ncaaf_oddsportal_league_model.py

@@ -0,0 +1,15 @@
+"""NCAAF OddsPortal league model."""
+
+# pylint: disable=line-too-long
+
+import requests_cache
+
+from ...league import League
+from ...oddsportal.oddsportal_league_model import OddsPortalLeagueModel
+
+
+class NCAAFOddsPortalLeagueModel(OddsPortalLeagueModel):
+    """NCAAF OddsPortal implementation of the league model."""
+
+    def __init__(self, session: requests_cache.CachedSession) -> None:
+        super().__init__(League.NCAAF, session)

{sportsball-0.3.19 → sportsball-0.3.22}/sportsball/data/oddsportal/oddsportal_game_model.py

@@ -24,17 +24,17 @@ def _create_oddsportal_game_model(
     response = session.get(url)
     response.raise_for_status()
 
-    soup = BeautifulSoup(url, "html.parser")
-    event_tag = soup.find("Event")
+    soup = BeautifulSoup(response.text, "html.parser")
+    event_tag = soup.find("div", id="react-event-header")
     if not isinstance(event_tag, Tag):
-        raise ValueError("event_tag is not a tag.")
-    event = json.loads(str(event_tag[":data"]))
+        raise ValueError(f"event_tag is not a tag for URL: {url}.")
+    event = json.loads(str(event_tag["data"]))
 
     event_body = event["eventBody"]
     event_data = event["eventData"]
     dt = datetime.datetime.fromtimestamp(event_body["startDate"])
     end_dt = None
-    if event_data["endDate"] is not False:
+    if event_body.get("endDate", False) is not False:
         end_dt = datetime.datetime.fromtimestamp(event_body["endDate"])
     version_id = str(event_data["versionId"])
     sport_id = str(event_data["sportId"])
@@ -54,11 +54,12 @@ def _create_oddsportal_game_model(
             src_response = session.get(src_url)
             src_response.raise_for_status()
             variables = src_response.text
-            variables = variables[variables.find('break}return e.next=9,g(r.data,"') :]
+            sentinel = 'break}return e.next=9,g(r.data,"'
+            variables = variables[variables.find(sentinel) + len(sentinel) :]
             variables = variables[
                 : variables.find('");case 9:return s=e.sent,l=JSON.parse(s),e.abrupt')
             ]
-            salt_str, password_str = variables.split('","')
+            password_str, salt_str = variables.split('","')
             salt = str.encode(salt_str)
             password = str.encode(password_str)
             break
@@ -68,6 +69,12 @@ def _create_oddsportal_game_model(
         raise ValueError("password is null.")
 
     bookie_names = event_body["providersNames"]
+    home_points = None
+    if event_body["homeResult"] != "":
+        home_points = float(event_body["homeResult"])
+    away_points = None
+    if event_body["awayResult"] != "":
+        away_points = float(event_body["awayResult"])
 
     return GameModel(
         dt=dt,
@@ -86,7 +93,7 @@ def _create_oddsportal_game_model(
                 dt,
                 event_data["home"],
                 league,
-                float(event_body["homeResult"]),
+                home_points,
                 version_id,
                 sport_id,
                 unique_id,
@@ -103,7 +110,7 @@ def _create_oddsportal_game_model(
                 dt,
                 event_data["away"],
                 league,
-                float(event_body["awayResult"]),
+                away_points,
                 version_id,
                 sport_id,
                 unique_id,

{sportsball-0.3.19 → sportsball-0.3.22}/sportsball/data/oddsportal/oddsportal_league_model.py

@@ -13,6 +13,16 @@ from ..league import League
 from ..league_model import LeagueModel
 from .oddsportal_game_model import create_oddsportal_game_model
 
+# Sports
+AMERICAN_FOOTBALL = "american-football"
+BASKETBALL = "basketball"
+
+# Countries
+USA = "usa"
+
+# Leagues
+NCAA = "ncaa"
+
 
 class OddsPortalLeagueModel(LeagueModel):
     """Odds Portal implementation of the league model."""
@@ -23,11 +33,13 @@ class OddsPortalLeagueModel(LeagueModel):
             case League.AFL:
                 return "/".join(["aussie-rules", "australia", "afl", ""])
             case League.NBA:
-                return "/".join(["basketball", "usa", "nba", ""])
+                return "/".join([BASKETBALL, USA, "nba", ""])
             case League.NCAAB:
-                return "/".join(["basketball", "usa", "ncaa", ""])
+                return "/".join([BASKETBALL, USA, NCAA, ""])
+            case League.NCAAF:
+                return "/".join([AMERICAN_FOOTBALL, USA, NCAA, ""])
             case League.NFL:
-                return "/".join(["american-football", "usa", "nfl", ""])
+                return "/".join([AMERICAN_FOOTBALL, USA, "nfl", ""])
             case _:
                 raise ValueError(f"Unsupported league: {self.league}")
 
@@ -41,7 +53,10 @@ class OddsPortalLeagueModel(LeagueModel):
             if jsonld["@type"] != "SportsEvent":
                 continue
             game_model = create_oddsportal_game_model(
-                self.session, jsonld["url"], self.league, True
+                self.session,
+                urllib.parse.urljoin(base_url, jsonld["url"]),
+                self.league,
+                True,
             )
             pbar.update(1)
             pbar.set_description(f"OddsPortal {game_model.dt}")
@@ -56,7 +71,11 @@ class OddsPortalLeagueModel(LeagueModel):
                 continue
             seen_urls.add(url)
 
-            response = self.session.get(url)
+            if url.endswith("results/"):
+                with self.session.cache_disabled():
+                    response = self.session.get(url)
+            else:
+                response = self.session.get(url)
             response.raise_for_status()
 
             soup = BeautifulSoup(response.text, "html.parser")
@@ -73,9 +92,15 @@ class OddsPortalLeagueModel(LeagueModel):
                 if not isinstance(tournament_component, Tag):
                     raise ValueError("tournament_component is not a tag.")
                 matches = json.loads(str(tournament_component[":sport-data"]))
-                for match in matches["tournamentGamesComponent"]["rows"]:
+                component = matches.get("tournamentGamesComponent", matches.get("d"))
+                if component.get("total") == 0:
+                    break
+                for match in component.get("rows", []):
                     game_model = create_oddsportal_game_model(
-                        self.session, match["url"], self.league, False
+                        self.session,
+                        urllib.parse.urljoin(url, match["url"]),
+                        self.league,
+                        False,
                     )
                     pbar.update(1)
                     pbar.set_description(f"OddsPortal {game_model.dt}")

{sportsball-0.3.19 → sportsball-0.3.22}/sportsball/data/oddsportal/oddsportal_team_model.py

@@ -25,7 +25,7 @@ def _create_oddsportal_team_model(
     dt: datetime.datetime,
     team_name: str,
     league: League,
-    points: float,
+    points: float | None,
     version_id: str,
     sport_id: str,
     unique_id: str,
@@ -38,7 +38,10 @@ def _create_oddsportal_team_model(
     team_idx: int,
 ) -> TeamModel:
     response = session.get(
-        f"https://www.oddsportal.com/feed/match-event/{version_id}-{sport_id}-{unique_id}-{default_bet_id}-{default_scope_id}-{xhash}.dat"
+        f"https://www.oddsportal.com/match-event/{version_id}-{sport_id}-{unique_id}-{default_bet_id}-{default_scope_id}-{xhash}.dat",
+        headers={
+            "X-Requested-With": "XMLHttpRequest",
+        },
     )
     response.raise_for_status()
     decoded_data = base64.b64decode(response.content).decode()
@@ -70,7 +73,7 @@ def _create_oddsportal_team_model(
     history = odds_data["history"][outcome_id]
     odds_models = []
     for bookie_id, bookie_name in bookie_names.items():
-        for odds, _, timestamp in history[bookie_id]:
+        for odds, _, timestamp in history.get(bookie_id, []):
             odds_models.append(
                 create_oddsportal_odds_model(
                     odds,
@@ -100,7 +103,7 @@ def _cached_create_oddsportal_team_model(
     dt: datetime.datetime,
     team_name: str,
     league: League,
-    points: float,
+    points: float | None,
     version_id: str,
     sport_id: str,
     unique_id: str,
@@ -136,7 +139,7 @@ def create_oddsportal_team_model(
     dt: datetime.datetime,
     team_name: str,
     league: League,
-    points: float,
+    points: float | None,
     version_id: str,
     sport_id: str,
     unique_id: str,

{sportsball-0.3.19 → sportsball-0.3.22}/sportsball/data/sportsreference/sportsreference_game_model.py

@@ -1,6 +1,6 @@
 """Sports Reference game model."""
 
-# pylint: disable=too-many-locals,too-many-statements,unused-argument,protected-access
+# pylint: disable=too-many-locals,too-many-statements,unused-argument,protected-access,too-many-arguments
 import datetime
 import io
 import logging
@@ -22,19 +22,76 @@ from .sportsreference_team_model import create_sportsreference_team_model
 from .sportsreference_venue_model import create_sportsreference_venue_model
 
 
-def _create_sportsreference_game_model(
+def _find_old_dt(
+    dfs: list[pd.DataFrame],
     session: requests_cache.CachedSession,
+    soup: BeautifulSoup,
     url: str,
     league: League,
-) -> GameModel:
-    # pylint: disable=too-many-branches
-    response = session.get(url)
-    response.raise_for_status()
-    soup = BeautifulSoup(response.text, "html.parser")
-    scorebox_meta_div = soup.find("div", class_="scorebox_meta")
-    if not isinstance(scorebox_meta_div, Tag):
-        raise ValueError("scorebox_meta_div is not a Tag.")
+    player_urls: set[str],
+    fg: dict[str, int],
+    fga: dict[str, int],
+    offensive_rebounds: dict[str, int],
+    assists: dict[str, int],
+    turnovers: dict[str, int],
+) -> tuple[datetime.datetime, list[TeamModel], str]:
+    teams: list[TeamModel] = []
+    dt = None
+    venue_name = None
+    for df in dfs:
+        test_row = df.iat[0, 0]
+        if "Prev Game" in test_row and len(df) == 2:
+            date_venue_split = df.iat[1, 0].split()
+            dt = parse(" ".join(date_venue_split[:5]))
+            venue_name = " ".join(date_venue_split[5:])
+            for col_idx in range(len(df.columns.values)):
+                team_row = df.iat[0, col_idx]
+                team_name_points = team_row.split("⇐")[0].strip()
+                points = int(team_name_points.split()[-1].strip())
+                team_name = " ".join(team_name_points.split()[:-1]).strip()
+                team_a = soup.find("a", text=team_name, href=True)
+                if not isinstance(team_a, Tag):
+                    raise ValueError("team_a is not a tag.")
+                team_url = urllib.parse.urljoin(url, str(team_a.get("href")))
+                teams.append(
+                    create_sportsreference_team_model(
+                        session,
+                        team_url,
+                        dt,
+                        league,
+                        player_urls,
+                        points,
+                        fg,
+                        fga,
+                        offensive_rebounds,
+                        assists,
+                        turnovers,
+                    )
+                )
+            break
+
+    if dt is None:
+        raise ValueError("dt is null.")
+    if venue_name is None:
+        raise ValueError("venue_name is null.")
+
+    return (dt, teams, venue_name)
+
 
+def _find_new_dt(
+    soup: BeautifulSoup,
+    scorebox_meta_div: Tag,
+    url: str,
+    session: requests_cache.CachedSession,
+    league: League,
+    player_urls: set[str],
+    scores: list[float],
+    fg: dict[str, int],
+    fga: dict[str, int],
+    offensive_rebounds: dict[str, int],
+    assists: dict[str, int],
+    turnovers: dict[str, int],
+) -> tuple[datetime.datetime, list[TeamModel], str]:
     in_divs = scorebox_meta_div.find_all("div")
     current_in_div_idx = 0
     in_div = in_divs[current_in_div_idx]
@@ -54,6 +111,39 @@ def _create_sportsreference_game_model(
     if not isinstance(scorebox_div, Tag):
         raise ValueError("scorebox_div is not a Tag.")
 
+    teams: list[TeamModel] = []
+    for a in scorebox_div.find_all("a"):
+        team_url = urllib.parse.urljoin(url, a.get("href"))
+        if "/schools/" in team_url:
+            teams.append(
+                create_sportsreference_team_model(
+                    session,
+                    team_url,
+                    dt,
+                    league,
+                    player_urls,
+                    scores[len(teams)],
+                    fg,
+                    fga,
+                    offensive_rebounds,
+                    assists,
+                    turnovers,
+                )
+            )
+
+    return (dt, teams, venue_name)
+
+
+def _create_sportsreference_game_model(
+    session: requests_cache.CachedSession,
+    url: str,
+    league: League,
+) -> GameModel:
+    # pylint: disable=too-many-branches
+    response = session.get(url)
+    response.raise_for_status()
+    soup = BeautifulSoup(response.text, "html.parser")
+
     player_urls = set()
     for a in soup.find_all("a"):
         player_url = urllib.parse.urljoin(url, a.get("href"))
@@ -99,25 +189,36 @@ def _create_sportsreference_game_model(
                 for idx, player in enumerate(players):
                     turnovers[player] = tovs[idx]
 
-    teams: list[TeamModel] = []
-    for a in scorebox_div.find_all("a"):
-        team_url = urllib.parse.urljoin(url, a.get("href"))
-        if "/schools/" in team_url:
-            teams.append(
-                create_sportsreference_team_model(
-                    session,
-                    team_url,
-                    dt,
-                    league,
-                    player_urls,
-                    scores[len(teams)],
-                    fg,
-                    fga,
-                    offensive_rebounds,
-                    assists,
-                    turnovers,
-                )
-            )
+    scorebox_meta_div = soup.find("div", class_="scorebox_meta")
+    if not isinstance(scorebox_meta_div, Tag):
+        dt, teams, venue_name = _find_old_dt(
+            dfs,
+            session,
+            soup,
+            url,
+            league,
+            player_urls,
+            fg,
+            fga,
+            offensive_rebounds,
+            assists,
+            turnovers,
+        )
+    else:
+        dt, teams, venue_name = _find_new_dt(
+            soup,
+            scorebox_meta_div,
+            url,
+            session,
+            league,
+            player_urls,
+            scores,
+            fg,
+            fga,
+            offensive_rebounds,
+            assists,
+            turnovers,
+        )
 
     season_type = SeasonType.REGULAR
     for h2 in soup.find_all("h2"):