wsba-hockey 1.1.9__py3-none-any.whl → 1.2.1__py3-none-any.whl

wsba_hockey/tools/utils/shared.py

@@ -1,207 +1,28 @@
-### CODE IN THIS DIRECTORY ORIGINALLY (FULLY OR PARITALLY) WRITTEN BY HARRY SHOMER IN THE "hockey_scraper" PACKAGE
-
-## shared.py ##
-"""
-This file is a bunch of the shared functions or just general stuff used by the different scrapers in the package.
-"""
 import os
 import time
 import json
-import logging
-import warnings
-import requests
 from datetime import datetime, timedelta
-from requests.adapters import HTTPAdapter
-from urllib3.util import Retry
-from . import save_pages as sp
-from . import config
-import inspect
+import re
+from bs4 import BeautifulSoup, SoupStrainer
 
-# Directory where this file lives
-FILE_DIR = os.path.dirname(os.path.realpath(__file__))
+## SHARED FUCNCTIONS ##
+# Most code in this file originates (entirely or partially) from the hockey_scraper package by Harry Shomer #
 
-# Name and Team fixes used 
-with open(os.path.join(FILE_DIR, "player_name_fixes.json"), "r" ,encoding="utf-8") as f:
-    Names = json.load(f)['fixes']
+dir = os.path.dirname(os.path.realpath(__file__))
 
-with open(os.path.join(FILE_DIR, "team_tri_codes.json"), "r" ,encoding="utf-8") as f:
+with open(os.path.join(dir, "team_tri_codes.json"), "r" ,encoding="utf-8") as f:
     TEAMS = json.load(f)['teams']
 
-
-def fix_name(name):
-    """
-    Check if a name falls under those that need fixing. If it does...fix it.
-
-    :param name: name in pbp
-
-    :return: Either the given parameter or the fixed name
-    """
-    return Names.get(name.upper(), name.upper()).upper()
-
-
 def get_team(team):
-    """
-    Get the fucking team
-    """
+    #Parse team header in HTML
     return TEAMS.get(team.upper(), team.upper()).upper()
     
-
-def custom_formatwarning(msg, *args, **kwargs): 
-    """
-    Override format for standard wanings
-    """
-    ansi_no_color = '\033[0m'
-    return "{msg}\n{no_color}".format(no_color=ansi_no_color, msg=msg)
-    
-warnings.formatwarning = custom_formatwarning
-
-
-def print_error(msg):
-    """
-    Implement own custom error using warning module. Prints in red
-
-    Reason why i still use warning for errors is so i can set to ignore them if i want to (e.g. live_scrape line 200).
-
-    :param msg: Str to print
-
-    :return: None
-    """
-    ansi_red_code = '\033[0;31m'
-    warning_msg = "{}Error: {}".format(ansi_red_code, msg)
-
-    # if config.LOG:
-    #     caller_file = os.path.basename(inspect.stack()[1].filename)
-    #     get_logger(caller_file).error(msg + " " + verbose)
-
-    warnings.warn(warning_msg) 
-
-
-def print_warning(msg):
-    """
-    Implement own custom warning using warning module. Prints in Orange.
-
-    :param msg: Str to print
-
-    :return: None
-    """
-    ansi_yellow_code = '\033[0;33m'
-    warning_msg = "{}Warning: {}".format(ansi_yellow_code, msg)
-
-    warnings.warn(warning_msg)
-
-
-def get_logger(python_file):
-    """
-    Create a basic logger to a log file
-
-    :param python_file: File that instantiates the logger instance
-    
-    :return: logger 
-    """
-    base_py_file = os.path.basename(python_file)
-
-    # If already exists we don't try to recreate it
-    if base_py_file in logging.Logger.manager.loggerDict.keys():
-        return logging.getLogger(base_py_file)
-
-    logger = logging.getLogger(base_py_file)
-    logger.setLevel(logging.INFO)  
-    
-    fh = logging.FileHandler("hockey_scraper_errors_{}.log".format(datetime.now().strftime("%Y-%m-%dT%H:%M:%S"))) 
-    fh.setFormatter(logging.Formatter('%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s', datefmt='%Y-%m-%d %I:%M:%S'))
-    logger.addHandler(fh)
-
-    return logger
-
-
-def log_error(err, py_file):
-    """
-    Log error when Logging is specified
-
-    :param err: Error to log
-    :param python_file: File that instantiates the logger instance
-
-    :return: None
-    """
-    if config.LOG:
-        get_logger(py_file).error(err)
-
-
-def get_season(date):
-    """
-    Get Season based on from_date
-
-    There is an exception for the 2019-2020 pandemic season. Accoding to the below url:
-        -  2019-2020 season ends in Oct. 2020
-        -  2020-2021 season begins in November 2020
-        -  https://nhl.nbcsports.com/2020/07/10/new-nhl-critical-dates-calendar-means-an-october-free-agent-frenzy/
-
-    :param date: date
-
-    :return: season -> ex: 2016 for 2016-2017 season
-    """
-    year = date[:4]
-    date = datetime.strptime(date, "%Y-%m-%d")
-    initial_bound = datetime.strptime('-'.join([year, '01-01']), "%Y-%m-%d")
-
-    # End bound for year1-year2 season is later for pandemic year
-    if initial_bound <= date <= season_end_bound(year):
-        return int(year) - 1
-
-    return int(year)
-
-
-def season_start_bound(year):
-    """
-    Get start bound for a season.
-
-    Notes:
-     - There is a bug in the schedule API for 2016 that causes the pushback to 09-30
-     - Pandemic season started in January
-
-    :param year: str of year for given date
-
-    :return: str of first date in season
-    """
-    if int(year) == 2016:
-        return "2016-09-30"
-        
-    if int(year) == 2020:
-        return '2021-01-01'
-
-    return "{}-09-01".format(str(year))
-
-
-
-def season_end_bound(year):
-    """
-    Determine the end bound of a given season. Changes depending on if it's the pandemic season or not
-
-    :param year: str of year for given date
-
-    :return: Datetime obj of last date in season
-    """
-    normal_end_bound = datetime.strptime('-'.join([str(year), '08-31']), "%Y-%m-%d")
-    pandemic_end_bound = datetime.strptime('-'.join([str(year), '10-31']), "%Y-%m-%d")
-
-    if int(year) == 2020:
-        return pandemic_end_bound
-
-    return normal_end_bound
-
-
 def convert_to_seconds(minutes):
-    """
-    Return minutes elapsed in time format to seconds elapsed
-
-    :param minutes: time elapsed
-
-    :return: time elapsed in seconds
-    """
+    #Convert time formatted as MM:SS in a period to raw seconds
     if minutes == '-16:0-':
-        return '1200'      # Sometimes in the html at the end of the game the time is -16:0-
+        return '1200'      #Sometimes in the html at the end of the game the time is -16:0-
 
-    # If the time is junk not much i can do
+    #Validate time (invalid times are generally ignored)
     try:
         x = time.strptime(minutes.strip(' '), '%M:%S')
     except ValueError:
@@ -209,189 +30,8 @@ def convert_to_seconds(minutes):
 
     return timedelta(hours=x.tm_hour, minutes=x.tm_min, seconds=x.tm_sec).total_seconds()
 
-
-def if_rescrape(user_rescrape):
-    """
-    If you want to re_scrape. If someone is a dumbass and feeds it a non-boolean it terminates the program
-
-    Note: Only matters when you have a directory specified
-
-    :param user_rescrape: Boolean
-
-    :return: None
-    """
-    if isinstance(user_rescrape, bool):
-        config.RESCRAPE = user_rescrape
-    else:
-        raise ValueError("Error: 'if_rescrape' must be a boolean. Not a {}".format(type(user_rescrape)))
-
-
-def add_dir(user_dir):
-    """
-    Add directory to store scraped docs if valid. Or create in the home dir
-
-    NOTE: After this functions docs_dir is either None or a valid directory
-
-    :param user_dir: If bool=True create in the home dire or if user provided directory on their machine
-
-    :return: None
-    """
-    # False so they don't want it
-    if not user_dir:
-        config.DOCS_DIR = False
-        return
-
-    # Something was given
-    # Either True or string to directory
-    # If boolean refer to the home directory
-    if isinstance(user_dir, bool):
-        config.DOCS_DIR = os.path.join(os.path.expanduser('~'), "hockey_scraper_data")
-        # Create if needed
-        if not os.path.isdir(config.DOCS_DIR):
-            print_warning("Creating the hockey_scraper_data directory in the home directory")
-            os.mkdir(config.DOCS_DIR)
-    elif isinstance(user_dir, str) and os.path.isdir(user_dir):
-        config.DOCS_DIR = user_dir
-    elif not (isinstance(user_dir, str) and isinstance(user_dir, bool)):
-        config.DOCS_DIR = False
-        print_error("The docs_dir argument provided is invalid")
-    else:
-        config.DOCS_DIR = False
-        print_error("The directory specified for the saving of scraped docs doesn't exist. Therefore:"
-              "\n1. All specified games will be scraped from their appropriate sources (NHL or ESPN)."
-              "\n2. All scraped files will NOT be saved at all. Please either create the directory you want them to be "
-              "deposited in or recheck the directory you typed in and start again.\n")
-
-
-def scrape_page(url):
-    """
-    Scrape a given url
-
-    :param url: url for page
-
-    :return: response object
-    """
-    response = requests.Session()
-    retries = Retry(total=10, backoff_factor=.1)
-    response.mount('http://', HTTPAdapter(max_retries=retries))
-
-    try:
-        response = response.get(url, timeout=5)
-        response.raise_for_status()
-        page = response.text
-    except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError):
-        page = None
-    except requests.exceptions.ReadTimeout:
-        # If it times out and it's the schedule print an error message...otherwise just make the page = None
-        if "schedule" in url:
-            raise Exception("Timeout Error: The NHL API took too long to respond to our request. "
-                                "Please Try Again (you may need to try a few times before it works). ")
-        else:
-            print_error("Timeout Error: The server took too long to respond to our request.")
-            page = None
-
-    # Pause for 1 second - make it more if you want
-    time.sleep(1)
-
-    return page
-
-
-
-def get_file(file_info, force=False):
-    """
-    Get the specified file.
-
-    If a docs_dir is provided we check if it exists. If it does we see if it contains that page (and saves if it
-    doesn't). If the docs_dir doesn't exist we just scrape from the source and not save.
-
-    :param file_info: Dictionary containing the info for the file.
-                      Contains the url, name, type, and season
-    :param force: Force a rescrape. Default is False
-
-    :return: page
-    """
-    file_info['dir'] = config.DOCS_DIR
-
-    # If everything checks out we'll retrieve it, otherwise we scrape it
-    if file_info['dir'] and sp.check_file_exists(file_info) and not config.RESCRAPE and not force:
-        page = sp.get_page(file_info)
-    else:
-        page = scrape_page(file_info['url'])
-        sp.save_page(page, file_info)
-
-    return page
-
-
-def check_data_format(data_format):
-    """
-    Checks if data_format specified (if it is at all) is either None, 'Csv', or 'pandas'.
-    It exits program with error message if input isn't good.
-
-    :param data_format: data_format provided 
-
-    :return: Boolean - True if good
-    """
-    if not data_format or data_format.lower() not in ['csv', 'pandas']:
-        raise ValueError('{} is an unspecified data format. The two options are Csv and Pandas '
-                            '(Csv is default)\n'.format(data_format))
-
-
-def check_valid_dates(from_date, to_date):
-    """
-    Check if it's a valid date range
-
-    :param from_date: date should scrape from
-    :param to_date: date should scrape to
-
-    :return: None
-    """
-    try:
-        if time.strptime(to_date, "%Y-%m-%d") < time.strptime(from_date, "%Y-%m-%d"):
-            raise ValueError("Error: The second date input is earlier than the first one")
-    except ValueError:
-        raise ValueError("Error: Incorrect format given for dates. They must be given like 'yyyy-mm-dd' "
-                            "(ex: '2016-10-01').")
-
-
-def to_csv(base_file_name, df, league, file_type):
-    """
-    Write DataFrame to csv file
-
-    :param base_file_name: name of file
-    :param df: DataFrame
-    :param league: nhl or nwhl
-    :param file_type: type of file despoiting
-
-    :return: None
-    """
-    docs_dir = config.DOCS_DIR
-
-    # This was a late addition so we add support here
-    if isinstance(docs_dir, str) and not os.path.isdir(os.path.join(docs_dir, "csvs")):
-        os.mkdir(os.path.join(docs_dir, "csvs"))
-
-    if df is not None:
-        if isinstance(docs_dir, str):
-            file_name = os.path.join(docs_dir, "csvs", '{}_{}_{}.csv'.format(league, file_type, base_file_name))
-        else:
-            file_name = '{}_{}_{}.csv'.format(league, file_type, base_file_name)
-
-        print("---> {} {} data deposited in file - {}".format(league, file_type, file_name))
-        df.to_csv(file_name, sep=',', encoding='utf-8')
-
-import re
-from bs4 import BeautifulSoup, SoupStrainer
-
-## html_pbp.py ##
 def get_contents(game_html):
-    """
-    Uses Beautiful soup to parses the html document.
-    Some parsers work for some pages but don't work for others....I'm not sure why so I just try them all here in order
-    
-    :param game_html: html doc
-    
-    :return: "soupified" html 
-    """
+    #Parse NHL HTML PBP document
     parsers = ["html5lib", "lxml", "html.parser"]
     strainer = SoupStrainer('td', attrs={'class': re.compile(r'bborder')})
 
@@ -409,16 +49,8 @@ def get_contents(game_html):
 
     return tds
 
-## html_shifts.py ##
 def get_soup(shifts_html):
-    """
-    Uses Beautiful soup to parses the html document.
-    Some parsers work for some pages but don't work for others....I'm not sure why so I just try them all here in order
-    
-    :param shifts_html: html doc
-    
-    :return: "soupified" html and player_shifts portion of html (it's a bunch of td tags)
-    """
+    #Convert html document to soup
     parsers = ["lxml", "html.parser", "html5lib"]
 
     for parser in parsers:
@@ -430,19 +62,12 @@ def get_soup(shifts_html):
 
     return td, get_teams(soup)
 
-
 def get_teams(soup):
-    """
-    Return the team for the TOI tables and the home team
-    
-    :param soup: souped up html
-    
-    :return: list with team and home team
-    """
+    #Find and return list of teams a given document's match (for HTML shifts parsing)
     team = soup.find('td', class_='teamHeading + border')  # Team for shifts
     team = team.get_text()
 
-    # Get Home Team
+    #Find home team
     teams = soup.find_all('td', {'align': 'center', 'style': 'font-size: 10px;font-weight:bold'})
     regex = re.compile(r'>(.*)<br/?>')
     home_team = regex.findall(str(teams[7]))

wsba_hockey/tools/xg_model.py

@@ -108,7 +108,12 @@ def fix_players(pbp):
         pbp[f'add_player_{i+1}_name'] = np.where(pbp[f'event_player_{i+1}_name'].isna(),pbp[f'event_player_{i+1}_id'].astype(str).replace(names_dict),np.nan)
         pbp[f'event_player_{i+1}_name'] = pbp[f'event_player_{i+1}_name'].combine_first(pbp[f'add_player_{i+1}_name'])
 
-    pbp['event_goalie_name'] = pbp['event_goalie_id'].astype(str).replace(names_dict)
+    #For the first three pbp seasons the event_goalie_id isn't included as a column
+    try:
+        pbp['event_goalie_name'] = pbp['event_goalie_id'].astype(str).replace(names_dict)
+    except KeyError:
+        pbp['event_goalie_id'] = np.where(pbp['event_team_venue']=='home',pbp['home_goalie_id'],pbp['away_goalie_id']).astype(str)
+        pbp['event_goalie_name'] = pbp['event_goalie_id'].astype(str).replace(names_dict)
 
     #Add hands
     pbp['event_player_1_hand'] = pbp['event_player_1_id'].astype(str).str.replace('.0','').replace(roster_dict)

wsba_hockey/wsba_main.py

@@ -34,7 +34,8 @@ SEASONS = [
     20212022,
     20222023,
     20232024,
-    20242025
+    20242025,
+    20252026
 ]
 
 CONVERT_SEASONS = {2007: 20072008, 
@@ -54,7 +55,28 @@ CONVERT_SEASONS = {2007: 20072008,
                    2021: 20212022, 
                    2022: 20222023, 
                    2023: 20232024, 
-                   2024: 20242025}
+                   2024: 20242025,
+                   2025: 20252026}
+
+SEASON_NAMES = {20072008: '2007-08', 
+                20082009: '2008-09',
+                20092010: '2009-10', 
+                20102011: '2010-11',
+                20112012: '2011-12', 
+                20122013: '2012-13',
+                20132014: '2013-14', 
+                20142015: '2014-15',
+                20152016: '2015-16', 
+                20162017: '2016-17',
+                20172018: '2017-18',
+                20182019: '2018-19', 
+                20192020: '2019-20',
+                20202021: '2020-21', 
+                20212022: '2021-22',
+                20222023: '2022-23', 
+                20232024: '2023-24',
+                20242025: '2024-25',
+                20252025: '2025-26'}
 
 CONVERT_TEAM_ABBR = {'L.A':'LAK',
                      'N.J':'NJD',
@@ -72,14 +94,13 @@ KNOWN_PROBS = {
     2008020259:'HTML data is completely missing for this game.',
     2008020409:'HTML data is completely missing for this game.',
     2008021077:'HTML data is completely missing for this game.',
+    2008030311:'Missing shifts data for game between Pittsburgh and Carolina',
     2009020081:'HTML pbp for this game between Pittsburgh and Carolina is missing all but the period start and first faceoff events, for some reason.',
     2009020658:'Missing shifts data for game between New York Islanders and Dallas.',
     2009020885:'Missing shifts data for game between Sharks and Blue Jackets.',
     2010020124:'Game between Capitals and Hurricanes is sporadically missing player on-ice data',
     2012020018:'HTML events contain mislabeled events.',
-    2013020971:'On March 10th, 2014, Stars forward Rich Peverley suffered from a cardiac episode midgame and as a result, the remainder of the game was postponed.  \nThe game resumed on April 9th, and the only goal scorer in the game, Blue Jackets forward Nathan Horton, did not appear in the resumed game due to injury.  Interestingly, Horton would never play in the NHL again.',
     2018021133:'Game between Lightning and Capitals has incorrectly labeled event teams (i.e. WSH TAKEAWAY - #71 CIRELLI (Cirelli is a Tampa Bay skater in this game)).',
-    2019020876:'Due to the frightening collapse of Blues defensemen Jay Bouwmeester, a game on February 2nd, 2020 between the Ducks and Blues was postponed.  \nWhen the game resumed, Ducks defensemen Hampus Lindholm, who assisted on a goal in the inital game, did not play in the resumed match.'
 }
 
 SHOT_TYPES = ['wrist','deflected','tip-in','slap','backhand','snap','wrap-around','poke','bat','cradle','between-legs']
@@ -99,7 +120,7 @@ def nhl_scrape_game(game_ids:list[int], split_shifts:bool = False, remove:list[s
     Given a set of game_ids (NHL API), return complete play-by-play information as requested.
 
     Args:
-        game_ids (List[int] or ['random', int, int, int]):
+        game_ids (int or List[int] or ['random', int, int, int]):
             List of NHL game IDs to scrape or use ['random', n, start_year, end_year] to fetch n random games.
         split_shifts (bool, optional):
             If True, returns a dict with separate 'pbp' and 'shifts' DataFrames. Default is False.
@@ -122,10 +143,13 @@ def nhl_scrape_game(game_ids:list[int], split_shifts:bool = False, remove:list[s
             - 'errors' (optional): list of game IDs that failed if errors=True
     """
     
+    #Wrap game_id in a list if only a single game_id is provided
+    game_ids = [game_ids] if type(game_ids) != list else game_ids
+
     pbps = []
     if game_ids[0] == 'random':
         #Randomize selection of game_ids
-        #Some ids returned may be invalid (for example, 2020021300)
+        #Some ids returned may be invalid (for example, 2020022000)
         num = game_ids[1]
         start = game_ids[2] if len(game_ids) > 1 else 2007
         end = game_ids[3] if len(game_ids) > 2 else (date.today().year)-1
@@ -190,7 +214,7 @@ def nhl_scrape_game(game_ids:list[int], split_shifts:bool = False, remove:list[s
             
             #Track error
             error_ids.append(game_id)
- 
+            
     #Add all pbps together
     if not pbps:
         print("\rNo data returned.")
@@ -1139,7 +1163,7 @@ def nhl_calculate_stats(pbp:pd.DataFrame, type:Literal['skater','goalie','team']
                                             'birthCountry':'Nationality'})
         
         #WSBA
-        complete['WSBA'] = complete['Goalie']+complete['Team']+complete['Season'].astype(str)
+        complete['WSBA'] = complete['ID'].astype(str).str.replace('.0','')+complete['Team']+complete['Season'].astype(str)
 
         #Add player age
         complete['Birthday'] = pd.to_datetime(complete['Birthday'])
@@ -1149,6 +1173,9 @@ def nhl_calculate_stats(pbp:pd.DataFrame, type:Literal['skater','goalie','team']
         #Find player headshot
         complete['Headshot'] = 'https://assets.nhle.com/mugs/nhl/'+complete['Season'].astype(str)+'/'+complete['Team']+'/'+complete['ID'].astype(int).astype(str)+'.png'
 
+        #Convert season name
+        complete['Season'] = complete['Season'].replace(SEASON_NAMES)
+
         head = ['Goalie','ID','Game'] if 'Game' in complete.columns else ['Goalie','ID']
         complete = complete[head+[
             "Season","Team",'WSBA',
@@ -1191,6 +1218,9 @@ def nhl_calculate_stats(pbp:pd.DataFrame, type:Literal['skater','goalie','team']
         complete['xGF%'] = complete['xGF']/(complete['xGF']+complete['xGA'])
         complete['FF%'] = complete['FF']/(complete['FF']+complete['FA'])
         complete['CF%'] = complete['CF']/(complete['CF']+complete['CA'])
+        
+        #Convert season name
+        complete['Season'] = complete['Season'].replace(SEASON_NAMES)
 
         head = ['Team','Game'] if 'Game' in complete.columns else ['Team']
         complete = complete[head+[
@@ -1274,7 +1304,7 @@ def nhl_calculate_stats(pbp:pd.DataFrame, type:Literal['skater','goalie','team']
         #Remove goalies that occasionally appear in a set
         complete = complete.loc[complete['Position']!='G']
         #Add WSBA ID
-        complete['WSBA'] = complete['Player']+complete['Season'].astype(str)+complete['Team']
+        complete['WSBA'] = complete['ID'].astype(str).str.replace('.0','')+complete['Season'].astype(str)+complete['Team']
 
         #Add per 60 stats
         for stat in PER_SIXTY:
@@ -1286,6 +1316,9 @@ def nhl_calculate_stats(pbp:pd.DataFrame, type:Literal['skater','goalie','team']
             for stat in PER_SIXTY[:3]:
                 type_metrics.append(f'{type.capitalize()}{stat}')
 
+        #Convert season name
+        complete['Season'] = complete['Season'].replace(SEASON_NAMES)
+
         head = ['Player','ID','Game'] if 'Game' in complete.columns else ['Player','ID']
         complete = complete[head+[
             "Season","Team",'WSBA',
@@ -1529,13 +1562,13 @@ class NHL_Database:
                 The initialized play-by-play dataset.
         """
 
-        print('Initializing database...')
+        print(f'Initializing database "{name}"...')
         self.name = name
 
         if game_ids:
-            self.pbp = nhl_scrape_game(game_ids)
+            self.pbp = nhl_apply_xG(nhl_scrape_game(game_ids))
         else:
-            self.pbp = nhl_scrape_game(['random',3,2010,2024]) if pbp.empty else pbp
+            self.pbp = nhl_apply_xG(nhl_scrape_game(['random',3,2007,2024])) if pbp.empty else pbp
 
         self.games = self.pbp['game_id'].drop_duplicates().to_list()
         self.stats = {}
@@ -1555,7 +1588,7 @@ class NHL_Database:
         """
 
         print('Adding games...')
-        self.pbp = pd.concat([self.pbp,wsba.nhl_scrape_game(game_ids)])
+        self.pbp = pd.concat([self.pbp,nhl_apply_xG(wsba.nhl_scrape_game(game_ids))])
 
         return self.pbp
     
@@ -1650,7 +1683,7 @@ class NHL_Database:
                 Root folder to export data into. Defaults to `self.name/`.
         """
 
-        print('Exporting data...')
+        print(f'Exporting data in database "{self.name}"...')
         start = time.perf_counter()
 
         # Use default path if none provided