wsba-hockey 1.1.9__py3-none-any.whl → 1.2.0__py3-none-any.whl

wsba_hockey/tools/utils/shared.py

@@ -1,207 +1,28 @@
-### CODE IN THIS DIRECTORY ORIGINALLY (FULLY OR PARITALLY) WRITTEN BY HARRY SHOMER IN THE "hockey_scraper" PACKAGE
-
-## shared.py ##
-"""
-This file is a bunch of the shared functions or just general stuff used by the different scrapers in the package.
-"""
 import os
 import time
 import json
-import logging
-import warnings
-import requests
 from datetime import datetime, timedelta
-from requests.adapters import HTTPAdapter
-from urllib3.util import Retry
-from . import save_pages as sp
-from . import config
-import inspect
+import re
+from bs4 import BeautifulSoup, SoupStrainer
 
-# Directory where this file lives
-FILE_DIR = os.path.dirname(os.path.realpath(__file__))
+## SHARED FUCNCTIONS ##
+# Most code in this file originates (entirely or partially) from the hockey_scraper package by Harry Shomer #
 
-# Name and Team fixes used 
-with open(os.path.join(FILE_DIR, "player_name_fixes.json"), "r" ,encoding="utf-8") as f:
-    Names = json.load(f)['fixes']
+dir = os.path.dirname(os.path.realpath(__file__))
 
-with open(os.path.join(FILE_DIR, "team_tri_codes.json"), "r" ,encoding="utf-8") as f:
+with open(os.path.join(dir, "team_tri_codes.json"), "r" ,encoding="utf-8") as f:
     TEAMS = json.load(f)['teams']
 
-
-def fix_name(name):
-    """
-    Check if a name falls under those that need fixing. If it does...fix it.
-
-    :param name: name in pbp
-
-    :return: Either the given parameter or the fixed name
-    """
-    return Names.get(name.upper(), name.upper()).upper()
-
-
 def get_team(team):
-    """
-    Get the fucking team
-    """
+    #Parse team header in HTML
     return TEAMS.get(team.upper(), team.upper()).upper()
     
-
-def custom_formatwarning(msg, *args, **kwargs): 
-    """
-    Override format for standard wanings
-    """
-    ansi_no_color = '\033[0m'
-    return "{msg}\n{no_color}".format(no_color=ansi_no_color, msg=msg)
-    
-warnings.formatwarning = custom_formatwarning
-
-
-def print_error(msg):
-    """
-    Implement own custom error using warning module. Prints in red
-
-    Reason why i still use warning for errors is so i can set to ignore them if i want to (e.g. live_scrape line 200).
-
-    :param msg: Str to print
-
-    :return: None
-    """
-    ansi_red_code = '\033[0;31m'
-    warning_msg = "{}Error: {}".format(ansi_red_code, msg)
-
-    # if config.LOG:
-    #     caller_file = os.path.basename(inspect.stack()[1].filename)
-    #     get_logger(caller_file).error(msg + " " + verbose)
-
-    warnings.warn(warning_msg) 
-
-
-def print_warning(msg):
-    """
-    Implement own custom warning using warning module. Prints in Orange.
-
-    :param msg: Str to print
-
-    :return: None
-    """
-    ansi_yellow_code = '\033[0;33m'
-    warning_msg = "{}Warning: {}".format(ansi_yellow_code, msg)
-
-    warnings.warn(warning_msg)
-
-
-def get_logger(python_file):
-    """
-    Create a basic logger to a log file
-
-    :param python_file: File that instantiates the logger instance
-    
-    :return: logger 
-    """
-    base_py_file = os.path.basename(python_file)
-
-    # If already exists we don't try to recreate it
-    if base_py_file in logging.Logger.manager.loggerDict.keys():
-        return logging.getLogger(base_py_file)
-
-    logger = logging.getLogger(base_py_file)
-    logger.setLevel(logging.INFO)  
-    
-    fh = logging.FileHandler("hockey_scraper_errors_{}.log".format(datetime.now().strftime("%Y-%m-%dT%H:%M:%S"))) 
-    fh.setFormatter(logging.Formatter('%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s', datefmt='%Y-%m-%d %I:%M:%S'))
-    logger.addHandler(fh)
-
-    return logger
-
-
-def log_error(err, py_file):
-    """
-    Log error when Logging is specified
-
-    :param err: Error to log
-    :param python_file: File that instantiates the logger instance
-
-    :return: None
-    """
-    if config.LOG:
-        get_logger(py_file).error(err)
-
-
-def get_season(date):
-    """
-    Get Season based on from_date
-
-    There is an exception for the 2019-2020 pandemic season. Accoding to the below url:
-        -  2019-2020 season ends in Oct. 2020
-        -  2020-2021 season begins in November 2020
-        -  https://nhl.nbcsports.com/2020/07/10/new-nhl-critical-dates-calendar-means-an-october-free-agent-frenzy/
-
-    :param date: date
-
-    :return: season -> ex: 2016 for 2016-2017 season
-    """
-    year = date[:4]
-    date = datetime.strptime(date, "%Y-%m-%d")
-    initial_bound = datetime.strptime('-'.join([year, '01-01']), "%Y-%m-%d")
-
-    # End bound for year1-year2 season is later for pandemic year
-    if initial_bound <= date <= season_end_bound(year):
-        return int(year) - 1
-
-    return int(year)
-
-
-def season_start_bound(year):
-    """
-    Get start bound for a season.
-
-    Notes:
-     - There is a bug in the schedule API for 2016 that causes the pushback to 09-30
-     - Pandemic season started in January
-
-    :param year: str of year for given date
-
-    :return: str of first date in season
-    """
-    if int(year) == 2016:
-        return "2016-09-30"
-        
-    if int(year) == 2020:
-        return '2021-01-01'
-
-    return "{}-09-01".format(str(year))
-
-
-
-def season_end_bound(year):
-    """
-    Determine the end bound of a given season. Changes depending on if it's the pandemic season or not
-
-    :param year: str of year for given date
-
-    :return: Datetime obj of last date in season
-    """
-    normal_end_bound = datetime.strptime('-'.join([str(year), '08-31']), "%Y-%m-%d")
-    pandemic_end_bound = datetime.strptime('-'.join([str(year), '10-31']), "%Y-%m-%d")
-
-    if int(year) == 2020:
-        return pandemic_end_bound
-
-    return normal_end_bound
-
-
 def convert_to_seconds(minutes):
-    """
-    Return minutes elapsed in time format to seconds elapsed
-
-    :param minutes: time elapsed
-
-    :return: time elapsed in seconds
-    """
+    #Convert time formatted as MM:SS in a period to raw seconds
     if minutes == '-16:0-':
-        return '1200'      # Sometimes in the html at the end of the game the time is -16:0-
+        return '1200'      #Sometimes in the html at the end of the game the time is -16:0-
 
-    # If the time is junk not much i can do
+    #Validate time (invalid times are generally ignored)
     try:
         x = time.strptime(minutes.strip(' '), '%M:%S')
     except ValueError:
@@ -209,189 +30,8 @@ def convert_to_seconds(minutes):
 
     return timedelta(hours=x.tm_hour, minutes=x.tm_min, seconds=x.tm_sec).total_seconds()
 
-
-def if_rescrape(user_rescrape):
-    """
-    If you want to re_scrape. If someone is a dumbass and feeds it a non-boolean it terminates the program
-
-    Note: Only matters when you have a directory specified
-
-    :param user_rescrape: Boolean
-
-    :return: None
-    """
-    if isinstance(user_rescrape, bool):
-        config.RESCRAPE = user_rescrape
-    else:
-        raise ValueError("Error: 'if_rescrape' must be a boolean. Not a {}".format(type(user_rescrape)))
-
-
-def add_dir(user_dir):
-    """
-    Add directory to store scraped docs if valid. Or create in the home dir
-
-    NOTE: After this functions docs_dir is either None or a valid directory
-
-    :param user_dir: If bool=True create in the home dire or if user provided directory on their machine
-
-    :return: None
-    """
-    # False so they don't want it
-    if not user_dir:
-        config.DOCS_DIR = False
-        return
-
-    # Something was given
-    # Either True or string to directory
-    # If boolean refer to the home directory
-    if isinstance(user_dir, bool):
-        config.DOCS_DIR = os.path.join(os.path.expanduser('~'), "hockey_scraper_data")
-        # Create if needed
-        if not os.path.isdir(config.DOCS_DIR):
-            print_warning("Creating the hockey_scraper_data directory in the home directory")
-            os.mkdir(config.DOCS_DIR)
-    elif isinstance(user_dir, str) and os.path.isdir(user_dir):
-        config.DOCS_DIR = user_dir
-    elif not (isinstance(user_dir, str) and isinstance(user_dir, bool)):
-        config.DOCS_DIR = False
-        print_error("The docs_dir argument provided is invalid")
-    else:
-        config.DOCS_DIR = False
-        print_error("The directory specified for the saving of scraped docs doesn't exist. Therefore:"
-              "\n1. All specified games will be scraped from their appropriate sources (NHL or ESPN)."
-              "\n2. All scraped files will NOT be saved at all. Please either create the directory you want them to be "
-              "deposited in or recheck the directory you typed in and start again.\n")
-
-
-def scrape_page(url):
-    """
-    Scrape a given url
-
-    :param url: url for page
-
-    :return: response object
-    """
-    response = requests.Session()
-    retries = Retry(total=10, backoff_factor=.1)
-    response.mount('http://', HTTPAdapter(max_retries=retries))
-
-    try:
-        response = response.get(url, timeout=5)
-        response.raise_for_status()
-        page = response.text
-    except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError):
-        page = None
-    except requests.exceptions.ReadTimeout:
-        # If it times out and it's the schedule print an error message...otherwise just make the page = None
-        if "schedule" in url:
-            raise Exception("Timeout Error: The NHL API took too long to respond to our request. "
-                                "Please Try Again (you may need to try a few times before it works). ")
-        else:
-            print_error("Timeout Error: The server took too long to respond to our request.")
-            page = None
-
-    # Pause for 1 second - make it more if you want
-    time.sleep(1)
-
-    return page
-
-
-
-def get_file(file_info, force=False):
-    """
-    Get the specified file.
-
-    If a docs_dir is provided we check if it exists. If it does we see if it contains that page (and saves if it
-    doesn't). If the docs_dir doesn't exist we just scrape from the source and not save.
-
-    :param file_info: Dictionary containing the info for the file.
-                      Contains the url, name, type, and season
-    :param force: Force a rescrape. Default is False
-
-    :return: page
-    """
-    file_info['dir'] = config.DOCS_DIR
-
-    # If everything checks out we'll retrieve it, otherwise we scrape it
-    if file_info['dir'] and sp.check_file_exists(file_info) and not config.RESCRAPE and not force:
-        page = sp.get_page(file_info)
-    else:
-        page = scrape_page(file_info['url'])
-        sp.save_page(page, file_info)
-
-    return page
-
-
-def check_data_format(data_format):
-    """
-    Checks if data_format specified (if it is at all) is either None, 'Csv', or 'pandas'.
-    It exits program with error message if input isn't good.
-
-    :param data_format: data_format provided 
-
-    :return: Boolean - True if good
-    """
-    if not data_format or data_format.lower() not in ['csv', 'pandas']:
-        raise ValueError('{} is an unspecified data format. The two options are Csv and Pandas '
-                            '(Csv is default)\n'.format(data_format))
-
-
-def check_valid_dates(from_date, to_date):
-    """
-    Check if it's a valid date range
-
-    :param from_date: date should scrape from
-    :param to_date: date should scrape to
-
-    :return: None
-    """
-    try:
-        if time.strptime(to_date, "%Y-%m-%d") < time.strptime(from_date, "%Y-%m-%d"):
-            raise ValueError("Error: The second date input is earlier than the first one")
-    except ValueError:
-        raise ValueError("Error: Incorrect format given for dates. They must be given like 'yyyy-mm-dd' "
-                            "(ex: '2016-10-01').")
-
-
-def to_csv(base_file_name, df, league, file_type):
-    """
-    Write DataFrame to csv file
-
-    :param base_file_name: name of file
-    :param df: DataFrame
-    :param league: nhl or nwhl
-    :param file_type: type of file despoiting
-
-    :return: None
-    """
-    docs_dir = config.DOCS_DIR
-
-    # This was a late addition so we add support here
-    if isinstance(docs_dir, str) and not os.path.isdir(os.path.join(docs_dir, "csvs")):
-        os.mkdir(os.path.join(docs_dir, "csvs"))
-
-    if df is not None:
-        if isinstance(docs_dir, str):
-            file_name = os.path.join(docs_dir, "csvs", '{}_{}_{}.csv'.format(league, file_type, base_file_name))
-        else:
-            file_name = '{}_{}_{}.csv'.format(league, file_type, base_file_name)
-
-        print("---> {} {} data deposited in file - {}".format(league, file_type, file_name))
-        df.to_csv(file_name, sep=',', encoding='utf-8')
-
-import re
-from bs4 import BeautifulSoup, SoupStrainer
-
-## html_pbp.py ##
 def get_contents(game_html):
-    """
-    Uses Beautiful soup to parses the html document.
-    Some parsers work for some pages but don't work for others....I'm not sure why so I just try them all here in order
-    
-    :param game_html: html doc
-    
-    :return: "soupified" html 
-    """
+    #Parse NHL HTML PBP document
     parsers = ["html5lib", "lxml", "html.parser"]
     strainer = SoupStrainer('td', attrs={'class': re.compile(r'bborder')})
 
@@ -409,16 +49,8 @@ def get_contents(game_html):
 
     return tds
 
-## html_shifts.py ##
 def get_soup(shifts_html):
-    """
-    Uses Beautiful soup to parses the html document.
-    Some parsers work for some pages but don't work for others....I'm not sure why so I just try them all here in order
-    
-    :param shifts_html: html doc
-    
-    :return: "soupified" html and player_shifts portion of html (it's a bunch of td tags)
-    """
+    #Convert html document to soup
     parsers = ["lxml", "html.parser", "html5lib"]
 
     for parser in parsers:
@@ -430,19 +62,12 @@ def get_soup(shifts_html):
 
     return td, get_teams(soup)
 
-
 def get_teams(soup):
-    """
-    Return the team for the TOI tables and the home team
-    
-    :param soup: souped up html
-    
-    :return: list with team and home team
-    """
+    #Find and return list of teams a given document's match (for HTML shifts parsing)
     team = soup.find('td', class_='teamHeading + border')  # Team for shifts
     team = team.get_text()
 
-    # Get Home Team
+    #Find home team
     teams = soup.find_all('td', {'align': 'center', 'style': 'font-size: 10px;font-weight:bold'})
     regex = re.compile(r'>(.*)<br/?>')
     home_team = regex.findall(str(teams[7]))

wsba_hockey/tools/xg_model.py

@@ -108,7 +108,12 @@ def fix_players(pbp):
         pbp[f'add_player_{i+1}_name'] = np.where(pbp[f'event_player_{i+1}_name'].isna(),pbp[f'event_player_{i+1}_id'].astype(str).replace(names_dict),np.nan)
         pbp[f'event_player_{i+1}_name'] = pbp[f'event_player_{i+1}_name'].combine_first(pbp[f'add_player_{i+1}_name'])
 
-    pbp['event_goalie_name'] = pbp['event_goalie_id'].astype(str).replace(names_dict)
+    #For the first three pbp seasons the event_goalie_id isn't included as a column
+    try:
+        pbp['event_goalie_name'] = pbp['event_goalie_id'].astype(str).replace(names_dict)
+    except KeyError:
+        pbp['event_goalie_id'] = np.where(pbp['event_team_venue']=='home',pbp['home_goalie_id'],pbp['away_goalie_id']).astype(str)
+        pbp['event_goalie_name'] = pbp['event_goalie_id'].astype(str).replace(names_dict)
 
     #Add hands
     pbp['event_player_1_hand'] = pbp['event_player_1_id'].astype(str).str.replace('.0','').replace(roster_dict)

wsba_hockey/wsba_main.py

@@ -34,7 +34,8 @@ SEASONS = [
     20212022,
     20222023,
     20232024,
-    20242025
+    20242025,
+    20252026
 ]
 
 CONVERT_SEASONS = {2007: 20072008, 
@@ -54,7 +55,28 @@ CONVERT_SEASONS = {2007: 20072008,
                    2021: 20212022, 
                    2022: 20222023, 
                    2023: 20232024, 
-                   2024: 20242025}
+                   2024: 20242025,
+                   2025: 20252026}
+
+SEASON_NAMES = {20072008: '2007-08', 
+                20082009: '2008-09',
+                20092010: '2009-10', 
+                20102011: '2010-11',
+                20112012: '2011-12', 
+                20122013: '2012-13',
+                20132014: '2013-14', 
+                20142015: '2014-15',
+                20152016: '2015-16', 
+                20162017: '2016-17',
+                20172018: '2017-18',
+                20182019: '2018-19', 
+                20192020: '2019-20',
+                20202021: '2020-21', 
+                20212022: '2021-22',
+                20222023: '2022-23', 
+                20232024: '2023-24',
+                20242025: '2024-25',
+                20252025: '2025-26'}
 
 CONVERT_TEAM_ABBR = {'L.A':'LAK',
                      'N.J':'NJD',
@@ -72,6 +94,7 @@ KNOWN_PROBS = {
     2008020259:'HTML data is completely missing for this game.',
     2008020409:'HTML data is completely missing for this game.',
     2008021077:'HTML data is completely missing for this game.',
+    2008030311:'Missing shifts data for game between Pittsburgh and Carolina',
     2009020081:'HTML pbp for this game between Pittsburgh and Carolina is missing all but the period start and first faceoff events, for some reason.',
     2009020658:'Missing shifts data for game between New York Islanders and Dallas.',
     2009020885:'Missing shifts data for game between Sharks and Blue Jackets.',
@@ -99,7 +122,7 @@ def nhl_scrape_game(game_ids:list[int], split_shifts:bool = False, remove:list[s
     Given a set of game_ids (NHL API), return complete play-by-play information as requested.
 
     Args:
-        game_ids (List[int] or ['random', int, int, int]):
+        game_ids (int or List[int] or ['random', int, int, int]):
             List of NHL game IDs to scrape or use ['random', n, start_year, end_year] to fetch n random games.
         split_shifts (bool, optional):
             If True, returns a dict with separate 'pbp' and 'shifts' DataFrames. Default is False.
@@ -122,10 +145,13 @@ def nhl_scrape_game(game_ids:list[int], split_shifts:bool = False, remove:list[s
             - 'errors' (optional): list of game IDs that failed if errors=True
     """
     
+    #Wrap game_id in a list if only a single game_id is provided
+    game_ids = [game_ids] if type(game_ids) != list else game_ids
+
     pbps = []
     if game_ids[0] == 'random':
         #Randomize selection of game_ids
-        #Some ids returned may be invalid (for example, 2020021300)
+        #Some ids returned may be invalid (for example, 2020022000)
         num = game_ids[1]
         start = game_ids[2] if len(game_ids) > 1 else 2007
         end = game_ids[3] if len(game_ids) > 2 else (date.today().year)-1
@@ -190,7 +216,7 @@ def nhl_scrape_game(game_ids:list[int], split_shifts:bool = False, remove:list[s
             
             #Track error
             error_ids.append(game_id)
- 
+            
     #Add all pbps together
     if not pbps:
         print("\rNo data returned.")
@@ -1149,6 +1175,9 @@ def nhl_calculate_stats(pbp:pd.DataFrame, type:Literal['skater','goalie','team']
         #Find player headshot
         complete['Headshot'] = 'https://assets.nhle.com/mugs/nhl/'+complete['Season'].astype(str)+'/'+complete['Team']+'/'+complete['ID'].astype(int).astype(str)+'.png'
 
+        #Convert season name
+        complete['Season'] = complete['Season'].replace(SEASON_NAMES)
+
         head = ['Goalie','ID','Game'] if 'Game' in complete.columns else ['Goalie','ID']
         complete = complete[head+[
             "Season","Team",'WSBA',
@@ -1191,6 +1220,9 @@ def nhl_calculate_stats(pbp:pd.DataFrame, type:Literal['skater','goalie','team']
         complete['xGF%'] = complete['xGF']/(complete['xGF']+complete['xGA'])
         complete['FF%'] = complete['FF']/(complete['FF']+complete['FA'])
         complete['CF%'] = complete['CF']/(complete['CF']+complete['CA'])
+        
+        #Convert season name
+        complete['Season'] = complete['Season'].replace(SEASON_NAMES)
 
         head = ['Team','Game'] if 'Game' in complete.columns else ['Team']
         complete = complete[head+[
@@ -1286,6 +1318,9 @@ def nhl_calculate_stats(pbp:pd.DataFrame, type:Literal['skater','goalie','team']
             for stat in PER_SIXTY[:3]:
                 type_metrics.append(f'{type.capitalize()}{stat}')
 
+        #Convert season name
+        complete['Season'] = complete['Season'].replace(SEASON_NAMES)
+
         head = ['Player','ID','Game'] if 'Game' in complete.columns else ['Player','ID']
         complete = complete[head+[
             "Season","Team",'WSBA',
@@ -1529,13 +1564,13 @@ class NHL_Database:
                 The initialized play-by-play dataset.
         """
 
-        print('Initializing database...')
+        print(f'Initializing database "{name}"...')
         self.name = name
 
         if game_ids:
-            self.pbp = nhl_scrape_game(game_ids)
+            self.pbp = nhl_apply_xG(nhl_scrape_game(game_ids))
         else:
-            self.pbp = nhl_scrape_game(['random',3,2010,2024]) if pbp.empty else pbp
+            self.pbp = nhl_apply_xG(nhl_scrape_game(['random',3,2007,2024])) if pbp.empty else pbp
 
         self.games = self.pbp['game_id'].drop_duplicates().to_list()
         self.stats = {}
@@ -1555,7 +1590,7 @@ class NHL_Database:
         """
 
         print('Adding games...')
-        self.pbp = pd.concat([self.pbp,wsba.nhl_scrape_game(game_ids)])
+        self.pbp = pd.concat([self.pbp,nhl_apply_xG(wsba.nhl_scrape_game(game_ids))])
 
         return self.pbp
     
@@ -1650,7 +1685,7 @@ class NHL_Database:
                 Root folder to export data into. Defaults to `self.name/`.
         """
 
-        print('Exporting data...')
+        print(f'Exporting data in database "{self.name}"...')
         start = time.perf_counter()
 
         # Use default path if none provided