wsba-hockey 0.1.2__py3-none-any.whl → 1.0.0__py3-none-any.whl

wsba_hockey/tools/utils/__init__.py

@@ -0,0 +1 @@
+### CODE IN THIS DIRECTORY ORIGINALLY (FULLY OR PARITALLY) WRITTEN BY HARRY SHOMER IN THE "hockey_scraper" PACKAGE

wsba_hockey/tools/utils/config.py

@@ -0,0 +1,14 @@
+"""
+Basic configurations
+"""
+
+# Directory where to save pages
+# When True assumes ~/hockey_scraper_data
+# Otherwise can take str to `existing` directory
+DOCS_DIR = False
+
+# Boolean that tells us whether or not we should re-scrape a given page if it's already saved
+RESCRAPE = False
+
+# Whether to log verbose errors to log file
+LOG = False

wsba_hockey/tools/utils/save_pages.py

@@ -0,0 +1,133 @@
+#
+#Saves the scraped docs so you don't have to re-scrape them every time you want to parse the docs. 
+#
+#\**** Don't mess with this unless you know what you're doing \****
+#
+import os
+import gzip
+
+
+def create_base_file_path(file_info):
+    """
+    Creates the base file path for a given file
+    
+    :param file_info: Dictionary containing the info on the file. Includes the name, season, file type, and the dir
+                      we want to deposit any data in.
+    
+    :return: path 
+    """
+    # Shitty fix for when you already have it saved but don't have nwhl folders
+    if 'nwhl' in file_info['type']:
+        if not os.path.isdir(os.path.join(file_info['dir'], 'docs', str(file_info['season']), file_info['type'])):
+            os.mkdir(os.path.join(file_info['dir'], 'docs', str(file_info['season']), file_info['type']))
+
+    return os.path.join(file_info['dir'], 'docs', str(file_info['season']), file_info['type'], file_info['name'] + ".txt")
+
+
+def is_compressed(file_info):
+    """
+    Check if stored file is compressed as we used to not save them as compressed.
+
+    :param file_info: Dictionary containing the info on the file. Includes the name, season, file type, and the dir
+                      we want to deposit any data in.
+    
+    return Boolean
+    """
+    return os.path.isfile(create_base_file_path(file_info) + ".gz")
+
+
+def create_dir_structure(dir_name):
+    """
+    Create the basic directory structure for docs_dir if not done yet.
+    Creates the docs and csvs subdir if it doesn't exist
+
+    :param dir_name: Name of dir to create
+
+    :return None
+    """
+    if not os.path.isdir(os.path.join(dir_name, 'docs')):
+        os.mkdir(os.path.join(dir_name, 'docs'))
+
+    if not os.path.isdir(os.path.join(dir_name, 'csvs')): 
+        os.mkdir(os.path.join(dir_name, 'csvs'))
+
+
+
+def create_season_dirs(file_info):
+    """
+    Creates the infrastructure to hold all the scraped docs for a season
+    
+    :param file_info: Dictionary containing the info on the file. Includes the name, season, file type, and the dir
+                      we want to deposit any data in.
+                        
+    :return: None
+    """
+    sub_folders = ["html_pbp", "json_pbp", "espn_pbp", "html_shifts_home", "html_shifts_away", 
+                   "json_shifts", "html_roster", "json_schedule", "espn_scoreboard"]
+
+    season_path = os.path.join(file_info['dir'], 'docs', str(file_info['season']))
+    os.mkdir(season_path)
+
+    for sub_f in sub_folders:
+        os.mkdir(os.path.join(season_path, sub_f))
+
+
+def check_file_exists(file_info):
+    """
+    Checks if the file exists. Also check if structure for holding scraped file exists to. If not, it creates it. 
+
+    :param file_info: Dictionary containing the info on the file. Includes the name, season, file type, and the dir
+                      we want to deposit any data in.
+
+    :return: Boolean - True if it exists
+    """
+    create_dir_structure(file_info['dir'])
+
+    # Check if the folder for the season for the given game was created yet...if not create it
+    if not os.path.isdir(os.path.join(file_info['dir'], 'docs', str(file_info['season']))):
+        create_season_dirs(file_info)
+
+    # May or may not be compressed due to file saved under older versions
+    non_compressed_file = os.path.isfile(create_base_file_path(file_info)) 
+    compressed_file = is_compressed(file_info)
+
+    return compressed_file or non_compressed_file
+
+
+def get_page(file_info):
+    """
+    Get the file so we don't need to re-scrape.
+
+    Try both compressed and non-compressed for backwards compatability issues (formerly non-compressed)
+
+    :param file_info: Dictionary containing the info on the file. Includes the name, season, file type, and the dir
+                      we want to deposit any data in.
+
+    :return: Response or None
+    """
+    base_file = create_base_file_path(file_info)
+
+    if is_compressed(file_info):
+        with gzip.open(base_file + ".gz", 'rb') as my_file:
+            return my_file.read().decode("utf-8").replace('\n', '')
+    else:
+        with open(base_file, 'r') as my_file:
+            return my_file.read().replace('\n', '')
+
+
+def save_page(page, file_info):
+    """
+    Save the page we just scraped.
+    
+    Note: It'll only get saved if the directory already exists!!!!!!. I'm not dealing with any fuck ups. That would 
+    involve checking if it's even a valid path and creating it. Make sure you get it right. 
+
+    :param page: File scraped
+    :param file_info: Dictionary containing the info on the file. Includes the name, season, file type, and the dir
+                      we want to deposit any data in.
+
+    :return: None
+    """
+    if file_info['dir'] and page is not None and page != '':
+        with gzip.open(create_base_file_path(file_info) + ".gz", 'wb') as file:
+            file.write(page.encode()) 

wsba_hockey/tools/utils/shared.py

@@ -0,0 +1,450 @@
+### CODE IN THIS DIRECTORY ORIGINALLY (FULLY OR PARITALLY) WRITTEN BY HARRY SHOMER IN THE "hockey_scraper" PACKAGE
+
+## shared.py ##
+"""
+This file is a bunch of the shared functions or just general stuff used by the different scrapers in the package.
+"""
+import os
+import time
+import json
+import logging
+import warnings
+import requests
+from datetime import datetime, timedelta
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+from . import save_pages as sp
+from . import config
+import inspect
+
+# Directory where this file lives
+FILE_DIR = os.path.dirname(os.path.realpath(__file__))
+
+# Name and Team fixes used 
+with open(os.path.join(FILE_DIR, "player_name_fixes.json"), "r" ,encoding="utf-8") as f:
+    Names = json.load(f)['fixes']
+
+with open(os.path.join(FILE_DIR, "team_tri_codes.json"), "r" ,encoding="utf-8") as f:
+    TEAMS = json.load(f)['teams']
+
+
+def fix_name(name):
+    """
+    Check if a name falls under those that need fixing. If it does...fix it.
+
+    :param name: name in pbp
+
+    :return: Either the given parameter or the fixed name
+    """
+    return Names.get(name.upper(), name.upper()).upper()
+
+
+def get_team(team):
+    """
+    Get the fucking team
+    """
+    return TEAMS.get(team.upper(), team.upper()).upper()
+    
+
+def custom_formatwarning(msg, *args, **kwargs): 
+    """
+    Override format for standard wanings
+    """
+    ansi_no_color = '\033[0m'
+    return "{msg}\n{no_color}".format(no_color=ansi_no_color, msg=msg)
+    
+warnings.formatwarning = custom_formatwarning
+
+
+def print_error(msg):
+    """
+    Implement own custom error using warning module. Prints in red
+
+    Reason why i still use warning for errors is so i can set to ignore them if i want to (e.g. live_scrape line 200).
+
+    :param msg: Str to print
+
+    :return: None
+    """
+    ansi_red_code = '\033[0;31m'
+    warning_msg = "{}Error: {}".format(ansi_red_code, msg)
+
+    # if config.LOG:
+    #     caller_file = os.path.basename(inspect.stack()[1].filename)
+    #     get_logger(caller_file).error(msg + " " + verbose)
+
+    warnings.warn(warning_msg) 
+
+
+def print_warning(msg):
+    """
+    Implement own custom warning using warning module. Prints in Orange.
+
+    :param msg: Str to print
+
+    :return: None
+    """
+    ansi_yellow_code = '\033[0;33m'
+    warning_msg = "{}Warning: {}".format(ansi_yellow_code, msg)
+
+    warnings.warn(warning_msg)
+
+
+def get_logger(python_file):
+    """
+    Create a basic logger to a log file
+
+    :param python_file: File that instantiates the logger instance
+    
+    :return: logger 
+    """
+    base_py_file = os.path.basename(python_file)
+
+    # If already exists we don't try to recreate it
+    if base_py_file in logging.Logger.manager.loggerDict.keys():
+        return logging.getLogger(base_py_file)
+
+    logger = logging.getLogger(base_py_file)
+    logger.setLevel(logging.INFO)  
+    
+    fh = logging.FileHandler("hockey_scraper_errors_{}.log".format(datetime.now().strftime("%Y-%m-%dT%H:%M:%S"))) 
+    fh.setFormatter(logging.Formatter('%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s', datefmt='%Y-%m-%d %I:%M:%S'))
+    logger.addHandler(fh)
+
+    return logger
+
+
+def log_error(err, py_file):
+    """
+    Log error when Logging is specified
+
+    :param err: Error to log
+    :param python_file: File that instantiates the logger instance
+
+    :return: None
+    """
+    if config.LOG:
+        get_logger(py_file).error(err)
+
+
+def get_season(date):
+    """
+    Get Season based on from_date
+
+    There is an exception for the 2019-2020 pandemic season. Accoding to the below url:
+        -  2019-2020 season ends in Oct. 2020
+        -  2020-2021 season begins in November 2020
+        -  https://nhl.nbcsports.com/2020/07/10/new-nhl-critical-dates-calendar-means-an-october-free-agent-frenzy/
+
+    :param date: date
+
+    :return: season -> ex: 2016 for 2016-2017 season
+    """
+    year = date[:4]
+    date = datetime.strptime(date, "%Y-%m-%d")
+    initial_bound = datetime.strptime('-'.join([year, '01-01']), "%Y-%m-%d")
+
+    # End bound for year1-year2 season is later for pandemic year
+    if initial_bound <= date <= season_end_bound(year):
+        return int(year) - 1
+
+    return int(year)
+
+
+def season_start_bound(year):
+    """
+    Get start bound for a season.
+
+    Notes:
+     - There is a bug in the schedule API for 2016 that causes the pushback to 09-30
+     - Pandemic season started in January
+
+    :param year: str of year for given date
+
+    :return: str of first date in season
+    """
+    if int(year) == 2016:
+        return "2016-09-30"
+        
+    if int(year) == 2020:
+        return '2021-01-01'
+
+    return "{}-09-01".format(str(year))
+
+
+
+def season_end_bound(year):
+    """
+    Determine the end bound of a given season. Changes depending on if it's the pandemic season or not
+
+    :param year: str of year for given date
+
+    :return: Datetime obj of last date in season
+    """
+    normal_end_bound = datetime.strptime('-'.join([str(year), '08-31']), "%Y-%m-%d")
+    pandemic_end_bound = datetime.strptime('-'.join([str(year), '10-31']), "%Y-%m-%d")
+
+    if int(year) == 2020:
+        return pandemic_end_bound
+
+    return normal_end_bound
+
+
+def convert_to_seconds(minutes):
+    """
+    Return minutes elapsed in time format to seconds elapsed
+
+    :param minutes: time elapsed
+
+    :return: time elapsed in seconds
+    """
+    if minutes == '-16:0-':
+        return '1200'      # Sometimes in the html at the end of the game the time is -16:0-
+
+    # If the time is junk not much i can do
+    try:
+        x = time.strptime(minutes.strip(' '), '%M:%S')
+    except ValueError:
+        return None
+
+    return timedelta(hours=x.tm_hour, minutes=x.tm_min, seconds=x.tm_sec).total_seconds()
+
+
+def if_rescrape(user_rescrape):
+    """
+    If you want to re_scrape. If someone is a dumbass and feeds it a non-boolean it terminates the program
+
+    Note: Only matters when you have a directory specified
+
+    :param user_rescrape: Boolean
+
+    :return: None
+    """
+    if isinstance(user_rescrape, bool):
+        config.RESCRAPE = user_rescrape
+    else:
+        raise ValueError("Error: 'if_rescrape' must be a boolean. Not a {}".format(type(user_rescrape)))
+
+
+def add_dir(user_dir):
+    """
+    Add directory to store scraped docs if valid. Or create in the home dir
+
+    NOTE: After this functions docs_dir is either None or a valid directory
+
+    :param user_dir: If bool=True create in the home dire or if user provided directory on their machine
+
+    :return: None
+    """
+    # False so they don't want it
+    if not user_dir:
+        config.DOCS_DIR = False
+        return
+
+    # Something was given
+    # Either True or string to directory
+    # If boolean refer to the home directory
+    if isinstance(user_dir, bool):
+        config.DOCS_DIR = os.path.join(os.path.expanduser('~'), "hockey_scraper_data")
+        # Create if needed
+        if not os.path.isdir(config.DOCS_DIR):
+            print_warning("Creating the hockey_scraper_data directory in the home directory")
+            os.mkdir(config.DOCS_DIR)
+    elif isinstance(user_dir, str) and os.path.isdir(user_dir):
+        config.DOCS_DIR = user_dir
+    elif not (isinstance(user_dir, str) and isinstance(user_dir, bool)):
+        config.DOCS_DIR = False
+        print_error("The docs_dir argument provided is invalid")
+    else:
+        config.DOCS_DIR = False
+        print_error("The directory specified for the saving of scraped docs doesn't exist. Therefore:"
+              "\n1. All specified games will be scraped from their appropriate sources (NHL or ESPN)."
+              "\n2. All scraped files will NOT be saved at all. Please either create the directory you want them to be "
+              "deposited in or recheck the directory you typed in and start again.\n")
+
+
+def scrape_page(url):
+    """
+    Scrape a given url
+
+    :param url: url for page
+
+    :return: response object
+    """
+    response = requests.Session()
+    retries = Retry(total=10, backoff_factor=.1)
+    response.mount('http://', HTTPAdapter(max_retries=retries))
+
+    try:
+        response = response.get(url, timeout=5)
+        response.raise_for_status()
+        page = response.text
+    except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError):
+        page = None
+    except requests.exceptions.ReadTimeout:
+        # If it times out and it's the schedule print an error message...otherwise just make the page = None
+        if "schedule" in url:
+            raise Exception("Timeout Error: The NHL API took too long to respond to our request. "
+                                "Please Try Again (you may need to try a few times before it works). ")
+        else:
+            print_error("Timeout Error: The server took too long to respond to our request.")
+            page = None
+
+    # Pause for 1 second - make it more if you want
+    time.sleep(1)
+
+    return page
+
+
+
+def get_file(file_info, force=False):
+    """
+    Get the specified file.
+
+    If a docs_dir is provided we check if it exists. If it does we see if it contains that page (and saves if it
+    doesn't). If the docs_dir doesn't exist we just scrape from the source and not save.
+
+    :param file_info: Dictionary containing the info for the file.
+                      Contains the url, name, type, and season
+    :param force: Force a rescrape. Default is False
+
+    :return: page
+    """
+    file_info['dir'] = config.DOCS_DIR
+
+    # If everything checks out we'll retrieve it, otherwise we scrape it
+    if file_info['dir'] and sp.check_file_exists(file_info) and not config.RESCRAPE and not force:
+        page = sp.get_page(file_info)
+    else:
+        page = scrape_page(file_info['url'])
+        sp.save_page(page, file_info)
+
+    return page
+
+
+def check_data_format(data_format):
+    """
+    Checks if data_format specified (if it is at all) is either None, 'Csv', or 'pandas'.
+    It exits program with error message if input isn't good.
+
+    :param data_format: data_format provided 
+
+    :return: Boolean - True if good
+    """
+    if not data_format or data_format.lower() not in ['csv', 'pandas']:
+        raise ValueError('{} is an unspecified data format. The two options are Csv and Pandas '
+                            '(Csv is default)\n'.format(data_format))
+
+
+def check_valid_dates(from_date, to_date):
+    """
+    Check if it's a valid date range
+
+    :param from_date: date should scrape from
+    :param to_date: date should scrape to
+
+    :return: None
+    """
+    try:
+        if time.strptime(to_date, "%Y-%m-%d") < time.strptime(from_date, "%Y-%m-%d"):
+            raise ValueError("Error: The second date input is earlier than the first one")
+    except ValueError:
+        raise ValueError("Error: Incorrect format given for dates. They must be given like 'yyyy-mm-dd' "
+                            "(ex: '2016-10-01').")
+
+
+def to_csv(base_file_name, df, league, file_type):
+    """
+    Write DataFrame to csv file
+
+    :param base_file_name: name of file
+    :param df: DataFrame
+    :param league: nhl or nwhl
+    :param file_type: type of file despoiting
+
+    :return: None
+    """
+    docs_dir = config.DOCS_DIR
+
+    # This was a late addition so we add support here
+    if isinstance(docs_dir, str) and not os.path.isdir(os.path.join(docs_dir, "csvs")):
+        os.mkdir(os.path.join(docs_dir, "csvs"))
+
+    if df is not None:
+        if isinstance(docs_dir, str):
+            file_name = os.path.join(docs_dir, "csvs", '{}_{}_{}.csv'.format(league, file_type, base_file_name))
+        else:
+            file_name = '{}_{}_{}.csv'.format(league, file_type, base_file_name)
+
+        print("---> {} {} data deposited in file - {}".format(league, file_type, file_name))
+        df.to_csv(file_name, sep=',', encoding='utf-8')
+
+import re
+from bs4 import BeautifulSoup, SoupStrainer
+
+## html_pbp.py ##
+def get_contents(game_html):
+    """
+    Uses Beautiful soup to parses the html document.
+    Some parsers work for some pages but don't work for others....I'm not sure why so I just try them all here in order
+    
+    :param game_html: html doc
+    
+    :return: "soupified" html 
+    """
+    parsers = ["html5lib", "lxml", "html.parser"]
+    strainer = SoupStrainer('td', attrs={'class': re.compile(r'bborder')})
+
+    for parser in parsers:
+        # parse_only only works with lxml for some reason
+        if parser == "lxml":
+            soup = BeautifulSoup(game_html, parser, parse_only=strainer)
+        else:
+            soup = BeautifulSoup(game_html, parser)
+
+        tds = soup.find_all("td", {"class": re.compile('.*bborder.*')})
+
+        if len(tds) > 0:
+            break
+
+    return tds
+
+## html_shifts.py ##
+def get_soup(shifts_html):
+    """
+    Uses Beautiful soup to parses the html document.
+    Some parsers work for some pages but don't work for others....I'm not sure why so I just try them all here in order
+    
+    :param shifts_html: html doc
+    
+    :return: "soupified" html and player_shifts portion of html (it's a bunch of td tags)
+    """
+    parsers = ["lxml", "html.parser", "html5lib"]
+
+    for parser in parsers:
+        soup = BeautifulSoup(shifts_html, parser)
+        td = soup.findAll(True, {'class': ['playerHeading + border', 'lborder + bborder']})
+
+        if len(td) > 0:
+            break
+
+    return td, get_teams(soup)
+
+
+def get_teams(soup):
+    """
+    Return the team for the TOI tables and the home team
+    
+    :param soup: souped up html
+    
+    :return: list with team and home team
+    """
+    team = soup.find('td', class_='teamHeading + border')  # Team for shifts
+    team = team.get_text()
+
+    # Get Home Team
+    teams = soup.find_all('td', {'align': 'center', 'style': 'font-size: 10px;font-weight:bold'})
+    regex = re.compile(r'>(.*)<br/?>')
+    home_team = regex.findall(str(teams[7]))
+
+    return [team, home_team[0]]