wsba-hockey 0.1.1__py3-none-any.whl → 0.1.4__py3-none-any.whl

wsba_hockey/tools/scraping.py

@@ -1,105 +1,166 @@
 import re
-from bs4 import BeautifulSoup, SoupStrainer
-import hockey_scraper.utils.shared as shared
-import hockey_scraper.nhl.pbp.html_pbp as html
-import hockey_scraper.nhl.game_scraper as gs
+from bs4 import BeautifulSoup
+import requests as rs
+import json as json_lib
+from tools.utils.shared import *
 import numpy as np
 import pandas as pd
 import warnings
-import requests as rs
-from zipfile import ZipFile
 warnings.filterwarnings('ignore')
 
 ### SCRAPING FUNCTIONS ###
 # Provided in this file are functions vital to the scraping functions in the WSBA Hockey Python package. #
 
-## JSON DATA ##
-def retreive_players(json,result = "id"):
-    #Given json data from an NHL API call, return dictionary with home and away players and either their id or their position.
+## ORDER OF OPERATIONS ##
+# Create game information to use with all functions
+# Retreive JSON data
+# Parse JSON data
+# Retreive and clean HTML pbp with player information
+# Parse HTML pbp, return parsed HTML
+# Combine pbp data
+# Retreive and analyze HTML shifts with player information for home and away teams
+# Parse shift events
+# Combine all data, return complete play-by-play
+
+## UTILITY FUNCTIONS ##
+def get_col():
+    return [
+        'season','season_type','game_id','game_date',"start_time","venue","venue_location",
+        'away_team_abbr','home_team_abbr','event_num','period','period_type',
+        'seconds_elapsed',"situation_code","strength_state","strength_state_venue","home_team_defending_side",
+        "event_type_code","event_type","description","penalty_duration",
+        "event_team_abbr","event_team_venue",
+        'num_on', 'players_on','ids_on','num_off','players_off','ids_off','shift_type',
+        "event_player_1_name","event_player_2_name","event_player_3_name",
+        "event_player_1_id","event_player_2_id","event_player_3_id",
+        "event_player_1_pos","event_player_2_pos","event_player_3_pos",
+        "event_goalie_name","event_goalie_id",
+        "shot_type","zone_code","x","y","x_fixed","y_fixed","x_adj","y_adj",
+        "event_skaters","away_skaters","home_skaters",
+        "event_distance","event_angle","event_length","seconds_since_last",
+        "away_score","home_score", "away_fenwick", "home_fenwick","away_sog","home_sog",
+        "away_on_1","away_on_2","away_on_3","away_on_4","away_on_5","away_on_6","away_goalie",
+        "home_on_1","home_on_2","home_on_3","home_on_4","home_on_5","home_on_6","home_goalie",
+        "away_on_1_id","away_on_2_id","away_on_3_id","away_on_4_id","away_on_5_id","away_on_6_id","away_goalie_id",
+        "home_on_1_id","home_on_2_id","home_on_3_id","home_on_4_id","home_on_5_id","home_on_6_id","home_goalie_id",
+        "event_coach","away_coach","home_coach"
+    ]
+
+
+## JSON FUNCTIONS ##
+def get_game_roster(json):
+    #Given raw json data, return game rosters
     roster = pd.json_normalize(json['rosterSpots'])
-    info = pd.json_normalize(json)
-    home = info['homeTeam.id'][0]
-    away = info['awayTeam.id'][0]
-
-    #Add up to four alternative names for each player in the game
-    roster['playerName'] = roster['firstName.default']+" "+roster['lastName.default']
-    try: roster['playerName_2'] = roster['firstName.cs']+" "+roster['lastName.default'] 
-    except: roster['playerName_2'] = ""
-    try: roster['playerName_3'] = roster['firstName.de']+" "+roster['lastName.default']
-    except: roster['playerName_3'] = ""
-    try: roster['playerName_4'] = roster['firstName.es']+" "+roster['lastName.default']
-    except: roster['playerName_4'] = ""
-
-    #For each home/away player their name is included as a key and their id or position is the value
-    home_players = {}
-    home_id = roster.loc[roster['teamId']==home]
-    hid = list(home_id['playerId'])+list(home_id['playerId'])+list(home_id['playerId'])+list(home_id['playerId'])
-    hpos = list(home_id['positionCode'])+list(home_id['positionCode'])+list(home_id['positionCode'])+list(home_id['positionCode'])
-    hp = list(home_id['playerName'])+list(home_id['playerName_2'])+list(home_id['playerName_3'])+list(home_id['playerName_4'])
-    
-    for id, pos, player in zip(hid,hpos,hp):
-        try: home_players.update({player.upper():
-                        {result:id if result == 'id' else pos}})
-        except:
-            continue
+    roster['full_name'] = (roster['firstName.default'] + " " + roster['lastName.default']).str.upper()
 
-    away_players = {}
-    away_id = roster.loc[roster['teamId']==away]
-    aid = list(away_id['playerId'])+list(away_id['playerId'])+list(away_id['playerId'])+list(away_id['playerId'])
-    apos = list(away_id['positionCode'])+list(away_id['positionCode'])+list(away_id['positionCode'])+list(away_id['positionCode'])
-    ap = list(away_id['playerName'])+list(away_id['playerName_2'])+list(away_id['playerName_3'])+list(away_id['playerName_4'])
-    
-    for id, pos, player in zip(aid,apos,ap):
-        try: away_players.update({player.upper():
-                        {result:id if result == 'id' else pos}})
-        except:
-            continue
-    
-    #Return: Dict of away and home players keyed with id or position as value
-    return {
-        'home':home_players,
-        'away':away_players
-    }
+    #Return: roster information
+    return roster
 
-def parse_json(json):
-    #Given json data from an NHL API call, return play-by-play data.
+def get_game_coaches(game_id):
+    #Given game info, return head coaches for away and home team
+    
+    #Retreive data
+    json = rs.get(f'https://api-web.nhle.com/v1/gamecenter/{game_id}/right-rail').json()
+    data = json['gameInfo']
 
-    events = pd.json_normalize(json['plays']).reset_index(drop=True)
-    info = pd.json_normalize(json)
-    roster = pd.json_normalize(json['rosterSpots'])
+    #Add coaches
+    try:
+        away = data['awayTeam']['headCoach']['default'].upper()
+        home = data['homeTeam']['headCoach']['default'].upper()
+        
+        coaches = {'away':away,
+                'home':home}
+    except KeyError:
+        return {}
 
-    #Game information
-    events['game_id'] = info['id'][0]
-    events['season'] = info['season'][0]
-    events['season_type'] = info['gameType'][0]
-    events['game_date'] = info['gameDate'][0]
-    events['start_time'] = info['startTimeUTC'][0]
-    events['venue'] = info['venue.default'][0]
-    events['venue_location'] = info['venueLocation.default'][0]
-    events['away_team_id'] = info['awayTeam.id'][0]
-    events['away_team_abbr'] = info['awayTeam.abbrev'][0]
-    events['home_team_id'] = info['homeTeam.id'][0]
-    events['home_team_abbr'] = info['homeTeam.abbrev'][0]
-
-    teams = {
-        info['awayTeam.id'][0]:info['awayTeam.abbrev'][0],
-        info['homeTeam.id'][0]:info['homeTeam.abbrev'][0]
-    }
-
-    #Create player information dicts used to create event_player columns
-    roster['playerName'] = roster['firstName.default']+" "+roster['lastName.default']
-    players = {}
-    players_pos = {}
-    ids = {}
-    for id, player in zip(list(roster['playerId']),list(roster['playerName'])):
-        players.update({id:player.upper()})
-    for id, pos in zip(list(roster['playerId']),list(roster['positionCode'])):
-        players_pos.update({id:pos.upper()})
-    for id, player in zip(list(roster['playerId']),list(roster['playerName'])):
-        ids.update({player.upper():id})
+    #Return: dict with coaches
+    return coaches
+    
+def get_game_info(game_id):
+    #Given game_id, return game information
+    
+    #Retreive data
+    api = f"https://api-web.nhle.com/v1/gamecenter/{game_id}/play-by-play"
+    json = rs.get(api).json()
+
+    #Games don't always have JSON shifts, for whatever reason
+    shifts = f"https://api.nhle.com/stats/rest/en/shiftcharts?cayenneExp=gameId={game_id}"
+    shifts = rs.get(shifts).json()
+    json_shifts = pd.json_normalize(shifts['data'])
+    
+    if shifts['total'] == 0:
+        json_shifts = pd.DataFrame()
+
+    #Split information
+    base = pd.json_normalize(json)
+    game_id = base['id'][0]
+    season = base['season'][0]
+    season_type = base['gameType'][0]
+    game_date = base['gameDate'][0]
+    game_state = base['gameState'][0]
+    start_time = base['startTimeUTC'][0]
+    venue = base['venue.default'][0]
+    venue_location = base['venueLocation.default'][0]
+    away_team_id = base['awayTeam.id'][0]
+    away_team_abbr = base['awayTeam.abbrev'][0]
+    home_team_id = base['homeTeam.id'][0]
+    home_team_abbr = base['homeTeam.abbrev'][0]
+
+    #Add roster
+    roster = get_game_roster(json)
+    #In the HTML parsing process, player are identified by a regex pattern (ABB #00 such as BOS #37) or number and name in the following format: #00 NAME (i.e. #37 BERGERON) so these are added as IDs of sorts.  
+    roster['descID'] = '#'+roster['sweaterNumber'].astype(str)+" "+roster['lastName.default'].str.upper()
+    roster['team_abbr'] = roster['teamId'].replace({
+        away_team_id:[away_team_abbr],
+        home_team_id:[home_team_abbr]
+    })
+    roster['key'] = roster['team_abbr'] + " #" + roster['sweaterNumber'].astype(str)
 
+    #Create an additional roster dictionary for use with HTML parsing
+    #Roster dict
+    roster_dict = {'away':{},
+                   'home':{}}
+    
+    #Evaluate and add players by team
+    for team in ['away','home']:
+        abbr = (away_team_abbr if team == 'away' else home_team_abbr)
+        rost = roster.loc[roster['team_abbr']==abbr]
+        
+        #Now iterate through team players
+        for player,id,num,pos,team_abbr,key in zip(rost['full_name'],rost['playerId'],rost['sweaterNumber'],rost['positionCode'],rost['team_abbr'],rost['key']):
+            roster_dict[team].update({str(num):[key, pos, player, team_abbr, id]})
+
+    #Return: game information
+    return {"game_id":str(game_id),
+            "season":season,
+            "season_type":season_type,
+            "game_date":game_date,
+            "game_state":game_state,
+            "start_time":start_time,
+            'venue':venue,
+            'venue_location':venue_location,
+            'away_team_id':away_team_id,
+            'away_team_abbr':away_team_abbr,
+            'home_team_id':home_team_id,
+            'home_team_abbr':home_team_abbr,
+            'events':pd.json_normalize(json['plays']).reset_index(drop=True),
+            'rosters':roster,
+            'HTML_rosters':roster_dict,
+            'coaches':get_game_coaches(game_id),
+            'json_shifts':json_shifts}
+
+def parse_json(info):
+    #Given game info, return JSON document
+
+    #Retreive data
+    events = info['events']
+
+    #Return error if game is set in the future
+    if info['game_state'] == 'FUT':
+        raise ValueError(f"Game {info['id'][0]} has not occured yet.")
+    
     #Test columns
-    cols = ['eventId', 'timeInPeriod', 'timeRemaining', 'situationCode', 'homeTeamDefendingSide', 'typeCode', 'typeDescKey', 'sortOrder', 'periodDescriptor.number', 'periodDescriptor.periodType', 'periodDescriptor.maxRegulationPeriods', 'details.eventOwnerTeamId', 'details.losingPlayerId', 'details.winningPlayerId', 'details.xCoord', 'details.yCoord', 'details.zoneCode', 'pptReplayUrl', 'details.shotType', 'details.scoringPlayerId', 'details.scoringPlayerTotal', 'details.assist1PlayerId', 'details.assist1PlayerTotal', 'details.assist2PlayerId', 'details.assist2PlayerTotal', 'details.goalieInNetId', 'details.awayScore', 'details.homeScore', 'details.highlightClipSharingUrl', 'details.highlightClipSharingUrlFr', 'details.highlightClip', 'details.highlightClipFr', 'details.discreteClip', 'details.discreteClipFr', 'details.shootingPlayerId', 'details.awaySOG', 'details.homeSOG', 'details.playerId', 'details.hittingPlayerId', 'details.hitteePlayerId', 'details.reason', 'details.typeCode', 'details.descKey', 'details.duration', 'details.servedByPlayerId', 'details.secondaryReason', 'details.blockingPlayerId', 'details.committedByPlayerId', 'details.drawnByPlayerId', 'game_id', 'season', 'season_type', 'game_date', 'away_team_id', 'away_team_abbr', 'home_team_id', 'home_team_abbr']
+    cols = ['eventId', 'timeInPeriod', 'timeRemaining', 'situationCode', 'homeTeamDefendingSide', 'typeCode', 'typeDescKey', 'sortOrder', 'periodDescriptor.number', 'periodDescriptor.periodType', 'periodDescriptor.maxRegulationPeriods', 'details.eventOwnerTeamId', 'details.losingPlayerId', 'details.winningPlayerId', 'details.xCoord', 'details.yCoord', 'details.zoneCode', 'pptReplayUrl', 'details.shotType', 'details.scoringPlayerId', 'details.scoringPlayerTotal', 'details.assist1PlayerId', 'details.assist1PlayerTotal', 'details.assist2PlayerId', 'details.assist2PlayerTotal', 'details.goalieInNetId', 'details.awayScore', 'details.homeScore', 'details.highlightClipSharingUrl', 'details.highlightClipSharingUrlFr', 'details.highlightClip', 'details.highlightClipFr', 'details.discreteClip', 'details.discreteClipFr', 'details.shootingPlayerId', 'details.awaySOG', 'details.homeSOG', 'details.playerId', 'details.hittingPlayerId', 'details.hitteePlayerId', 'details.reason', 'details.typeCode', 'details.descKey', 'details.duration', 'details.servedByPlayerId', 'details.secondaryReason', 'details.blockingPlayerId', 'details.committedByPlayerId', 'details.drawnByPlayerId', 'game_id', 'season', 'season_type', 'game_date']
 
     for col in cols:
         try:events[col]
@@ -120,46 +181,38 @@ def parse_json(json):
 
     events['event_player_3_id'] = events['details.assist2PlayerId']
 
-    events['event_team_status'] = np.where(events['home_team_id']==events['details.eventOwnerTeamId'],"home","away")
+    events['event_team_venue'] = np.where(events['details.eventOwnerTeamId']==info['home_team_id'],"home","away")
 
     #Coordinate adjustments:
     #The WSBA NHL Scraper includes three sets of coordinates per event:
     # x, y - Raw coordinates from JSON pbpp
     # x_fixed, y_fixed - Coordinates fixed to the right side of the ice (x is always greater than 0)
     # x_adj, y_adj - Adjusted coordinates configuring away events with negative x vlaues while home events are always positive
-    events['x_fixed'] = abs(events['details.xCoord'])
-    events['y_fixed'] = np.where(events['details.xCoord']<0,-events['details.yCoord'],events['details.yCoord'])
-    events['x_adj'] = np.where(events['event_team_status']=="home",events['x_fixed'],-events['x_fixed'])
-    events['y_adj'] = np.where(events['event_team_status']=="home",events['y_fixed'],-events['y_fixed'])
-    events['event_distance'] = np.sqrt(((89 - events['x_fixed'])**2) + (events['y_fixed']**2))
-    events['event_angle'] = np.degrees(np.arctan2(abs(events['y_fixed']), abs(89 - events['x_fixed'])))
-    
-    events['event_team_abbr'] = events['details.eventOwnerTeamId'].replace(teams)
-
-    #Event player information includes ids (included in the JSON events), names (from "rosterSpots"), and positions (also from "rosterSpots")
-    events['event_player_1_name'] = events['event_player_1_id'].replace(players)
-    events['event_player_2_name'] = events['event_player_2_id'].replace(players)
-    events['event_player_3_name'] = events['event_player_3_id'].replace(players)
-
-    events['event_player_1_pos'] = events['event_player_1_id'].replace(players_pos)
-    events['event_player_2_pos'] = events['event_player_2_id'].replace(players_pos)
-    events['event_player_3_pos'] = events['event_player_3_id'].replace(players_pos)
-
-    events['event_goalie_name'] = events['details.goalieInNetId'].replace(players)
-
-    #Create situations given situation code (this is reconfigured with on ice skaters when provided shifts data)
-    events['away_skaters'] = events['situationCode'].astype(str).str.slice(start=1,stop=2)
-    events['home_skaters'] = events['situationCode'].astype(str).str.slice(start=2,stop=3)
-    events['event_skaters'] = np.where(events['event_team_abbr']==events['home_team_abbr'],events['home_skaters'],events['away_skaters'])
-    events['event_skaters_against'] = np.where(events['event_team_abbr']==events['home_team_abbr'],events['away_skaters'],events['home_skaters'])
-
-    events['strength_state'] = events['event_skaters']+"v"+events['event_skaters_against']
-    events['strength'] = np.where(events['event_skaters']==events['event_skaters_against'],
-                                  "EV",np.where(
-                                      events['event_skaters']>events['event_skaters_against'],
-                                      "PP","SH"
-                                  ))
     
+    #Some games (mostly preseason and all star games) do not include coordinates.  
+    try:
+        events['x_fixed'] = abs(events['details.xCoord'])
+        events['y_fixed'] = np.where(events['details.xCoord']<0,-events['details.yCoord'],events['details.yCoord'])
+        events['x_adj'] = np.where(events['event_team_venue']=="home",events['x_fixed'],-events['x_fixed'])
+        events['y_adj'] = np.where(events['event_team_venue']=="home",events['y_fixed'],-events['y_fixed'])
+        events['event_distance'] = np.sqrt(((89 - events['x_fixed'])**2) + (events['y_fixed']**2))
+        events['event_angle'] = np.degrees(np.arctan2(abs(events['y_fixed']), abs(89 - events['x_fixed'])))
+    except TypeError:
+        print(f"No coordinates found for game {info['id'][0]}...")
+    
+        events['x_fixed'] = np.nan
+        events['y_fixed'] = np.nan
+        events['x_adj'] = np.nan
+        events['y_adj'] = np.nan
+        events['event_distance'] = np.nan
+        events['event_angle'] = np.nan
+    
+    
+    events['event_team_abbr'] = events['details.eventOwnerTeamId'].replace({
+        info['away_team_id']:[info['away_team_abbr']],
+        info['home_team_id']:[info['home_team_abbr']]
+    })
+
     #Rename columns to follow WSBA naming conventions
     events = events.rename(columns={
         "eventId":"event_id",
@@ -184,14 +237,12 @@ def parse_json(json):
     })
 
     #Period time adjustments (only 'seconds_elapsed' is included in the resulting data)
-    events['period_time_simple'] = events['period_time_elasped'].str.replace(":","",regex=True)
-    events['period_seconds_elapsed'] = np.where(events['period_time_simple'].str.len()==3,
-                                           ((events['period_time_simple'].str[0].astype(int)*60)+events['period_time_simple'].str[-2:].astype(int)),
-                                           ((events['period_time_simple'].str[0:2].astype(int)*60)+events['period_time_simple'].str[-2:].astype(int)))
-    events['period_seconds_remaining'] = 1200-events['period_seconds_elapsed']
+    events['period_seconds_elapsed'] = events['period_time_elasped'].apply(convert_to_seconds)
     events['seconds_elapsed'] = ((events['period']-1)*1200)+events['period_seconds_elapsed']
+
+    events = events.loc[(events['event_type']!="")]
     
-    #The following code is utilized to generate score and fenwick columns for each event
+    #Assign score and fenwick for each event
     fenwick_events = ['missed-shot','shot-on-goal','goal']
     ag = 0
     ags = []
@@ -202,16 +253,16 @@ def parse_json(json):
     afs = []
     hf = 0
     hfs = []
-    for event,team in zip(list(events['event_type']),list(events['event_team_status'])):
+    for event,team in zip(list(events['event_type']),list(events['event_team_venue'])):
         if event in fenwick_events:
             if team == "home":
-                hf = hf+1
+                hf += 1
                 if event == 'goal':
-                    hg = hg+1
+                    hg += 1
             else:
-                af = af+1
+                af += 1
                 if event == 'goal':
-                    ag = ag+1
+                    ag += 1
        
         ags.append(ag)
         hgs.append(hg)
@@ -222,84 +273,561 @@ def parse_json(json):
     events['home_score'] = hgs
     events['away_fenwick'] = afs
     events['home_fenwick'] = hfs
-    
-    events = events.loc[(events['event_type']!="")&(events['event_type']!="game-end")]
 
-    #Return: dataframe with parsed games in event
+    #Return: dataframe with parsed game
     return events
 
+### ESPN SCRAPING FUNCTIONS ###
+def espn_game_id(date,away,home):
+    #Given a date formatted as YYYY-MM-DD and teams, return game id from ESPN schedule
+    date = date.replace("-","")
+
+    #Retreive data
+    api = f"https://site.api.espn.com/apis/site/v2/sports/hockey/nhl/scoreboard?dates={date}"
+    schedule = pd.json_normalize(rs.get(api).json()['events'])
+
+    #Create team abbreviation columns
+    schedule['away_team_abbr'] = schedule['shortName'].str[:3].str.strip(" ")
+    schedule['home_team_abbr'] = schedule['shortName'].str[-3:].str.strip(" ")
+    
+    #Modify team abbreviations as necessary
+    schedule = schedule.replace({
+        "LA":"LAK",
+        "NJ":"NJD",
+        "SJ":"SJS",
+        "TB":"TBL",
+    })
+
+    #Retreive game id
+    game_id = schedule.loc[(schedule['away_team_abbr']==away)&
+                           (schedule['home_team_abbr']==home),'id'].tolist()[0]
 
+    #Return: ESPN game id
+    return game_id
 
-## HTML DATA ##
-def get_soup(shifts_html):
-    #Parses provided shifts html with BeautifulSoup
-    #Utilizes method from Harry Shomer's hockey_scraper package
-    parsers = ["lxml", "html.parser", "html5lib"]
+def parse_espn(date,away,home):
+    #Given a date formatted as YYYY-MM-DD and teams, return game events
+    game_id = espn_game_id(date,away,home)
+    url = f'https://www.espn.com/nhl/playbyplay/_/gameId/{game_id}'
+    
+    #Code modified from Patrick Bacon
 
-    for parser in parsers:
-        soup = BeautifulSoup(shifts_html, parser)
-        td = soup.findAll(True, {'class': ['playerHeading + border', 'lborder + bborder']})
+    #Retreive game events as json
+    page = rs.get(url, headers={'User-Agent': 'Mozilla/5.0'}, timeout = 500)
+    soup = BeautifulSoup(page.content.decode('ISO-8859-1'), 'lxml', multi_valued_attributes = None)
+    json = json_lib.loads(str(soup).split('"playGrps":')[1].split(',"tms"')[0])
 
-        if len(td) > 0:
-            break
-    return td, get_teams(soup)
+    #DataFrame of time-related info for events
+    clock_df = pd.DataFrame()
 
+    for period in range(0, len(json)):
+        clock_df = clock_df._append(pd.DataFrame(json[period]))
 
-def get_teams(soup):
-    #Collects teams in given shifts html (parsed by Beautiful Soup)
-    #Utilizes method from Harry Shomer's hockey_scraper package
-    team = soup.find('td', class_='teamHeading + border')  # Team for shifts
-    team = team.get_text()
+    clock_df = clock_df[~pd.isna(clock_df.clock)]
+
+    # Needed to add .split(',"st":3')[0] for playoffs
+
+    #DataFrame of coordinates for events
+    coords_df = pd.DataFrame(json_lib.loads(str(soup).split('plays":')[1].split(',"st":1')[0].split(',"st":2')[0].split(',"st":3')[0]))
+
+    clock_df = clock_df.assign(
+        clock = clock_df.clock.apply(lambda x: x['displayValue'])
+    )
+
+    coords_df = coords_df.assign(
+        coords_x = coords_df[~pd.isna(coords_df.coordinate)].coordinate.apply(lambda x: x['x']).astype(int),
+        coords_y = coords_df[~pd.isna(coords_df.coordinate)].coordinate.apply(lambda y: y['y']).astype(int),
+        event_player_1_name = coords_df[~pd.isna(coords_df.athlete)]['athlete'].apply(lambda x: x['name'])
+    )
+
+    #Combine
+    espn_events = coords_df.merge(clock_df.loc[:, ['id', 'clock']])
+
+    espn_events = espn_events.assign(
+        period = espn_events['period'].apply(lambda x: x['number']),
+        minutes = espn_events['clock'].str.split(':').apply(lambda x: x[0]).astype(int),
+        seconds = espn_events['clock'].str.split(':').apply(lambda x: x[1]).astype(int),
+        event_type = espn_events['type'].apply(lambda x: x['txt'])
+    )
+
+    espn_events = espn_events.assign(coords_x = np.where((pd.isna(espn_events.coords_x)) & (pd.isna(espn_events.coords_y)) &
+                (espn_events.event_type=='Face Off'), 0, espn_events.coords_x
+    ),
+                      coords_y = np.where((pd.isna(espn_events.coords_x)) & (pd.isna(espn_events.coords_y)) &
+                (espn_events.event_type=='Face Off'), 0, espn_events.coords_y))
+
+    espn_events = espn_events[(~pd.isna(espn_events.coords_x)) & (~pd.isna(espn_events.coords_y)) & (~pd.isna(espn_events.event_player_1_name))]
+
+    espn_events = espn_events.assign(
+        coords_x = espn_events.coords_x.astype(int),
+        coords_y = espn_events.coords_y.astype(int)
+    )
+    
+    #Rename events
+    #The turnover event includes just one player in the event information, meaning takeaways will have no coordinates for play-by-plays created by ESPN scraping
+    espn_events['event_type'] = espn_events['event_type'].replace({
+        "Face Off":'faceoff',
+        "Hit":'hit',
+        "Shot":'shot-on-goal',
+        "Missed":'missed-shot',
+        "Blocked":'blocked-shot',
+        "Goal":'goal',
+        "Turnover":'giveaway',
+        "Delayed Penalty":'delayed-penalty',
+        "Penalty":'penalty',
+    })
+
+    #Period time adjustments (only 'seconds_elapsed' is included in the resulting data)
+    espn_events['period_time_simple'] = espn_events['clock'].str.replace(":","",regex=True)
+    espn_events['period_seconds_elapsed'] = np.where(espn_events['period_time_simple'].str.len()==3,
+                                            ((espn_events['period_time_simple'].str[0].astype(int)*60)+espn_events['period_time_simple'].str[-2:].astype(int)),
+                                            ((espn_events['period_time_simple'].str[0:2].astype(int)*60)+espn_events['period_time_simple'].str[-2:].astype(int)))
+    espn_events['seconds_elapsed'] = ((espn_events['period']-1)*1200)+espn_events['period_seconds_elapsed']
+
+    espn_events = espn_events.rename(columns = {'text':'description'})
+
+    #Add event team
+    espn_events['event_team_abbr'] = espn_events['homeAway'].replace({
+        "away":away,
+        "home":home
+    })
+
+    #Some games (mostly preseason and all star games) do not include coordinates.  
+    try:
+        espn_events['x_fixed'] = abs(espn_events['coords_x'])
+        espn_events['y_fixed'] = np.where(espn_events['coords_x']<0,-espn_events['coords_y'],espn_events['coords_y'])
+        espn_events['x_adj'] = np.where(espn_events['homeAway']=="home",espn_events['x_fixed'],-espn_events['x_fixed'])
+        espn_events['y_adj'] = np.where(espn_events['homeAway']=="home",espn_events['y_fixed'],-espn_events['y_fixed'])
+        espn_events['event_distance'] = np.sqrt(((89 - espn_events['x_fixed'])**2) + (espn_events['y_fixed']**2))
+        espn_events['event_angle'] = np.degrees(np.arctan2(abs(espn_events['y_fixed']), abs(89 - espn_events['x_fixed'])))
+    except TypeError:
+        print(f"No coordinates found for ESPN game...")
+    
+        espn_events['x_fixed'] = np.nan
+        espn_events['y_fixed'] = np.nan
+        espn_events['x_adj'] = np.nan
+        espn_events['y_adj'] = np.nan
+        espn_events['event_distance'] = np.nan
+        espn_events['event_angle'] = np.nan
+
+    #Assign score and fenwick for each event
+    fenwick_events = ['missed-shot','shot-on-goal','goal']
+    ag = 0
+    ags = []
+    hg = 0
+    hgs = []
+
+    af = 0
+    afs = []
+    hf = 0
+    hfs = []
+    for event,team in zip(list(espn_events['event_type']),list(espn_events['homeAway'])):
+        if event in fenwick_events:
+            if team == "home":
+                hf += 1
+                if event == 'goal':
+                    hg += 1
+            else:
+                af += 1
+                if event == 'goal':
+                    ag += 1
+       
+        ags.append(ag)
+        hgs.append(hg)
+        afs.append(af)
+        hfs.append(hf)
+
+    espn_events['away_score'] = ags
+    espn_events['home_score'] = hgs
+    espn_events['away_fenwick'] = afs
+    espn_events['home_fenwick'] = hfs
+    #Return: play-by-play events in supplied game from ESPN
+    return espn_events
+    
+## HTML PBP FUNCTIONS ##
+def strip_html_pbp(td,rosters):
+    #Given html row, parse data from HTML pbp
+    #Harry Shomer's Code (modified)
+    
+    #HTML Parsing
+    for y in range(len(td)):
+        # Get the 'br' tag for the time column...this get's us time remaining instead of elapsed and remaining combined
+        if y == 3:
+            td[y] = td[y].get_text()   # This gets us elapsed and remaining combined-< 3:0017:00
+            index = td[y].find(':')
+            td[y] = td[y][:index+3]
+        elif (y == 6 or y == 7) and td[0] != '#':
+            # 6 & 7-> These are the player 1 ice one's
+            # The second statement controls for when it's just a header
+            baz = td[y].find_all('td')
+            bar = [baz[z] for z in range(len(baz)) if z % 4 != 0]  # Because of previous step we get repeats...delete some
+
+            # The setup in the list is now: Name/Number->Position->Blank...and repeat
+            # Now strip all the html
+            players = []
+            for i in range(len(bar)):
+                if i % 3 == 0:
+                    try:
+                        #Using the supplied json we can bind player name and id to number and team
+                        #Find number and team of player then lookup roster dictionary
+                        
+                        number = bar[i].get_text().strip('\n')  # Get number and strip leading/trailing newlines
+                        if y == 6:
+                            team = 'away'
+                        else:
+                            team = 'home'
+                        
+                        id = rosters[team][str(number)][4]
+                        name = rosters[team][str(number)][2]
+                        position = rosters[team][str(number)][1]
+                        
+                    except KeyError:
+                        name = ''
+                        number = ''
+                        id = ''
+                elif i % 3 == 1:
+                    if name != '':
+                        players.append([name, number, position, id])
+
+            td[y] = players
+        else:
+            td[y] = td[y].get_text()
+
+    return td
+
+
+def clean_html_pbp(info):
+    #Harry Shomer's Code (modified)
 
-    # Get Home Team
-    teams = soup.find_all('td', {'align': 'center', 'style': 'font-size: 10px;font-weight:bold'})
-    regex = re.compile(r'>(.*)<br/?>')
-    home_team = regex.findall(str(teams[7]))
+    game_id = info['game_id']
+    #Retreive data
+    season = info['season']
+    doc = f"https://www.nhl.com/scores/htmlreports/{season}/PL{game_id[-6:]}.HTM"
+    html = rs.get(doc).content
+    soup = get_contents(html)
 
-    return [team, home_team[0]]
+    #Rosters
+    rosters = info['HTML_rosters']
 
-#PARSE FUNCTIONS
-def analyze_shifts(shift, name, team, home_team, player_ids):
+    # Create a list of lists (each length 8)...corresponds to 8 columns in html pbp
+    td = [soup[i:i + 8] for i in range(0, len(soup), 8)]
+
+    cleaned_html = [strip_html_pbp(x,rosters) for x in td]
+
+    return cleaned_html
+
+def parse_html(info):
+    #Given game info, return HTML event data
+
+    #Retreive game information and html events
+    rosters = info['HTML_rosters']
+    events = clean_html_pbp(info)
+
+    teams = {info['away_team_abbr']:['away'],
+             info['home_team_abbr']:['home']}
+    
+    #Parsing
+    event_log = []
+    for event in events:
+        events_dict = {}
+        if event[0] == "#" or event[4] in ['GOFF', 'EGT', 'PGSTR', 'PGEND', 'ANTHEM','SPC','PBOX','SOC'] or event[3]=='-16:0-':
+            continue
+        else:
+            #Event info
+            events_dict['event_num'] = int(event[0])
+            events_dict['period'] = int(event[1])
+            events_dict['strength'] = re.sub(u'\xa0'," ",event[2])
+            events_dict['period_time_elapsed'] = event[3]
+            events_dict['seconds_elapsed'] = convert_to_seconds(event[3]) + (1200*(int(event[1])-1))
+            events_dict['event_type'] = event[4]
+
+            desc = re.sub(u'\xa0'," ",event[5])
+            events_dict['description'] = desc
+
+            events_dict['shot_type'] = desc.split(",")[1].lower().strip(" ") if event[4] in ['BLOCK','MISS','SHOT','GOAL'] else ""
+            zone = [x for x in desc.split(',') if 'Zone' in x]
+            if not zone:
+                events_dict['zone_code'] = None
+            elif zone[0].find("Off") != -1:
+                events_dict['zone_code'] = 'O'
+            elif zone[0].find("Neu") != -1:
+                events_dict['zone_code'] = 'N'
+            elif zone[0].find("Def") != -1:
+                events_dict['zone_code'] = 'D'
+
+            #Convert team names for compatiblity
+            replace = [('LAK',"L.A"),('NJD',"N.J"),('SJS',"S.J"),('TBL',"T.B")]
+            for name, repl in replace:
+                desc = desc.replace(repl,name)
+            
+            event_team = desc[0:3] if desc[0:3] in teams.keys() else ""
+            events_dict['event_team_abbr'] = event_team
+
+            events_dict['away_team_abbr'] = info['away_team_abbr']
+            events_dict['home_team_abbr'] = info['home_team_abbr']
+
+            away_skaters = 0
+            away_goalie = 0
+            #Away on-ice
+            for i in range(len(event[6])):
+                player = event[6][i][0]
+                pos = event[6][i][2]
+                id = event[6][i][3]
+                
+                if pos == 'G':
+                    events_dict['away_goalie'] = player
+                    events_dict['away_goalie_id'] = id
+                    away_goalie += 1
+                else:
+                    events_dict[f'away_on_{i+1}'] = player
+                    events_dict[f'away_on_{i+1}_id'] = id
+                    away_skaters += 1
+
+            home_skaters = 0
+            home_goalie = 0
+            #Home on-ice
+            for i in range(len(event[7])):
+                player = event[7][i][0]
+                pos = event[7][i][2]    
+                id = event[7][i][3]
+                
+                if pos == 'G':
+                    events_dict['home_goalie'] = player
+                    events_dict['home_goalie_id'] = id
+                    home_goalie += 1
+                else:
+                    events_dict[f'home_on_{i+1}'] = player
+                    events_dict[f'home_on_{i+1}_id'] = id
+                    home_skaters += 1
+            
+            event_players = []
+            #Determine parsing route based on event
+            if event[4] in ['FAC','HIT','BLOCK','PENL']:
+                #Regex to find team and player number involved (finds all for each event)
+                #Code is modified from Harry Shomer in order to account for periods in a team abbreviation
+                regex = re.compile(r'([A-Z]{2,3}|\b[A-Z]\.[A-Z])\s+#(\d+)')
+                fac = regex.findall(desc)
+                #Filter incorrectly parsed teams
+                repl = []
+                for team, num in fac:
+                    if team in teams.keys():
+                        repl.append((team,num))
+                fac = repl
+
+                #Find first event player
+                ep1_num = ''
+                for i in range(len(fac)):
+                    team, num = fac[i]
+                    if team == event_team:
+                        ep1_num = num
+                        event_players.append(fac[i])
+                    else:
+                        continue
+                    
+                #Find other players
+                for i in range(len(fac)):
+                    team, num = fac[i]
+                    if num == ep1_num:
+                        continue
+                    else:
+                        event_players.append(fac[i])
+            elif event[4]=='GOAL':
+                #Parse goal
+                regex = re.compile(r'#(\d+)\s+')
+                goal = regex.findall(desc)
+                
+                #Add all involved players
+                for point in goal:
+                    #In this loop, point is a player number.  We can assign event_team to all players in a goal
+                    event_players.append((event_team,str(point)))
+            elif event[4]=='DELPEN':
+                #Don't parse DELPEN events 
+                #These events typically have no text but when they do it is often erroneous or otherwise problematic
+
+                ""
+            else:
+                #Parse single or no player events
+                regex = re.compile(r'#\d+')
+                fac = regex.findall(desc)
+
+                for i in range(len(fac)):
+                    num = fac[i].replace("#","")
+                    event_players.append((event_team,str(num)))
+
+            for i in range(len(event_players)):
+                #For each player, evaluate their event data, then retreive information from rosters
+                team, num = event_players[i]
+                
+                status = teams[team]
+                data = rosters[status[0]]
+
+                events_dict[f'event_player_{i+1}_name'] = data[str(num)][2]
+                events_dict[f'event_player_{i+1}_id'] = data[str(num)][4]
+                events_dict[f'event_player_{i+1}_pos'] = data[str(num)][1]
+
+            events_dict['away_skaters'] = away_skaters
+            events_dict['home_skaters'] = home_skaters
+            events_dict['away_goalie_in'] = away_goalie
+            events_dict['home_goalie_in'] = home_goalie
+
+            event_skaters = away_skaters if info['away_team_abbr'] == event_team else home_skaters
+            event_skaters_against = away_skaters if info['home_team_abbr'] == event_team else home_skaters
+            events_dict['strength_state'] = f'{event_skaters}v{event_skaters_against}'
+            events_dict['event_skaters'] = np.where(event_team == info['home_team_abbr'],home_skaters,away_skaters)
+
+        event_log.append(pd.DataFrame([events_dict]))
+    
+    data = pd.concat(event_log)
+    data['event_type'] = data['event_type'].replace({
+     "PGSTR": "pre-game-start",
+     "PGEND": "pre-game-end",
+     'GSTR':"game-start",
+     "ANTHEM":"anthem",
+     "PSTR":"period-start",
+     'FAC':"faceoff",
+     "SHOT":"shot-on-goal",
+     "BLOCK":"blocked-shot",
+     "STOP":"stoppage",
+     "MISS":"missed-shot",
+     "HIT":"hit",
+     "GOAL":"goal",
+     "GIVE":"giveaway",
+     "TAKE":"takeaway",
+     "DELPEN":"delayed-penalty",
+     "PENL":"penalty",
+     "CHL":"challenge",
+     "PEND":"period-end",
+     "GEND":"game-end"
+    })
+
+    #Return: parsed HTML pbp
+    return data
+
+def combine_pbp(info):
+    #Given game info, return complete play-by-play data for provided game
+
+    html_pbp = parse_html(info)
+
+    #Route data combining - json if season is after 2009-2010:
+    if str(info['season']) in ['20052006','20062007','20072008','20082009','20092010']:
+        #ESPN x HTML
+        espn_pbp = parse_espn(str(info['game_date']),info['away_team_abbr'],info['home_team_abbr']).rename(columns={'coords_x':'x',"coords_y":'y'}).drop(columns=['event_player_1_name'])
+        merge_col = ['period','seconds_elapsed','event_type','event_team_abbr']
+
+        df = pd.merge(html_pbp,espn_pbp,how='left',on=merge_col)
+
+    else:
+        #JSON x HTML
+        json_pbp = parse_json(info)
+        #Modify merge conditions and merge pbps
+        merge_col = ['period','seconds_elapsed','event_type','event_team_abbr','event_player_1_id']
+        html_pbp = html_pbp.drop(columns=['event_player_2_id','event_player_3_id','shot_type','zone_code'],errors='ignore')
+
+        #While rare sometimes column 'event_player_1_id' is interpreted differently between the two dataframes. 
+        html_pbp['event_player_1_id'] = html_pbp['event_player_1_id'].astype(object)
+        json_pbp['event_player_1_id'] = json_pbp['event_player_1_id'].astype(object)
+
+        df = pd.merge(html_pbp,json_pbp,how='left',on=merge_col)
+
+    #Add game info
+    info_col = ['season','season_type','game_id','game_date',"venue","venue_location",
+        'away_team_abbr','home_team_abbr']
+    
+    for col in info_col:
+        df[col] = info[col]
+
+    #Fill period_type column and assign shifts a sub-500 event code
+    df['period_type'] = np.where(df['period']<4,"REG",np.where(np.logical_and(df['period']==5,df['season_type']==2),"SO","OT"))
+    try: df['event_type_code'] = np.where(df['event_type']!='change',df['event_type_code'],499)
+    except:
+        ""
+    df = df.sort_values(['period','seconds_elapsed']).reset_index()
+
+    df['event_team_venue'] = np.where(df['event_team_abbr'].isna(),"",np.where(df['home_team_abbr']==df['event_team_abbr'],"home","away"))
+    
+    #Correct strength state for penalty shots and shootouts - most games dont have shifts in shootout and are disculuded otherwise
+    df['strength_state'] = np.where(np.logical_and(df['period'].astype(str)=='5',df['event_type'].isin(['missed-shot','shot-on-goal','goal'])),"1v0",df['strength_state'])
+    df['strength_state'] = np.where(df['description'].str.contains('Penalty Shot',case=False),"1v0",df['strength_state'])
+
+    col = [col for col in get_col() if col in df.columns.to_list()]
+    #Return: complete play-by-play information for provided game
+    return df[col]
+
+## SHIFT SCRAPING FUNCTIONS ##
+def parse_shifts_json(info):
+    #Given game info, return json shift chart
+
+    log = info['json_shifts']
+    #Filter non-shift events and duplicate events
+    log = log.loc[log['detailCode']==0].drop_duplicates(subset=['playerId','shiftNumber'])
+
+    #Add full name columns
+    log['player_name'] = (log['firstName'] + " " + log['lastName']).str.upper()
+
+    log = log.rename(columns={
+        'playerId':'player_id',
+        'teamAbbrev':'event_team_abbr',
+        'startTime':'start',
+        'endTime':'end'
+    })
+
+    #Convert time columns
+    log['start'] = log['start'].astype(str).apply(convert_to_seconds)
+    log['end'] = log['end'].astype(str).apply(convert_to_seconds)
+    log = log[['player_name','player_id',
+                'period','event_team_abbr',
+                'start','duration','end']]
+    
+    #Recalibrate duration
+    log['duration'] = log['end'] - log['start']
+
+    #Return: JSON shifts (seperated by team)
+    away = log.loc[log['event_team_abbr']==info['away_team_abbr']]
+    home = log.loc[log['event_team_abbr']==info['home_team_abbr']]
+
+    return {'away':away,
+            'home':home}
+
+def analyze_shifts(shift, id, name, pos, team):
     #Collects teams in given shifts html (parsed by Beautiful Soup)
     #Modified version of Harry Shomer's analyze_shifts function in the hockey_scraper package
     shifts = dict()
 
     shifts['player_name'] = name.upper()
+    shifts['player_id'] = id
+    shifts['player_pos'] = pos
     shifts['period'] = '4' if shift[1] == 'OT' else '5' if shift[1] == 'SO' else shift[1]
-    shifts['team_abbr'] = shared.get_team(team.strip(' '))
-    shifts['start'] = shared.convert_to_seconds(shift[2].split('/')[0])
-    shifts['duration'] = shared.convert_to_seconds(shift[4].split('/')[0])
+    shifts['event_team_abbr'] = get_team(team.strip(' '))
+    shifts['start'] = convert_to_seconds(shift[2].split('/')[0])
+    shifts['duration'] = convert_to_seconds(shift[4].split('/')[0])
 
-    # I've had problems with this one...if there are no digits the time is fucked up
+    #Sometimes there are no digits
     if re.compile(r'\d+').findall(shift[3].split('/')[0]):
-        shifts['end'] = shared.convert_to_seconds(shift[3].split('/')[0])
+        shifts['end'] = convert_to_seconds(shift[3].split('/')[0])
     else:
         shifts['end'] = shifts['start'] + shifts['duration']
-
-    try:
-        if home_team == team:
-            shifts['player_id'] = player_ids['home'][name.upper()]['id']
-        else:
-            shifts['player_id'] = player_ids['away'][name.upper()]['id']
-    except KeyError:
-        shifts['player_id'] = None
-
     return shifts
 
-def parse_shifts(html, player_ids, game_id):
-    #Two-stage parsing of shifts data for a single team in a provided game
-    #Stage one: create dataframe with raw individual shifts
-    #Stage two: convert shift events to play-by-play structure created with json_parsing
+def parse_shifts_html(info,home):
+    #Parsing of shifts data for a single team in a provided game
     #Modified version of Harry Shomer's parse_shifts function in the hockey_scraper package
 
+    #Roster info prep
+    roster = info['HTML_rosters']
 
+    rosters = roster['home' if home else 'away']
+    
     all_shifts = []
-    columns = ['game_id', 'player_name', 'player_id', 'period', 'team_abbr', 'start', 'end', 'duration']
+    #columns = ['game_id', 'player_name', 'player_id', 'period', 'team_abbr', 'start', 'end', 'duration']
 
-    td, teams = get_soup(html)
+    #Retreive HTML
+    game_id = info['game_id']
+    season = info['season']
+    link = f"https://www.nhl.com/scores/htmlreports/{season}/T{'H' if home else 'V'}{game_id[-6:]}.HTM"
+    doc = rs.get(link).content
+    td, teams = get_soup(doc)
 
     team = teams[0]
-    home_team = teams[1]
     players = dict()
 
     # Iterates through each player shifts table with the following data:
@@ -308,37 +836,55 @@ def parse_shifts(html, player_ids, game_id):
         t = t.get_text()
         if ',' in t:     # If a comma exists it is a player
             name = t
+            
             name = name.split(',')
-            name = ' '.join([name[1].strip(' '), name[0][2:].strip(' ')])
-            #name = shared.fix_name(name)
-            #This has been excluded as means to control the differences in names between the JSON and HTML documents
-            players[name] = dict()
-            players[name]['number'] = name[0][:2].strip()
-            players[name]['shifts'] = []
+            number = int(name[0][:2].strip())
+            id = rosters[str(number)][4]
+            players[id] = dict()
+
+            #HTML shift functions assess one team at a time, which simplifies the lookup process with number to name and id
+            
+            players[id]['name'] = rosters[str(number)][2]
+            players[id]['pos'] = rosters[str(number)][1]
+
+            players[id]['shifts'] = []
         else:
-            players[name]['shifts'].extend([t])
+            players[id]['shifts'].extend([t])
 
     for key in players.keys():
         # Create lists of shifts-table columns for analysis
         players[key]['shifts'] = [players[key]['shifts'][i:i + 5] for i in range(0, len(players[key]['shifts']), 5)]
 
+        name = players[key]['name']
+        pos = players[key]['pos']
+
         # Parsing
-        shifts = [analyze_shifts(shift, key, team, home_team, player_ids) for shift in players[key]['shifts']]
+        shifts = [analyze_shifts(shift, key, name, pos, team) for shift in players[key]['shifts']]
         all_shifts.extend(shifts)
 
     df = pd.DataFrame(all_shifts)
-    df['game_id'] = str(game_id)
 
-    shifts_raw = df[columns]
+    shifts_raw = df[df['duration'] > 0]
 
-    shifts_raw = shifts_raw[shifts_raw['duration'] > 0]
+    #Return: single-team individual shifts by player
+    return shifts_raw
+
+def parse_shift_events(info,home):
+    #Given game info and home team conditional, parse and convert document to shift events congruent to html play-by-play
+    
+    #Determine whether to use JSON shifts or HTML shifts
+    if len(info['json_shifts']) == 0:
+        shift = parse_shifts_html(info,home)
+    else:
+        shift = parse_shifts_json(info)['home' if home else 'away']
+
+    rosters = info['rosters']
 
-    # Second-stage beginds here
     # Identify shift starts for each shift event
-    shifts_on = shifts_raw.groupby(['team_abbr', 'period', 'start']).agg(
+    shifts_on = shift.groupby(['event_team_abbr', 'period', 'start']).agg(
         num_on=('player_name', 'size'),
         players_on=('player_name', lambda x: ', '.join(x)),
-        ids_on=('player_id', lambda x: ', '.join(map(str, x)))
+        ids_on=('player_id', lambda x: ', '.join(map(str,x))),
     ).reset_index()
 
     shifts_on = shifts_on.rename(columns={
@@ -346,10 +892,10 @@ def parse_shifts(html, player_ids, game_id):
     })
 
     # Identify shift stops for each shift event
-    shifts_off = shifts_raw.groupby(['team_abbr', 'period', 'end']).agg(
+    shifts_off = shift.groupby(['event_team_abbr', 'period', 'end']).agg(
         num_off=('player_name', 'size'),
         players_off=('player_name', lambda x: ', '.join(x)),
-        ids_off=('player_id', lambda x: ', '.join(map(str, x)))
+        ids_off=('player_id', lambda x: ', '.join(map(str,x))),
     ).reset_index()
 
     shifts_off = shifts_off.rename(columns={
@@ -357,57 +903,29 @@ def parse_shifts(html, player_ids, game_id):
     })
 
     # Merge and sort by time in game
-    shifts = pd.merge(shifts_on, shifts_off, on=['team_abbr', 'period', 'seconds_elapsed'], how='outer')
+    shifts = pd.merge(shifts_on, shifts_off, on=['event_team_abbr', 'period', 'seconds_elapsed'], how='outer')
     
-    shifts = shifts.sort_values('seconds_elapsed')
-
-    #Modify columns of new total shifts dataframe
-    shifts['period'] = shifts['period'].astype(int)
+    shifts['seconds_elapsed'] = shifts['seconds_elapsed'] + (1200*(shifts['period'].astype(int)-1))
     shifts['event_type'] = 'change'
-    shifts['seconds_elapsed'] = shifts['seconds_elapsed'] + (1200 * (shifts['period']-1))
-    shifts['game_seconds_remaining'] = 3600 - shifts['seconds_elapsed']
-
-    # Handle missing values at the start and end of periods
-    shifts['players_on'] = shifts['players_on'].fillna('None')
-    shifts['players_off'] = shifts['players_off'].fillna('None')
-    shifts['ids_on'] = shifts['ids_on'].fillna('0')
-    shifts['ids_off'] = shifts['ids_off'].fillna('0')
-    shifts['num_on'] = shifts['num_on'].fillna(0).astype(int)
-    shifts['num_off'] = shifts['num_off'].fillna(0).astype(int)
-
-    #Manual Team Rename
-    shifts['team_abbr'] = shifts['team_abbr'].replace({
-        "L.A":"LAK",
-        "N.J":"NJD",
-        "S.J":"SJS",
-        "T.B":"TBL"
-    })
 
-    #Return: shift events formatted similarly to json pbp: shootout changes are discluded
-    return shifts.loc[shifts['period']<5].rename(columns={'team_abbr':'event_team_abbr'})
+    #Shift events similar to html (remove shootout shifts)
+    shifts = shifts.loc[shifts['period'].astype(int)<5].sort_values(['period','seconds_elapsed'])
+
+    #Generate on-ice columns
+    skater_names = list(rosters.loc[rosters['positionCode']!="G",'playerId'].astype(str))
+    goalie_names = list(rosters.loc[rosters['positionCode']=="G",'playerId'].astype(str))
+    team = list(shift['event_team_abbr'])[0]
 
-def construct_skaters_matrix(rosters, shifts, team_abbr, home=True):
-    #Given roster info (from the retreive_players function), shifts df, and team, generate on_ice columns for shift events
-    #These on-ice columns configure the on-ice players for events in the json play by play as well
     skaters = pd.DataFrame()
     goalies = pd.DataFrame()
-    if home:
-        team = {key:value for key, value in rosters['home'].items() if value['pos'] != "G"}
-    else:
-        team = {key:value for key, value in rosters['away'].items() if value['pos'] != "G"}
-
-    names = list(team.keys())
-    try: names.remove("")
-    except ValueError: ""
-
-    for player in names:
+    for player in skater_names:
         #For each player in the game, determine when they began and ended shifts.  
         #With player names as columns, 1 represents a shift event a player was on the ice for while 0 represents off the ice
         on_ice = (np.cumsum(
-            shifts.loc[(shifts['event_team_abbr'] == team_abbr), 'players_on']
+            shifts.loc[(shifts['event_team_abbr'] == team), 'ids_on']
             .apply(str)
             .apply(lambda x: int(bool(re.search(player, x)))) -
-            shifts.loc[(shifts['event_team_abbr'] == team_abbr), 'players_off']
+            shifts.loc[(shifts['event_team_abbr'] == team), 'ids_off']
             .apply(str)
             .apply(lambda x: int(bool(re.search(player, x))))
         ))
@@ -415,32 +933,22 @@ def construct_skaters_matrix(rosters, shifts, team_abbr, home=True):
     
     skaters = skaters.fillna(0).astype(int)
 
-
     on_skaters = (skaters == 1).stack().reset_index()
     on_skaters = on_skaters[on_skaters[0]].groupby("level_0")["level_1"].apply(list).reset_index()
     
     max_players = 6
     for i in range(max_players):
-        on_skaters[f"{'home' if home else 'away'}_on_{i+1}"] = on_skaters["level_1"].apply(lambda x: x[i] if i < len(x) else " ")
+        on_skaters[f"{'home' if home else 'away'}_on_{i+1}_id"] = on_skaters["level_1"].apply(lambda x: x[i] if i < len(x) else " ")
     
     on_skaters = on_skaters.drop(columns=["level_1"]).rename(columns={"level_0": "row"})
     
-    #Repeat above process with goaltenders
-    if home:
-        team = {key:value for key, value in rosters['home'].items() if value['pos'] == "G"}
-    else:
-        team = {key:value for key, value in rosters['away'].items() if value['pos'] == "G"}
-    
-    names = list(team.keys())
-    try: names.remove("")
-    except ValueError: ""
-
-    for player in names:
+    #Repeat this process with goaltenders
+    for player in goalie_names:
         on_ice = (np.cumsum(
-            shifts.loc[(shifts['event_team_abbr'] == team_abbr), 'players_on']
+            shifts.loc[(shifts['event_team_abbr'] == team), 'ids_on']
             .apply(str)
             .apply(lambda x: int(bool(re.search(player, x)))) -
-            shifts.loc[(shifts['event_team_abbr'] == team_abbr), 'players_off']
+            shifts.loc[(shifts['event_team_abbr'] == team), 'ids_off']
             .apply(str)
             .apply(lambda x: int(bool(re.search(player, x))))
         ))
@@ -453,7 +961,7 @@ def construct_skaters_matrix(rosters, shifts, team_abbr, home=True):
     
     max_players = 1
     for i in range(max_players):
-        on_goalies[f"{'home' if home else 'away'}_goalie"] = on_goalies["level_1"].apply(lambda x: x[i] if i < len(x) else " ")
+        on_goalies[f"{'home' if home else 'away'}_goalie_id"] = on_goalies["level_1"].apply(lambda x: x[i] if i < len(x) else " ")
     
     on_goalies = on_goalies.drop(columns=["level_1"]).rename(columns={"level_0": "row"})
     
@@ -461,87 +969,100 @@ def construct_skaters_matrix(rosters, shifts, team_abbr, home=True):
     on_players = pd.merge(on_skaters,on_goalies,how='outer',on=['row'])
 
     shifts['row'] = shifts.index
-    
+
+    if home:
+        shifts['home_team_abbr'] = team
+    else:
+        shifts['away_team_abbr'] = team
     #Return: shift events with newly added on-ice columns.  NAN values are replaced with string "REMOVE" as means to create proper on-ice columns for json pbp
-    return pd.merge(shifts,on_players,how="outer",on=['row']).replace(np.nan,"REMOVE")
+    return pd.merge(shifts,on_players,how="outer",on=['row']).replace(np.nan,"")
 
-def combine_shifts(home_shift,away_shift,json,game_id):
-    #Given shifts html documents for home and away team, return shift events complete with both teams' changes in the provided game
-    data = retreive_players(json,result="pos")
-    data_id = retreive_players(json)
+## FINALIZE PBP FUNCTIONS ##
+def combine_shifts(info):
+    #Given game info, return complete shift events
 
-    away = parse_shifts(away_shift,data_id,game_id).sort_values(by=['period','seconds_elapsed'])
-    home = parse_shifts(home_shift,data_id,game_id).sort_values(by=['period','seconds_elapsed'])
+    #JSON Prep
+    roster = info['rosters']
 
-    away['row'] = away.index
-    home['row'] = home.index
-    
-    away_shifts = construct_skaters_matrix(data,away,pd.json_normalize(json)['awayTeam.abbrev'][0],False).fillna("REMOVE")
-    home_shifts = construct_skaters_matrix(data,home,pd.json_normalize(json)['homeTeam.abbrev'][0],True).fillna("REMOVE")
+    #Quickly combine shifts data
+    away = parse_shift_events(info,False)
+    home = parse_shift_events(info,True)
 
-    shifts = pd.concat([away_shifts,home_shifts]).sort_values(by=['period','seconds_elapsed'])
+    #Combine shifts
+    data = pd.concat([away,home]).sort_values(['period','seconds_elapsed'])
+
+    #Add game info
+    info_col = ['season','season_type','game_id','game_date',"venue","venue_location",
+        'away_team_abbr','home_team_abbr']
     
-    #Return: shifts dataframe with both teams' changes
-    return shifts.drop(columns=['row'])
+    for col in info_col:
+        data[col] = info[col]
+
+    #Create player information dicts to create on-ice names
+    roster['playerId'] = roster['playerId'].astype(str)
+    players = roster.set_index("playerId")['full_name'].to_dict()
 
-def fix_names(shifts_df,json):
-    #Uses alternative names provided in the json to search shifts and ensure both shifts and json dataframes use the same name for each player
-    data = pd.json_normalize(json['rosterSpots'])
-    data['fullName'] = (data['firstName.default']+" "+data['lastName.default']).str.upper()
+    for i in range(0,7):
+        if i == 6:
+            data['away_goalie'] = data['away_goalie_id'].replace(players)
+            data['home_goalie'] = data['home_goalie_id'].replace(players)
+        else:
+            data[f'away_on_{i+1}'] = data[f'away_on_{i+1}_id'].replace(players)
+            data[f'home_on_{i+1}'] = data[f'home_on_{i+1}_id'].replace(players)
 
-    alt_name_col = ['firstName.cs',	'firstName.de',	'firstName.es',	'firstName.fi',	'firstName.sk',	'firstName.sv']
-    for i in range(len(alt_name_col)):
-        try: data['fullName.'+str(i+1)] = np.where(data[alt_name_col[i]].notna(),(data[alt_name_col[i]].astype(str)+" "+data['lastName.default'].astype(str)).str.upper(),np.nan)
-        except: continue
+    data = data.sort_values(['period','seconds_elapsed'])
+    #Fill on-ice columns down
+    on_ice_col = ['away_on_1','away_on_2','away_on_3','away_on_4','away_on_5','away_on_6',
+                'away_on_1_id','away_on_2_id','away_on_3_id','away_on_4_id','away_on_5_id','away_on_6_id',
+                'home_on_1','home_on_2','home_on_3','home_on_4','home_on_5','home_on_6',
+                'home_on_1_id','home_on_2_id','home_on_3_id','home_on_4_id','home_on_5_id','home_on_6_id',
+                'away_goalie','home_goalie','away_goalie_id','home_goalie_id']
 
-    name_col = ['fullName',	'fullName.1',	'fullName.2',	'fullName.3',	'fullName.4',	'fullName.5', 'fullName.6']
+    for col in on_ice_col:
+        data[col] = data[col].ffill()
 
-    for name in name_col:
-        try: data[name]
-        except:
-            data[name] = np.nan
+    #Create strength state information
+    away_on = ['away_on_1_id','away_on_2_id','away_on_3_id','away_on_4_id','away_on_5_id','away_on_6_id',]
+    home_on = ['home_on_1_id','home_on_2_id','home_on_3_id','home_on_4_id','home_on_5_id','home_on_6_id',]
+    data['away_skaters'] = data[away_on].replace(r'^\s*$', np.nan, regex=True).notna().sum(axis=1)
+    data['home_skaters'] = data[home_on].replace(r'^\s*$', np.nan, regex=True).notna().sum(axis=1)
+    data['strength_state'] = np.where(data['event_team_abbr']==data['away_team_abbr'],data['away_skaters'].astype(str)+"v"+data['home_skaters'].astype(str),data['home_skaters'].astype(str)+"v"+data['away_skaters'].astype(str))
 
-    names_dfs = []
-    for name in name_col[1:len(name_col)]:
-        names_dfs.append(data[[name,'fullName']].rename(columns={name:"alt",
-                                            "fullName":'default'}))
+    #Return: full shifts data converted to play-by-play format
+    col = [col for col in get_col() if col in data.columns.to_list()]
+    return data[col]
 
-    names_df = pd.concat(names_dfs)
+def combine_data(info):
+    #Given game info, return complete play-by-play data
 
-    replace = {}
-    for default, alt in zip(names_df['default'],names_df['alt']):
-        if alt == np.nan or alt == "" or str(alt) == 'nan':
-            continue
-        else:
-            replace.update({alt:default})
-    
-    return shifts_df.replace(replace,regex=True)
+    game_id = info['game_id']
 
-def combine_data(json,html):
-    #Given json pbp and html shifts, total game play-by-play data is provided with additional and corrected details
-    df = pd.concat([json,html])
+    pbp = combine_pbp(info)
+    shifts = combine_shifts(info)
 
-    #Fill period_type column and assign shifts a sub-500 event code
-    df['period_type'] = np.where(df['period']<4,"REG",np.where(df['period']==4,"OT","SO"))
-    df['event_type_code'] = np.where(df['event_type']!='change',df['event_type_code'],499)
+    #Combine data    
+    df = pd.concat([pbp,shifts])
 
     #Create priority columns designed to order events that occur at the same time in a game
-    start_pri = ['period-start','game-start']
     even_pri = ['takeaway','giveaway','missed-shot','hit','shot-on-goal','blocked-shot']
-    df['priority'] = np.where(df['event_type'].isin(start_pri),0,
-                              np.where(df['event_type'].isin(even_pri),1,
+    df['priority'] = np.where(df['event_type'].isin(even_pri),1,
                               np.where(df['event_type']=='goal',2,
                               np.where(df['event_type']=='stoppage',3,
-                              np.where(df['event_type']=='penalty',4,
-                              np.where(df['event_type']=='change',5,
+                              np.where(df['event_type']=='delayed-penalty',4,
+                              np.where(df['event_type']=='penalty',5,
                               np.where(df['event_type']=='period-end',6,
-                              np.where(df['event_type']=='game-end',7,
-                              np.where(df['event_type']=='faceoff',8,9)))))))))
+                              np.where(df['event_type']=='change',7,
+                              np.where(df['event_type']=='game-end',8,
+                              np.where(df['event_type']=='period-start',9,
+                              np.where(df['event_type']=='faceoff',10,0))))))))))
+                              
+    df[['period','seconds_elapsed']] =  df[['period','seconds_elapsed']].astype(int)
+    df = df.sort_values(['period','seconds_elapsed','priority'])
     
-    df = df.sort_values(by=['period','seconds_elapsed','priority']).reset_index()
-    #Recreate event_num column to accurately depict the order of all events, including changes
+    #Recalibrate event_num column to accurately depict the order of all events, including changes
+    df.reset_index(inplace=True,drop=True)
     df['event_num'] = df.index+1
-    df['event_team_status'] = np.where(df['event_team_abbr'].isna(),"",np.where(df['home_team_abbr']==df['event_team_abbr'],"home","away"))
+    df['event_team_venue'] = np.where(df['event_team_abbr'].isna(),"",np.where(df['home_team_abbr']==df['event_team_abbr'],"home","away"))
     df['event_type_last'] = df['event_type'].shift(1)
     df['event_type_last_2'] = df['event_type_last'].shift(1)
     df['event_type_next'] = df['event_type'].shift(-1)
@@ -550,76 +1071,36 @@ def combine_data(json,html):
     period_end_secs = [0,1200,2400,3600,4800,6000,7200,8400,9600,10800]
     #Define shifts by "line-change" or "on-the-fly"
     df['shift_type'] = np.where(df['event_type']=='change',np.where(np.logical_or(np.logical_or(df['event_type_last'].isin(lag_events),df['event_type_last_2'].isin(lag_events),df['event_type_next'].isin(lead_events)),df['seconds_elapsed'].isin(period_end_secs)),"line-change","on-the-fly"),"")
-
-    #Descrpitions:
-    #HTML pbp includes descriptions for each event; without the HTML pbp, play descriptions must be generated
-    #Different, more originally formatting is employed with these descriptions in comparison to that provided in the HTML pbp
-    df['start_end_desc'] = np.where(df['event_type'].isin(['period-start','period-end']),df['away_team_abbr'] + "vs" + df['home_team_abbr'] + ": Period " + df['period'].astype(str) + " " + df['event_type'].str.replace("period-","",regex=True).str.capitalize(),np.nan)
-    df['take_give_desc'] = np.where(df['event_type'].isin(['takeaway','giveaway']),df['event_team_abbr'] + " " + df['event_type'].str.upper() + " by " + df['event_player_1_name'],np.nan)
-    df['stoppage_desc'] = np.where(df['event_type']=='stoppage',"STOPPAGE: " + df['reason'].str.replace("-"," ",regex=True).str.capitalize(),np.nan)
-    df['blocked_desc'] = np.where(df['event_type']=='blocked-shot',df['event_team_abbr'] + " SHOT from " + df['event_player_1_name'] + " BLOCKED by " + df['event_player_2_name'],np.nan)
-    df['missed_desc'] = np.where(df['event_type']=='missed-shot',df['event_team_abbr'] + " SHOT by " + df['event_player_1_name'] + " MISSED: " + df['reason'].astype(str).str.replace("-"," ",regex=True),np.nan)
-    df['sog_desc'] = np.where(df['event_type']=='shot-on-goal',df['event_team_abbr'] + " SHOT by " + df['event_player_1_name'] + " SAVED by " + df['event_goalie_name'],np.nan)
-    df['goal_desc'] = np.where(df['event_type']=='goal',df['event_team_abbr'] + " GOAL SCORED by " + df['event_player_1_name'],np.nan)
-    df['assist_desc'] = np.where(np.logical_and(df['event_type']=='goal',df['event_player_2_name'].notna())," ASSISTED by " + df['event_player_2_name'],"")
-    df['assist2_desc'] = np.where(np.logical_and(df['event_type']=='goal',df['event_player_3_name'].notna())," and ASSISTED by " + df['event_player_3_name'],"")
-    df['goal_desc_complete'] = df['goal_desc'] + df['assist_desc'] + df['assist2_desc']
-    df['hit_desc'] = np.where(df['event_type']=='hit',df['event_team_abbr'] + " HIT by " + df['event_player_1_name'] + " on " + df['event_player_2_name'],np.nan)
-    df['faceoff_desc'] = np.where(df['event_type']=='faceoff',"FACEOFF WON by " + df['event_player_1_name'] + " AGAINST " + df['event_player_2_name'],np.nan)
-    df['penalty_desc'] = np.where(df['event_type']=='penalty',df['event_team_abbr'] + " PENALTY on " + df['event_player_1_name'] + ": " + df['penalty_duration'].astype(str).str.replace(".0","",regex=True) + " minutes for " + df['penalty_description'].astype(str).str.replace("-"," ",regex=True).str.upper(),np.nan)
-    
-    df['description'] = df['start_end_desc'].combine_first(df['take_give_desc'])\
-                                            .combine_first(df['stoppage_desc'])\
-                                            .combine_first(df['blocked_desc'])\
-                                            .combine_first(df['missed_desc'])\
-                                            .combine_first(df['sog_desc'])\
-                                            .combine_first(df['goal_desc_complete'])\
-                                            .combine_first(df['hit_desc'])\
-                                            .combine_first(df['faceoff_desc'])\
-                                            .combine_first(df['penalty_desc'])
-    ffill_col = ['season','season_type','game_id','game_date',
-                 "start_time","venue","venue_location",
-        'away_team_abbr','home_team_abbr','home_team_defending_side',
-        'away_score','away_fenwick',
-        'home_score','home_fenwick',
-        'away_goalie','home_goalie']
-    away_on = ['away_on_1','away_on_2','away_on_3','away_on_4','away_on_5','away_on_6']
-    home_on = ['home_on_1','home_on_2','home_on_3','home_on_4','home_on_5','home_on_6']
-
-    #Forward fill appropriate columns
-    for col in ffill_col+away_on+home_on:
+    df['description'] = df['description'].combine_first(df['event_team_abbr']+" CHANGE: "+df['shift_type'])
+    try:
+        df['event_type_code'] = np.where(df['event_type']=='change',499,df['event_type_code'])
+    except:
+        ""
+
+    #Add time since last event and overall event length
+    df['seconds_since_last'] = df['seconds_elapsed'] - df['seconds_elapsed'].shift(1)
+    df['event_length'] = df['seconds_since_last'].shift(-1)
+
+    #Add fixed strength state column
+    df['strength_state_venue'] = df['away_skaters'].astype(str)+'v'+df['home_skaters'].astype(str)
+
+    #Retrieve coaches
+    coaches = info['coaches']
+    if not coaches:
+        df['away_coach'] = ""
+        df['home_coach'] = ""
+        df['event_coach'] = ""
+    else:
+        df['away_coach'] = coaches['away']
+        df['home_coach'] = coaches['home']
+        df['event_coach'] = np.where(df['event_team_abbr']==df['home_team_abbr'],coaches['home'],np.where(df['event_team_abbr']==df['away_team_abbr'],coaches['away'],""))
+        
+    #Forward fill as necessary
+    cols = ['period_type','home_team_defending_side','away_score','away_fenwick','home_score','home_fenwick','away_coach','home_coach']
+    for col in cols:
+        try: df[col]
+        except: df[col] = ""
         df[col] = df[col].ffill()
 
-    #Now that forward fill is complete, replace "REMOVE" with nan
-    df.replace("REMOVE",np.nan,inplace=True)
-    
-    #Reconfigure strength state and sitution codes
-    df['away_skaters'] = df[away_on].replace(r'^\s*$', np.nan, regex=True).notna().sum(axis=1)
-    df['home_skaters'] = df[home_on].replace(r'^\s*$', np.nan, regex=True).notna().sum(axis=1)
-    df['away_goalie_in'] = np.where(df['away_goalie'].replace(r'^\s*$', np.nan, regex=True).notna(),1,0)
-    df['home_goalie_in'] = np.where(df['home_goalie'].replace(r'^\s*$', np.nan, regex=True).notna(),1,0)
-
-    df['event_skaters'] = np.where(df['event_team_abbr']==df['home_team_abbr'],df['home_skaters'],df['away_skaters'])
-    df['event_skaters_against'] = np.where(df['event_team_abbr']==df['home_team_abbr'],df['away_skaters'],df['home_skaters'])
-
-    df['strength_state'] = df['event_skaters'].astype(str) + "v" + df['event_skaters_against'].astype(str)
-    df['situation_code'] = np.where(df['situation_code'].isna(),df['away_goalie_in'].astype(str) + df['away_skaters'].astype(str) + df['home_skaters'].astype(str) + df['home_goalie_in'].astype(str),df['situation_code'])
-    
-    col = [
-        'season','season_type','game_id','game_date',"start_time","venue","venue_location",
-        'away_team_abbr','home_team_abbr','event_num','period','period_type',
-        'seconds_elapsed', "situation_code","strength_state","home_team_defending_side","shift_type",
-        "event_type_code","event_type","description","reason","penalty_duration","penalty_description",
-        "event_team_abbr",'num_on', 'players_on', 'ids_on', 'num_off', 'players_off', 'ids_off',
-        "event_team_status","event_player_1_id","event_player_2_id","event_player_3_id",
-        "event_player_1_name","event_player_2_name","event_player_3_name","event_player_1_pos","event_player_2_pos",
-        "event_player_3_pos","event_goalie_id",
-        "event_goalie_name","shot_type","zone_code","x","y","x_fixed","y_fixed","x_adj","y_adj",
-        "event_skaters","away_skaters","home_skaters",
-        "event_distance","event_angle","away_score","home_score", "away_fenwick", "home_fenwick",
-        "away_on_1","away_on_2","away_on_3","away_on_4","away_on_5","away_on_6","away_goalie",
-        "home_on_1","home_on_2","home_on_3","home_on_4","home_on_5","home_on_6","home_goalie"
-    ]
-
     #Return: complete play-by-play with all important data for each event in a provided game
-    return df[col].replace(r'^\s*$', np.nan, regex=True)
+    return df[[col for col in get_col() if col in df.columns.to_list()]].replace(r'^\s*$', np.nan, regex=True)