wsba-hockey 1.0.3__py3-none-any.whl → 1.0.5__py3-none-any.whl

wsba_hockey/tools/scraping.py

@@ -1,11 +1,12 @@
 import re
-from bs4 import BeautifulSoup
-import requests as rs
-import json as json_lib
-from .utils.shared import *
+import warnings
+import os
 import numpy as np
 import pandas as pd
-import warnings
+import requests as rs
+import json as json_lib
+from bs4 import BeautifulSoup
+from wsba_hockey.tools.utils.shared import *
 warnings.filterwarnings('ignore')
 
 ### SCRAPING FUNCTIONS ###
@@ -27,8 +28,9 @@ def get_col():
     return [
         'season','season_type','game_id','game_date',"start_time","venue","venue_location",
         'away_team_abbr','home_team_abbr','event_num','period','period_type',
-        'seconds_elapsed',"situation_code","strength_state","strength_state_venue","home_team_defending_side",
-        "event_type_code","event_type","description","penalty_duration",
+        'seconds_elapsed',"strength_state","strength_state_venue","home_team_defending_side",
+        "event_type_code","event_type","description","event_reason",
+        "penalty_type","penalty_duration","penalty_attribution",
         "event_team_abbr","event_team_venue",
         'num_on', 'players_on','ids_on','num_off','players_off','ids_off','shift_type',
         "event_player_1_name","event_player_2_name","event_player_3_name",
@@ -38,7 +40,7 @@ def get_col():
         "shot_type","zone_code","x","y","x_fixed","y_fixed","x_adj","y_adj",
         "event_skaters","away_skaters","home_skaters",
         "event_distance","event_angle","event_length","seconds_since_last",
-        "away_score","home_score", "away_fenwick", "home_fenwick","away_sog","home_sog",
+        "away_score","home_score", "away_fenwick", "home_fenwick",
         "away_on_1","away_on_2","away_on_3","away_on_4","away_on_5","away_on_6","away_goalie",
         "home_on_1","home_on_2","home_on_3","home_on_4","home_on_5","home_on_6","home_goalie",
         "away_on_1_id","away_on_2_id","away_on_3_id","away_on_4_id","away_on_5_id","away_on_6_id","away_goalie_id",
@@ -46,6 +48,34 @@ def get_col():
         "event_coach","away_coach","home_coach"
     ]
 
+def med_x_coord(group):
+    #Calculate median x coordinate of a corsi shot for a team in a period to determine the direction they are shooting in that period (for coordinate adjustments and geometric calculations)
+    med_x = group.loc[group['event_type'].isin(['blocked-shot','missed-shot','shot-on-goal','goal']),'x'].median(skipna=True)
+    group['med_x'] = med_x
+
+    return group
+
+def adjust_coords(pbp):
+    #Given JSON or ESPN pbp data, return pbp with adjusted coordinates
+
+    #Recalibrate coordinates
+    #Determine the direction teams are shooting in a given period
+    pbp = pbp.groupby(['event_team_venue','period','game_id'],group_keys=False).apply(med_x_coord)
+
+    pbp = pbp.reset_index(drop=True)
+
+    #Adjust coordinates
+    pbp['x_adj'] = np.where((((pbp['event_team_venue']=='home')&(pbp['med_x'] < 0))|((pbp['event_team_venue']=='away')&(pbp['med_x'] > 0))),-pbp['x'],pbp['x'])
+
+    #Adjust y if necessary
+    pbp['y_adj'] = np.where((pbp['x']==pbp['x_adj']),pbp['y'],-pbp['y'])
+
+    #Calculate event distance and angle relative to venue location
+    pbp['event_distance'] = np.where(pbp['event_team_venue']=='home',np.sqrt(((89 - pbp['x_adj'])**2) + (pbp['y_adj']**2)),np.sqrt((((-89) - pbp['x_adj'])**2) + (pbp['y_adj']**2)))
+    pbp['event_angle'] = np.where(pbp['event_team_venue']=='away',np.degrees(np.arctan2(abs(pbp['y_adj']), abs(89 - pbp['x_adj']))),np.degrees(np.arctan2(abs(pbp['y_adj']), abs((-89) - pbp['x_adj']))))
+
+    #Return: pbp with adjiusted coordinates
+    return pbp
 
 ## JSON FUNCTIONS ##
 def get_game_roster(json):
@@ -183,31 +213,6 @@ def parse_json(info):
 
     events['event_team_venue'] = np.where(events['details.eventOwnerTeamId']==info['home_team_id'],"home","away")
 
-    #Coordinate adjustments:
-    #The WSBA NHL Scraper includes three sets of coordinates per event:
-    # x, y - Raw coordinates from JSON pbpp
-    # x_fixed, y_fixed - Coordinates fixed to the right side of the ice (x is always greater than 0)
-    # x_adj, y_adj - Adjusted coordinates configuring away events with negative x vlaues while home events are always positive
-    
-    #Some games (mostly preseason and all star games) do not include coordinates.  
-    try:
-        events['x_fixed'] = abs(events['details.xCoord'])
-        events['y_fixed'] = np.where(events['details.xCoord']<0,-events['details.yCoord'],events['details.yCoord'])
-        events['x_adj'] = np.where(events['event_team_venue']=="home",events['x_fixed'],-events['x_fixed'])
-        events['y_adj'] = np.where(events['event_team_venue']=="home",events['y_fixed'],-events['y_fixed'])
-        events['event_distance'] = np.sqrt(((89 - events['x_fixed'])**2) + (events['y_fixed']**2))
-        events['event_angle'] = np.degrees(np.arctan2(abs(events['y_fixed']), abs(89 - events['x_fixed'])))
-    except TypeError:
-        print(f"No coordinates found for game {info['id'][0]}...")
-    
-        events['x_fixed'] = np.nan
-        events['y_fixed'] = np.nan
-        events['x_adj'] = np.nan
-        events['y_adj'] = np.nan
-        events['event_distance'] = np.nan
-        events['event_angle'] = np.nan
-    
-    
     events['event_team_abbr'] = events['details.eventOwnerTeamId'].replace({
         info['away_team_id']:[info['away_team_abbr']],
         info['home_team_id']:[info['home_team_abbr']]
@@ -226,223 +231,43 @@ def parse_json(info):
         "typeDescKey":"event_type",
         "details.shotType":"shot_type",
         "details.duration":"penalty_duration",
-        "details.descKey":"penalty_description",
-        "details.reason":"reason",
+        "details.descKey":"penalty_type",
+        "details.typeCode":'penalty_attribution',
+        "details.reason":"event_reason",
         "details.zoneCode":"zone_code",
         "details.xCoord":"x",
         "details.yCoord":"y",
         "details.goalieInNetId": "event_goalie_id",
-        "details.awaySOG":"away_SOG",
-        "details.homeSOG":"home_SOG"
+        "details.awaySOG":"away_sog",
+        "details.homeSOG":"home_sog"
     })
 
+    #Coordinate adjustments:
+    # x, y - Raw coordinates from JSON pbp
+    # x_adj, y_adj - Adjusted coordinates configuring the away offensive zone to the left and the home offensive zone to the right
+    #Some games (mostly preseason and all star games) do not include coordinates. 
+    
+    try:
+        events = adjust_coords(events)
+
+    except KeyError:
+        print(f"No coordinates found for game {info['game_id'][0]}...")
+    
+        events['x_adj'] = np.nan
+        events['y_adj'] = np.nan
+        events['event_distance'] = np.nan
+        events['event_angle'] = np.nan
+        
     #Period time adjustments (only 'seconds_elapsed' is included in the resulting data)
     events['period_seconds_elapsed'] = events['period_time_elasped'].apply(convert_to_seconds)
     events['seconds_elapsed'] = ((events['period']-1)*1200)+events['period_seconds_elapsed']
 
     events = events.loc[(events['event_type']!="")]
-    
-    #Assign score and fenwick for each event
-    fenwick_events = ['missed-shot','shot-on-goal','goal']
-    ag = 0
-    ags = []
-    hg = 0
-    hgs = []
-
-    af = 0
-    afs = []
-    hf = 0
-    hfs = []
-    for event,team in zip(list(events['event_type']),list(events['event_team_venue'])):
-        if event in fenwick_events:
-            if team == "home":
-                hf += 1
-                if event == 'goal':
-                    hg += 1
-            else:
-                af += 1
-                if event == 'goal':
-                    ag += 1
-       
-        ags.append(ag)
-        hgs.append(hg)
-        afs.append(af)
-        hfs.append(hf)
-
-    events['away_score'] = ags
-    events['home_score'] = hgs
-    events['away_fenwick'] = afs
-    events['home_fenwick'] = hfs
 
     #Return: dataframe with parsed game
     return events
 
-### ESPN SCRAPING FUNCTIONS ###
-def espn_game_id(date,away,home):
-    #Given a date formatted as YYYY-MM-DD and teams, return game id from ESPN schedule
-    date = date.replace("-","")
-
-    #Retreive data
-    api = f"https://site.api.espn.com/apis/site/v2/sports/hockey/nhl/scoreboard?dates={date}"
-    schedule = pd.json_normalize(rs.get(api).json()['events'])
-
-    #Create team abbreviation columns
-    schedule['away_team_abbr'] = schedule['shortName'].str[:3].str.strip(" ")
-    schedule['home_team_abbr'] = schedule['shortName'].str[-3:].str.strip(" ")
-    
-    #Modify team abbreviations as necessary
-    schedule = schedule.replace({
-        "LA":"LAK",
-        "NJ":"NJD",
-        "SJ":"SJS",
-        "TB":"TBL",
-    })
-
-    #Retreive game id
-    game_id = schedule.loc[(schedule['away_team_abbr']==away)&
-                           (schedule['home_team_abbr']==home),'id'].tolist()[0]
-
-    #Return: ESPN game id
-    return game_id
 
-def parse_espn(date,away,home):
-    #Given a date formatted as YYYY-MM-DD and teams, return game events
-    game_id = espn_game_id(date,away,home)
-    url = f'https://www.espn.com/nhl/playbyplay/_/gameId/{game_id}'
-    
-    #Code modified from Patrick Bacon
-
-    #Retreive game events as json
-    page = rs.get(url, headers={'User-Agent': 'Mozilla/5.0'}, timeout = 500)
-    soup = BeautifulSoup(page.content.decode('ISO-8859-1'), 'lxml', multi_valued_attributes = None)
-    json = json_lib.loads(str(soup).split('"playGrps":')[1].split(',"tms"')[0])
-
-    #DataFrame of time-related info for events
-    clock_df = pd.DataFrame()
-
-    for period in range(0, len(json)):
-        clock_df = clock_df._append(pd.DataFrame(json[period]))
-
-    clock_df = clock_df[~pd.isna(clock_df.clock)]
-
-    # Needed to add .split(',"st":3')[0] for playoffs
-
-    #DataFrame of coordinates for events
-    coords_df = pd.DataFrame(json_lib.loads(str(soup).split('plays":')[1].split(',"st":1')[0].split(',"st":2')[0].split(',"st":3')[0]))
-
-    clock_df = clock_df.assign(
-        clock = clock_df.clock.apply(lambda x: x['displayValue'])
-    )
-
-    coords_df = coords_df.assign(
-        coords_x = coords_df[~pd.isna(coords_df.coordinate)].coordinate.apply(lambda x: x['x']).astype(int),
-        coords_y = coords_df[~pd.isna(coords_df.coordinate)].coordinate.apply(lambda y: y['y']).astype(int),
-    )
-
-    #Combine
-    espn_events = coords_df.merge(clock_df.loc[:, ['id', 'clock']])
-
-    espn_events = espn_events.assign(
-        period = espn_events['period'].apply(lambda x: x['number']),
-        minutes = espn_events['clock'].str.split(':').apply(lambda x: x[0]).astype(int),
-        seconds = espn_events['clock'].str.split(':').apply(lambda x: x[1]).astype(int),
-        event_type = espn_events['type'].apply(lambda x: x['txt'])
-    )
-
-    espn_events = espn_events.assign(coords_x = np.where((pd.isna(espn_events.coords_x)) & (pd.isna(espn_events.coords_y)) &
-                (espn_events.event_type=='Face Off'), 0, espn_events.coords_x
-    ),
-                      coords_y = np.where((pd.isna(espn_events.coords_x)) & (pd.isna(espn_events.coords_y)) &
-                (espn_events.event_type=='Face Off'), 0, espn_events.coords_y))
-
-    espn_events = espn_events[(~pd.isna(espn_events.coords_x)) & (~pd.isna(espn_events.coords_y))]
-
-    espn_events = espn_events.assign(
-        coords_x = espn_events.coords_x.astype(int),
-        coords_y = espn_events.coords_y.astype(int)
-    )
-    
-    #Rename events
-    #The turnover event includes just one player in the event information, meaning takeaways will have no coordinates for play-by-plays created by ESPN scraping
-    espn_events['event_type'] = espn_events['event_type'].replace({
-        "Face Off":'faceoff',
-        "Hit":'hit',
-        "Shot":'shot-on-goal',
-        "Missed":'missed-shot',
-        "Blocked":'blocked-shot',
-        "Goal":'goal',
-        "Turnover":'giveaway',
-        "Delayed Penalty":'delayed-penalty',
-        "Penalty":'penalty',
-    })
-
-    #Period time adjustments (only 'seconds_elapsed' is included in the resulting data)
-    espn_events['period_time_simple'] = espn_events['clock'].str.replace(":","",regex=True)
-    espn_events['period_seconds_elapsed'] = np.where(espn_events['period_time_simple'].str.len()==3,
-                                            ((espn_events['period_time_simple'].str[0].astype(int)*60)+espn_events['period_time_simple'].str[-2:].astype(int)),
-                                            ((espn_events['period_time_simple'].str[0:2].astype(int)*60)+espn_events['period_time_simple'].str[-2:].astype(int)))
-    espn_events['seconds_elapsed'] = ((espn_events['period']-1)*1200)+espn_events['period_seconds_elapsed']
-
-    espn_events = espn_events.rename(columns = {'text':'description'})
-
-    #Add event team
-    espn_events['event_team_abbr'] = espn_events['homeAway'].replace({
-        "away":away,
-        "home":home
-    })
-
-    #Some games (mostly preseason and all star games) do not include coordinates.  
-    try:
-        espn_events['x_fixed'] = abs(espn_events['coords_x'])
-        espn_events['y_fixed'] = np.where(espn_events['coords_x']<0,-espn_events['coords_y'],espn_events['coords_y'])
-        espn_events['x_adj'] = np.where(espn_events['homeAway']=="home",espn_events['x_fixed'],-espn_events['x_fixed'])
-        espn_events['y_adj'] = np.where(espn_events['homeAway']=="home",espn_events['y_fixed'],-espn_events['y_fixed'])
-        espn_events['event_distance'] = np.sqrt(((89 - espn_events['x_fixed'])**2) + (espn_events['y_fixed']**2))
-        espn_events['event_angle'] = np.degrees(np.arctan2(abs(espn_events['y_fixed']), abs(89 - espn_events['x_fixed'])))
-    except TypeError:
-        print(f"No coordinates found for ESPN game...")
-    
-        espn_events['x_fixed'] = np.nan
-        espn_events['y_fixed'] = np.nan
-        espn_events['x_adj'] = np.nan
-        espn_events['y_adj'] = np.nan
-        espn_events['event_distance'] = np.nan
-        espn_events['event_angle'] = np.nan
-
-    #Assign score and fenwick for each event
-    fenwick_events = ['missed-shot','shot-on-goal','goal']
-    ag = 0
-    ags = []
-    hg = 0
-    hgs = []
-
-    af = 0
-    afs = []
-    hf = 0
-    hfs = []
-    for event,team in zip(list(espn_events['event_type']),list(espn_events['homeAway'])):
-        if event in fenwick_events:
-            if team == "home":
-                hf += 1
-                if event == 'goal':
-                    hg += 1
-            else:
-                af += 1
-                if event == 'goal':
-                    ag += 1
-       
-        ags.append(ag)
-        hgs.append(hg)
-        afs.append(af)
-        hfs.append(hf)
-
-    espn_events['away_score'] = ags
-    espn_events['home_score'] = hgs
-    espn_events['away_fenwick'] = afs
-    espn_events['home_fenwick'] = hfs
-    #Return: play-by-play events in supplied game from ESPN
-    return espn_events
-    
 ## HTML PBP FUNCTIONS ##
 def strip_html_pbp(td,rosters):
     #Given html row, parse data from HTML pbp
@@ -529,7 +354,7 @@ def parse_html(info):
     event_log = []
     for event in events:
         events_dict = {}
-        if event[0] == "#" or event[4] in ['GOFF', 'EGT', 'PGSTR', 'PGEND', 'ANTHEM','SPC','PBOX','SOC'] or event[3]=='-16:0-':
+        if event[0] == "#" or event[4] in ['GOFF', 'EGT', 'PGSTR', 'PGEND', 'ANTHEM', 'SPC', 'PBOX', 'EISTR', 'EIEND','EGPID'] or event[3]=='-16:0-':
             continue
         else:
             #Event info
@@ -660,10 +485,15 @@ def parse_html(info):
                 status = teams[team]
                 data = rosters[status[0]]
 
-                events_dict[f'event_player_{i+1}_name'] = data[str(num)][2]
-                events_dict[f'event_player_{i+1}_id'] = data[str(num)][4]
-                events_dict[f'event_player_{i+1}_pos'] = data[str(num)][1]
+                #In rare instances the event player is not on the event team (i.e. "WSH TAKEAWAY - #71 CIRELLI, Off. Zone" when #71 CIRELLI is on TBL)
+                try:
+                    events_dict[f'event_player_{i+1}_name'] = data[str(num)][2]
+                    events_dict[f'event_player_{i+1}_id'] = data[str(num)][4]
+                    events_dict[f'event_player_{i+1}_pos'] = data[str(num)][1]
+                except:
+                    ''
 
+            #Event skaters and strength-state information
             events_dict['away_skaters'] = away_skaters
             events_dict['home_skaters'] = home_skaters
             events_dict['away_goalie_in'] = away_goalie
@@ -683,7 +513,7 @@ def parse_html(info):
      'GSTR':"game-start",
      "ANTHEM":"anthem",
      "PSTR":"period-start",
-     'FAC':"faceoff",
+     "FAC":"faceoff",
      "SHOT":"shot-on-goal",
      "BLOCK":"blocked-shot",
      "STOP":"stoppage",
@@ -695,14 +525,28 @@ def parse_html(info):
      "DELPEN":"delayed-penalty",
      "PENL":"penalty",
      "CHL":"challenge",
+     "SOC":'shootout-complete',
      "PEND":"period-end",
      "GEND":"game-end"
     })
-
+    
     #Return: parsed HTML pbp
     return data
 
-def combine_pbp(info):
+def assign_target(data):
+    #Assign target number to plays to assist with merging
+
+    #New sort
+    data = data.sort_values(['period','seconds_elapsed','event_type','event_team_abbr','event_player_1_id','event_player_2_id'])
+
+    #Target number distingushes events that occur in the same second to assist in merging the JSON and HTML
+    #Sometimes the target number may not reflect the same order as the event number in either document (especially in earlier seasons where the events are out of order in the HTML or JSON)
+    data['target_num'] = np.where(data['event_type'].isin(['penalty','blocked-shot','missed-shot','shot-on-goal','goal']),data['event_type'].isin(['penalty','blocked-shot','missed-shot','shot-on-goal','goal']).cumsum(),0)
+
+    #Revert sort and return dataframe
+    return data.reset_index()
+
+def combine_pbp(info,sources):
     #Given game info, return complete play-by-play data for provided game
 
     html_pbp = parse_html(info)
@@ -710,23 +554,49 @@ def combine_pbp(info):
     #Route data combining - json if season is after 2009-2010:
     if str(info['season']) in ['20052006','20062007','20072008','20082009','20092010']:
         #ESPN x HTML
-        espn_pbp = parse_espn(str(info['game_date']),info['away_team_abbr'],info['home_team_abbr']).rename(columns={'coords_x':'x',"coords_y":'y'})
-        merge_col = ['period','seconds_elapsed','event_type','event_team_abbr']
+        #espn_pbp = parse_espn(str(info['game_date']),info['away_team_abbr'],info['home_team_abbr']).rename(columns={'coords_x':'x',"coords_y":'y'}).sort_values(['period','seconds_elapsed']).reset_index()
+        #merge_col = ['period','seconds_elapsed','event_type','event_team_abbr']
 
-        df = pd.merge(html_pbp,espn_pbp,how='left',on=merge_col)
+        #Merge pbp
+        #df = pd.merge(html_pbp,espn_pbp,how='left',on=merge_col)
+        print('In-repair, please try again later...')
 
     else:
         #JSON x HTML
         json_pbp = parse_json(info)
-        #Modify merge conditions and merge pbps
-        merge_col = ['period','seconds_elapsed','event_type','event_team_abbr','event_player_1_id']
-        html_pbp = html_pbp.drop(columns=['event_player_2_id','event_player_3_id','shot_type','zone_code'],errors='ignore')
 
-        #While rare sometimes column 'event_player_1_id' is interpreted differently between the two dataframes. 
-        html_pbp['event_player_1_id'] = html_pbp['event_player_1_id'].astype(object)
-        json_pbp['event_player_1_id'] = json_pbp['event_player_1_id'].astype(object)
+        if sources:
+            dirs_html = f'sources/{info['season']}/HTML/'
+            dirs_json = f'sources/{info['season']}/JSON/'
+
+            if not os.path.exists(dirs_html):
+                os.makedirs(dirs_html)
+            if not os.path.exists(dirs_json):
+                os.makedirs(dirs_json)
+
+            html_pbp.to_csv(f'{dirs_html}{info['game_id']}_HTML.csv',index=False)
+            json_pbp.to_csv(f'{dirs_json}{info['game_id']}_JSON.csv',index=False)
+        
+        #Assign target numbers
+        html_pbp = assign_target(html_pbp)
+        json_pbp = assign_target(json_pbp)
+
+        #Merge on index if the df lengths are the same and the events are in the same general order; merge on columns otherwise
+        if (len(html_pbp) == len(json_pbp)) and (html_pbp['event_type'].equals(json_pbp['event_type'])) and (html_pbp['seconds_elapsed'].equals(json_pbp['seconds_elapsed'])):
+            html_pbp = html_pbp.drop(columns=['period','seconds_elapsed','event_type','event_team_abbr','event_player_1_id','event_player_2_id','event_player_3_id','shot_type','zone_code'],errors='ignore').reset_index()
+            df = pd.merge(html_pbp,json_pbp,how='left',left_index=True,right_index=True).sort_values(['event_num'])
+        else:
+            print(f' merging on columns...',end="")
+            #Modify merge conditions and merge pbps
+            merge_col = ['period','seconds_elapsed','event_type','event_team_abbr','event_player_1_id','target_num']
+            html_pbp = html_pbp.drop(columns=['event_player_2_id','event_player_3_id','shot_type','zone_code'],errors='ignore')
+
+            #While rare sometimes column 'event_player_1_id' is interpreted differently between the two dataframes. 
+            html_pbp['event_player_1_id'] = html_pbp['event_player_1_id'].astype(object)
+            json_pbp['event_player_1_id'] = json_pbp['event_player_1_id'].astype(object)
 
-        df = pd.merge(html_pbp,json_pbp,how='left',on=merge_col)
+            #Merge pbp
+            df = pd.merge(html_pbp,json_pbp,how='left',on=merge_col).sort_values(['event_num'])
 
     #Add game info
     info_col = ['season','season_type','game_id','game_date',"venue","venue_location",
@@ -745,7 +615,7 @@ def combine_pbp(info):
     df['event_team_venue'] = np.where(df['event_team_abbr'].isna(),"",np.where(df['home_team_abbr']==df['event_team_abbr'],"home","away"))
     
     #Correct strength state for penalty shots and shootouts - most games dont have shifts in shootout and are disculuded otherwise
-    df['strength_state'] = np.where(np.logical_and(df['period'].astype(str)=='5',df['event_type'].isin(['missed-shot','shot-on-goal','goal'])),"1v0",df['strength_state'])
+    df['strength_state'] = np.where((df['period'].astype(str)=='5')&(df['event_type'].isin(['missed-shot','shot-on-goal','goal']))&(df['season_type']==2),"1v0",df['strength_state'])
     df['strength_state'] = np.where(df['description'].str.contains('Penalty Shot',case=False),"1v0",df['strength_state'])
 
     col = [col for col in get_col() if col in df.columns.to_list()]
@@ -977,7 +847,7 @@ def parse_shift_events(info,home):
     return pd.merge(shifts,on_players,how="outer",on=['row']).replace(np.nan,"")
 
 ## FINALIZE PBP FUNCTIONS ##
-def combine_shifts(info):
+def combine_shifts(info,sources):
     #Given game info, return complete shift events
 
     #JSON Prep
@@ -1027,21 +897,35 @@ def combine_shifts(info):
     data['home_skaters'] = data[home_on].replace(r'^\s*$', np.nan, regex=True).notna().sum(axis=1)
     data['strength_state'] = np.where(data['event_team_abbr']==data['away_team_abbr'],data['away_skaters'].astype(str)+"v"+data['home_skaters'].astype(str),data['home_skaters'].astype(str)+"v"+data['away_skaters'].astype(str))
 
-    #Return: full shifts data converted to play-by-play format
+    #Create final shifts df
     col = [col for col in get_col() if col in data.columns.to_list()]
-    return data[col]
+    full_shifts = data[col]
+    
+    #Export sources if true
+    if sources:
+        dirs = f'sources/{info['season']}/SHIFTS/'
+
+        if not os.path.exists(dirs):
+            os.makedirs(dirs)
+
+        full_shifts.to_csv(f'{dirs}{info['game_id']}_SHIFTS.csv',index=False)
+
+    #Return: full shifts data converted to play-by-play format
+    return full_shifts
 
-def combine_data(info):
+def combine_data(info,sources):
     #Given game info, return complete play-by-play data
 
     game_id = info['game_id']
 
-    pbp = combine_pbp(info)
-    shifts = combine_shifts(info)
+    pbp = combine_pbp(info,sources)
+    shifts = combine_shifts(info,sources)
 
     #Combine data    
     df = pd.concat([pbp,shifts])
 
+    df['event_num'] = df['event_num'].replace(np.nan,0)
+
     #Create priority columns designed to order events that occur at the same time in a game
     even_pri = ['takeaway','giveaway','missed-shot','hit','shot-on-goal','blocked-shot']
     df['priority'] = np.where(df['event_type'].isin(even_pri),1,
@@ -1054,9 +938,9 @@ def combine_data(info):
                               np.where(df['event_type']=='game-end',8,
                               np.where(df['event_type']=='period-start',9,
                               np.where(df['event_type']=='faceoff',10,0))))))))))
-                              
+
     df[['period','seconds_elapsed']] =  df[['period','seconds_elapsed']].astype(int)
-    df = df.sort_values(['period','seconds_elapsed','priority'])
+    df = df.sort_values(['period','seconds_elapsed','event_num','priority'])
     
     #Recalibrate event_num column to accurately depict the order of all events, including changes
     df.reset_index(inplace=True,drop=True)
@@ -1093,13 +977,20 @@ def combine_data(info):
         df['away_coach'] = coaches['away']
         df['home_coach'] = coaches['home']
         df['event_coach'] = np.where(df['event_team_abbr']==df['home_team_abbr'],coaches['home'],np.where(df['event_team_abbr']==df['away_team_abbr'],coaches['away'],""))
-        
+
+    #Assign score, corsi, fenwick, and penalties for each event
+    for venue in ['away','home']:
+        df[f'{venue}_score'] = ((df['event_team_venue']==venue)&(df['event_type']=='goal')).cumsum()
+        df[f'{venue}_corsi'] = ((df['event_team_venue']==venue)&(df['event_type'].isin(['blocked-shot','missed-shot','shot-on-goal','goal']))).cumsum()
+        df[f'{venue}_fenwick'] = ((df['event_team_venue']==venue)&(df['event_type'].isin(['missed-shot','shot-on-goal','goal']))).cumsum()
+        df[f'{venue}_penalties'] = ((df['event_team_venue']==venue)&(df['event_type']=='penalty')).cumsum()
+       
     #Forward fill as necessary
-    cols = ['period_type','home_team_defending_side','away_score','away_fenwick','home_score','home_fenwick','away_coach','home_coach']
+    cols = ['period_type','home_team_defending_side','away_coach','home_coach']
     for col in cols:
         try: df[col]
         except: df[col] = ""
         df[col] = df[col].ffill()
-
+     
     #Return: complete play-by-play with all important data for each event in a provided game
     return df[[col for col in get_col() if col in df.columns.to_list()]].replace(r'^\s*$', np.nan, regex=True)