TopDownHockey-Scraper 6.0.5__py3-none-any.whl → 6.0.9__py3-none-any.whl

TopDownHockey_Scraper/TopDownHockey_NHL_Scraper.py

@@ -18,6 +18,7 @@ import xml.etree.ElementTree as ET
 import xmltodict
 from xml.parsers.expat import ExpatError
 from requests.exceptions import ChunkedEncodingError
+import traceback
 
 print('Successfully did local install plus update')
 
@@ -834,8 +835,6 @@ def scrape_html_shifts(season, game_id, live = True):
         
     home_shifts = alldf
 
-    home_shifts.to_csv('/Users/patrickbacon/compact_topdownhockey/home_shifts.csv', index = False)
-
     if live == True:
 
         home_shifts = home_shifts.assign(shift_number = home_shifts.shift_number.astype(int))
@@ -956,8 +955,6 @@ def scrape_html_shifts(season, game_id, live = True):
         
     away_shifts = alldf
 
-    away_shifts.to_csv('/Users/patrickbacon/compact_topdownhockey/away_shifts.csv', index = False)
-
     if live == True:
 
         away_shifts = away_shifts.assign(shift_number = away_shifts.shift_number.astype(int))
@@ -1039,7 +1036,7 @@ def scrape_html_shifts(season, game_id, live = True):
         
         # Additional logic to handle period 1 scrape when we don't have goalie shifts yet. 
             
-        if len(home_shifts[(home_shifts.name.isin(goalie_names))]) == 0 or len(away_shifts[(away_shifts.name.isin(goalie_names))]):
+        if len(home_shifts[(home_shifts.name.isin(goalie_names))]) == 0 or len(away_shifts[(away_shifts.name.isin(goalie_names))]) == 0:
         
             pbp_html_url = f'https://www.nhl.com/scores/htmlreports/{season}/GS0{game_id}.HTM'
             pbp_soup = BeautifulSoup(requests.get(pbp_html_url).content)
@@ -1050,26 +1047,27 @@ def scrape_html_shifts(season, game_id, live = True):
 
             away_teams = pd.read_html(str(goalie_table))[0][:2]
             away_team = away_teams[0].iloc[0]
-            
-            away_goalies = pd.read_html(str(goalie_table))[0][2:4]
+
+            away_goalies = pd.read_html(str(goalie_table))[0][2:4][
+                ~pd.isna(pd.read_html(str(goalie_table))[0][2:4])[0]
+            ]
             away_goalies = away_goalies[~pd.isna(away_goalies[6])]
-            
+
             away_goalies = away_goalies.assign(team = away_team).rename(columns = {0:'number', 2:'name', 6:'TOI'}).loc[:, ['number', 'name', 'TOI', 'team']]
 
-            home_teams = pd.read_html(str(goalie_table))[0][6:7]
+            away_goalies = away_goalies[away_goalies.TOI!='TOT']
+
+            home_teams = pd.read_html(str(goalie_table))[0][6:8][
+                ~pd.isna(pd.read_html(str(goalie_table))[0][6:8])[0]
+            ]
             home_team = home_teams[0].iloc[0]
-            
+
             home_goalies = pd.read_html(str(goalie_table))[0][8:10]
             home_goalies = home_goalies[~pd.isna(home_goalies[6])]
-            
+
             home_goalies = home_goalies.assign(team = home_team).rename(columns = {0:'number', 2:'name', 6:'TOI'}).loc[:, ['number', 'name', 'TOI', 'team']]
 
-            home_goalies = pd.read_html(str(goalie_table))[0][8:9]
-
-            # Temporary to test. Will fix later.
-            
-            # home_goalies = home_goalies.assign(TOI = '11:26')
-            # away_goalies = away_goalies.assign(TOI = '11:26')
+            home_goalies = home_goalies[home_goalies.TOI!='TOT']
 
         if len(home_shifts[(home_shifts.name.isin(goalie_names))]) == 0:
 
@@ -1108,8 +1106,6 @@ def scrape_html_shifts(season, game_id, live = True):
     global all_shifts
     
     all_shifts = pd.concat([home_shifts, away_shifts])
-
-    #all_shifts.to_csv('/Users/patrickbacon/compact_topdownhockey/all_shifts.csv', index = False)
     
     all_shifts = all_shifts.assign(start_time = all_shifts.shift_start.str.split('/').str[0])
     
@@ -1359,7 +1355,6 @@ def scrape_html_shifts(season, game_id, live = True):
     global changes_on
     global changes_off
     myshifts = all_shifts
-    #myshifts.to_csv('/Users/patrickbacon/compact_topdownhockey/tmp.csv', index = False)
     #print('Printing my shifts')
 
     #print(myshifts)
@@ -1416,7 +1411,8 @@ def scrape_html_events(season, game_id):
     #global stripped_html
     #global eventdf
     stripped_html = hs_strip_html(tds)
-    length = int(len(stripped_html)/8)
+    length = (len(stripped_html) // 8) * 8
+    stripped_html = stripped_html[:length]
     eventdf = pd.DataFrame(np.array(stripped_html).reshape(length, 8)).rename(
     columns = {0:'index', 1:'period', 2:'strength', 3:'time', 4:'event', 5:'description', 6:'away_skaters', 7:'home_skaters'})
     split = eventdf.time.str.split(':')
@@ -2080,8 +2076,6 @@ def merge_and_prepare(events, shifts):
                                           np.where(merged.event.isin(['PGSTR', 'PGEND', 'PSTR', 'PEND', 'ANTHEM']), -1, 1))).sort_values(
         by = ['game_seconds', 'period', 'event_index'])
 
-    merged.to_csv('/Users/patrickbacon/compact_topdownhockey/first_merged.csv', index = False)
-
     merged = merged.assign(change_before_event = np.where(
         (
             (merged.away_on_ice!='') & (merged.event.shift()=='CHANGE') & (merged.away_on_ice!=merged.away_on_ice.shift()) | 
@@ -2471,7 +2465,9 @@ def full_scrape_1by1(game_id_list, live = False, shift_to_espn = True):
                 if len(event_coords[(event_coords.event.isin(ewc)) & (pd.isna(event_coords.coords_x))]) > 0:
                     raise ExpatError('Bad takes, dude!')
                 event_coords['game_id'] = int(game_id)
+                print('Attempting to merge events again')
                 events = single.merge(event_coords, on = ['event_player_1', 'game_seconds', 'version', 'period', 'game_id', 'event'], how = 'left')
+                print('Merged events again, we have this many rows:', len(events))
                 try:
                     events = fix_missing(single, event_coords, events)
                 except IndexError as e:
@@ -2530,8 +2526,11 @@ def full_scrape_1by1(game_id_list, live = False, shift_to_espn = True):
                         print('Scraping ESPN Events')
                         print('Here is the ESPN ID:', espn_id)
                         event_coords = scrape_espn_events(int(espn_id))
+                        print('Scraped ESPN Events, we have this many rows:', len(event_coords))
                         event_coords['coordinate_source'] = 'espn'
+                        print('Attempting to merge events')
                         events = single.merge(event_coords, on = ['event_player_1', 'game_seconds', 'period', 'version', 'event'], how = 'left').drop(columns = ['espn_id'])
+                        print('Merged events, we have this many rows:', len(events))
                         try:
                             events = fix_missing(single, event_coords, events)
                         except IndexError as e:
@@ -2690,16 +2689,19 @@ def full_scrape_1by1(game_id_list, live = False, shift_to_espn = True):
             
         except AttributeError as e:
             print(str(game_id) + ' does not have an HTML report. Here is the error: ' + str(e))
+            print(traceback.format_exc())
             i = i + 1
             continue
             
         except IndexError as e:
             print(str(game_id) + ' has an issue with the HTML Report. Here is the error: ' + str(e))
+            print(traceback.format_exc())
             i = i + 1
             continue
             
         except ValueError as e:
             print(str(game_id) + ' has an issue with the HTML Report. Here is the error: ' + str(e))
+            print(traceback.format_exc())
             i = i + 1
             continue
 
@@ -2796,6 +2798,7 @@ def full_scrape(game_id_list, live = True, shift = False):
     hidden_patrick = 0
     
     df = full_scrape_1by1(game_id_list, live, shift_to_espn = shift)
+    print('Full scrape complete, we have this many rows:', len(df))
 
     # Fixing the Pettersson issue for event player. Just going downstream for this.
     try:

{topdownhockey_scraper-6.0.5.dist-info → topdownhockey_scraper-6.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: TopDownHockey_Scraper
-Version: 6.0.5
+Version: 6.0.9
 Summary: The TopDownHockey Scraper
 Home-page: https://github.com/TopDownHockey/TopDownHockey_Scraper
 Author: Patrick Bacon

topdownhockey_scraper-6.0.9.dist-info/RECORD

@@ -0,0 +1,7 @@
+TopDownHockey_Scraper/TopDownHockey_EliteProspects_Scraper.py,sha256=j-7gTk-cp_0LyZihNxm67xH9KdA3Fx4xrFKKu3-9-rU,42245
+TopDownHockey_Scraper/TopDownHockey_NHL_Scraper.py,sha256=rg_7RJo1eWL9dWlRweCat2v21fWX1Z45olcZN859BN4,163992
+topdownhockey_scraper-6.0.9.dist-info/licenses/LICENSE,sha256=2bm9uFabQZ3Ykb_SaSU_uUbAj2-htc6WJQmS_65qD00,1073
+topdownhockey_scraper-6.0.9.dist-info/METADATA,sha256=nM1FoBq-lslyopV84S10eukQQ4pPMSyFGA2S_5xfa8g,5670
+topdownhockey_scraper-6.0.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+topdownhockey_scraper-6.0.9.dist-info/top_level.txt,sha256=PBd96GLGFq97ZDLd7_4ZCx8_ZFr_wdWKs5SIpGl5xCs,22
+topdownhockey_scraper-6.0.9.dist-info/RECORD,,

topdownhockey_scraper-6.0.5.dist-info/RECORD

@@ -1,7 +0,0 @@
-TopDownHockey_Scraper/TopDownHockey_EliteProspects_Scraper.py,sha256=j-7gTk-cp_0LyZihNxm67xH9KdA3Fx4xrFKKu3-9-rU,42245
-TopDownHockey_Scraper/TopDownHockey_NHL_Scraper.py,sha256=kRZPB-pfRCDn6K2UK4ZHYlo09bDHxF7B34w8VE59GoI,163837
-topdownhockey_scraper-6.0.5.dist-info/licenses/LICENSE,sha256=2bm9uFabQZ3Ykb_SaSU_uUbAj2-htc6WJQmS_65qD00,1073
-topdownhockey_scraper-6.0.5.dist-info/METADATA,sha256=yvjnUIQ66Z80Oi02-mWvV2GdFMvvBk2O-lgoGOB4kx0,5670
-topdownhockey_scraper-6.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-topdownhockey_scraper-6.0.5.dist-info/top_level.txt,sha256=PBd96GLGFq97ZDLd7_4ZCx8_ZFr_wdWKs5SIpGl5xCs,22
-topdownhockey_scraper-6.0.5.dist-info/RECORD,,