TopDownHockey-Scraper 3.2.7__py3-none-any.whl → 4.0.0__py3-none-any.whl

TopDownHockey_Scraper/TopDownHockey_EliteProspects_Scraper.py

@@ -28,10 +28,10 @@ def tableDataText(table):
 
     headerow = [td.get_text(strip=True) for td in trs[0].find_all('th')] # header row
     if headerow: # if there is a header row include first
-        rows._append(headerow)
+        rows.append(headerow)
         trs = trs[1:]
     for tr in trs: # for every table row
-        rows._append([td.get_text(strip=True) for td in tr.find_all('td')]) # data row
+        rows.append([td.get_text(strip=True) for td in tr.find_all('td')]) # data row
         
     df_rows = pd.DataFrame(rows[1:], columns=rows[0])
     
@@ -97,7 +97,7 @@ def getskaters(league, year):
                     # Extract href links in table
                     href_row = []
                     for link in player_table.find_all('a'):
-                        href_row._append(link.attrs['href'])
+                        href_row.append(link.attrs['href'])
 
                     # Create data frame, rename and only keep links to players
                     df_links = pd.DataFrame(href_row)  
@@ -107,7 +107,7 @@ def getskaters(league, year):
                     # Add links to players
                     df_players['link']=df_links['link'] 
 
-                    players._append(df_players)
+                    players.append(df_players)
 
                     # Wait 3 seconds before going to next
                     #time.sleep(1)
@@ -209,7 +209,7 @@ def getgoalies(league, year):
                     # Extract href links in table
                     href_row = []
                     for link in player_table.find_all('a'):
-                        href_row._append(link.attrs['href'])
+                        href_row.append(link.attrs['href'])
 
                     # Create data frame, rename and only keep links to players
                     df_links = pd.DataFrame(href_row)  
@@ -219,7 +219,7 @@ def getgoalies(league, year):
                     # Add links to players
                     df_players['link']=df_links['link'] 
 
-                    players._append(df_players)
+                    players.append(df_players)
 
                     # Wait 3 seconds before going to next
                     # time.sleep(1)
@@ -252,7 +252,7 @@ def get_info(link):
     """
     A function that is built strictly for the back end and should not be run by the user.
     """
-    
+
     page = requests.get(link, timeout = 500)
     soup = BeautifulSoup(page.content, "html.parser")
 
@@ -265,99 +265,80 @@ def get_info(link):
         soup = BeautifulSoup(page.content, "html.parser")
         time.sleep(60)
 
-    if soup.find("title") != None:
-        player = soup.find("title").string.replace(" - Elite Prospects" ,"")
+    lis = soup.find_all('li')
 
-    else: player = "-"
-        
-    if soup.find("div", {"class":"order-11 ep-list__item ep-list__item--in-card-body ep-list__item--is-compact"})!=None:
-        rights = soup.find("div", {"class":"order-11 ep-list__item ep-list__item--in-card-body ep-list__item--is-compact"}
-         ).find("div", {"class":"col-xs-12 col-18 text-right p-0"}).find("span").string.split("\n")[1].split("/")[0].strip()
-        status = soup.find("div", {"class":"order-11 ep-list__item ep-list__item--in-card-body ep-list__item--is-compact"}
-         ).find("div", {"class":"col-xs-12 col-18 text-right p-0"}).find("span").string.split("\n")[1].split("/")[1].strip()
-    else:
-        rights = "-"
-        status = "-"
-                 
-    if (soup.find("div", {"class":"col-xs-12 col-17 text-right p-0 ep-text-color--black"}))!= None:
-        if 'dob' in (soup.find("div", {"class":"col-xs-12 col-17 text-right p-0 ep-text-color--black"})).find("a")['href']:
-            dob = soup.find("div", {"class":"col-xs-12 col-17 text-right p-0 ep-text-color--black"}).find("a")['href'].split("dob=", 1)[1].split("&sort", 1)[0]
-        else: 
-            dob = "-"
+    relevant_lis = [li for li in lis if li.find('span') is not None]
 
+    # player
+    
+    if soup.find("title") != None:
+        player = soup.find("title").string.replace(' - Stats, Contract, Salary & More', '')
     else:
-        dob = "-"
-
-    if soup.find("div", {"class":"order-6 order-sm-3 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}) != None:
-        if "cm" in soup.find("div", {"class":"order-6 order-sm-3 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}
-                            ).find(
-        "div", {"class":"col-xs-12 col-18 text-right p-0 ep-text-color--black"}).string:
-            height = soup.find("div", {"class":"order-6 order-sm-3 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}
-                              ).find(
-            "div", {"class":"col-xs-12 col-18 text-right p-0 ep-text-color--black"}).string.split(" / ")[1].split("cm")[0].strip()
-        else: 
-            height = "-"
-
+        player = '-'
+    
+    # status
+    
+    if [li for li in relevant_lis if li.find('span').text=='NHL Rights'] != []:
+        rights = [li for li in relevant_lis if li.find('span').text=='NHL Rights'][0].find('a').text.split(' /')[0]
+    else:
+        rights = '-'
+    
+    # rights
+    
+    if [li for li in relevant_lis if li.find('span').text=='NHL Rights'] != []:
+        status = [li for li in relevant_lis if li.find('span').text=='NHL Rights'][0].find('a').text.split(' / ')[1]
+    else:
+        status = '-'
+    
+    # dob
+    
+    if [li.find('a')['href'].split('dob=', 1)[1].split('&sort', 1)[0] for li in relevant_lis if li.span.text == 'Date of Birth'] != []:
+        dob = [li.find('a')['href'].split('dob=', 1)[1].split('&sort', 1)[0] for li in relevant_lis if li.span.text == 'Date of Birth'][0]
+    else:
+        dob = '-'
+    
+    # height
+    
+    if [li for li in relevant_lis if li.find('span').text=='Height'] != []:
+        height = [li for li in relevant_lis if li.find('span').text=='Height'][0].text.split('Height')[1].split(' cm')[0]
     else: 
-        height = "-"
-
-    if soup.find("div", {"class":"order-7 order-sm-5 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}) != None:
-        if soup.find("div", {"class":"order-7 order-sm-5 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}
-                    ).find(
-            "div", {"class":"col-xs-12 col-18 text-right p-0 ep-text-color--black"}).string.split("\n")[1].split("lbs")[0].strip() == '- / -':
-                weight = "-"
-        else: 
-            weight = soup.find("div", {"class":"order-7 order-sm-5 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}
-                              ).find(
-            "div", {"class":"col-xs-12 col-18 text-right p-0 ep-text-color--black"}).string.split("\n")[1].split("lbs")[0].strip()
-
-    else: weight = "-"
-
-    if soup.find("div", {"class":"order-2 order-sm-4 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}
-                         ) != None:
-        if soup.find("div", {"class":"order-2 order-sm-4 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}
-                         ).find(
-            "div", {"class":"col-xs-12 col-17 text-right p-0 ep-text-color--black"}).find("a") != None:
-
-            birthplace = soup.find("div", {"class":"order-2 order-sm-4 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}
-                         ).find(
-                        "div", {"class":"col-xs-12 col-17 text-right p-0 ep-text-color--black"}).find("a").string.replace("\n", "").strip()
-
-        else: 
-            birthplace = "-"
+        height = '-'
+    
+    # weight
+    
+    if [li for li in relevant_lis if li.find('span').text=='Weight'] != []:
+        weight = [li for li in relevant_lis if li.find('span').text=='Weight'][0].text.split('Weight')[1].split(' cm')[0]
     else: 
-        birthplace = "-"
-
-    if soup.find("div", {"class":"order-3 order-sm-6 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}) != None:
-        if soup.find("div", {"class":"order-3 order-sm-6 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}
-                    ).find(
-            "div", {"class":"col-xs-12 col-18 text-right p-0 ep-text-color--black"}).find("a") != None:
-                nation = soup.find("div", {"class":"order-3 order-sm-6 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}
-                                  ).find(
-                "div", {"class":"col-xs-12 col-18 text-right p-0 ep-text-color--black"}).find("a").string.replace("\n", "").strip()
-        else: nation = "-"
-
+        weight = '-'
+    
+    # birthplace
+    
+    if [li for li in relevant_lis if li.find('span').text=='Place of Birth'] != []:
+        birthplace = [li for li in relevant_lis if li.find('span').text=='Place of Birth'][0].text.split('Birth')[1]
     else:
-        nation = "-"
-
-    if soup.find("div", {"class":"order-8 order-sm-7 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}) !=None:
-        shoots = soup.find("div", {"class":"order-8 order-sm-7 ep-list__item ep-list__item--col-2 ep-list__item--in-card-body ep-list__item--is-compact"}
-                          ).find(
-            "div", {"class":"col-xs-12 col-18 text-right p-0 ep-text-color--black"}).string.replace("\n", "").strip()
-
+        birthplace = '-'
+    
+    # nation
+    
+    if [li for li in relevant_lis if li.find('span').text=='Nation'] != []:
+        nation = [li for li in relevant_lis if li.find('span').text=='Nation'][0].text.split('Nation')[1]
     else:
-        shoots = "-"
-
-    if soup.find("div", {"class":"order-12 ep-list__item ep-list__item--in-card-body ep-list__item--is-compact"}) != None:
-        draft = soup.find("div", {"class":"order-12 ep-list__item ep-list__item--in-card-body ep-list__item--is-compact"}
-                         ).find(
-            "div", {"class":"col-xs-12 col-18 text-right p-0"}).find("a").string.replace("\n", "").strip()
-    else: 
-        draft = "-"
-
-    #height = np.where(height=="- / -", "-", height)
-
-    #print(player + " scraped!")
+        nation = '-'
+    
+    # shoots
+    
+    if [li for li in relevant_lis if li.find('span').text=='Shoots'] != []:
+        shoots = [li for li in relevant_lis if li.find('span').text=='Shoots'][0].text.split('Shoots')[1]
+    else:
+        shoots = '-'
+    
+    # draft
+    
+    if [li for li in relevant_lis if li.find('span').text=='Drafted'] != []:
+        draft = [li for li in relevant_lis if li.find('span').text=='Drafted'][0].text.split('Drafted')[1]
+    else:
+        draft = '-'
+    
     return(player, rights, status, dob, height, weight, birthplace, nation, shoots, draft, link)  
     
 def get_player_information(dataframe):
@@ -382,17 +363,17 @@ def get_player_information(dataframe):
     for i in range(0, len(list(set(dataframe.link)))):
         try:
             myresult = get_info(((list(set(dataframe.link))[i])))
-            myplayer._append(myresult[0])
-            myrights._append(myresult[1])
-            mystatus._append(myresult[2])
-            mydob._append(myresult[3])
-            myheight._append(myresult[4])
-            myweight._append(myresult[5])
-            mybirthplace._append(myresult[6])
-            mynation._append(myresult[7])
-            myshot._append(myresult[8])
-            mydraft._append(myresult[9])
-            mylink._append(myresult[10])
+            myplayer.append(myresult[0])
+            myrights.append(myresult[1])
+            mystatus.append(myresult[2])
+            mydob.append(myresult[3])
+            myheight.append(myresult[4])
+            myweight.append(myresult[5])
+            mybirthplace.append(myresult[6])
+            mynation.append(myresult[7])
+            myshot.append(myresult[8])
+            mydraft.append(myresult[9])
+            mylink.append(myresult[10])
             print(myresult[0] + " scraped! That's " + str(i + 1) + " down! Only " + str(len(list(set(dataframe.link))) - (i + 1)) +  " left to go!")
         except KeyboardInterrupt:
             print("You interrupted this one manually. The output here will be every player you've scraped so far. Good bye!")

TopDownHockey_Scraper/TopDownHockey_NHL_Scraper.py

@@ -1912,6 +1912,8 @@ def scrape_espn_ids_single_game(game_date, home_team, away_team):
     game_date = pd.to_datetime(this_date))
     
     gamedays = gamedays._append(fax)
+
+    gamedays = gamedays[gamedays.espn_id!='gameId']
     
     gamedays = gamedays.assign(
         home_team = np.where(gamedays.home_team=='ST LOUIS BLUES', 'ST. LOUIS BLUES', gamedays.home_team),
@@ -1977,8 +1979,9 @@ def scrape_espn_ids_single_game(game_date, home_team, away_team):
                     np.where(gamedays.away_team=='GOLDEN', 'VGK',
                     np.where(gamedays.away_team=='KNIGHTS', 'VGK',
                     np.where(gamedays.away_team=='CAPITALS', 'WSH',
-                    np.where(gamedays.away_team=='JETS', 'WPG', 'mistake'
-                            ))))))))))))))))))))))))))))))))))))
+                    np.where(gamedays.away_team=='JETS', 'WPG',
+                    np.where(gamedays.away_team=='CLUB', 'UTA', 'mistake'
+                            )))))))))))))))))))))))))))))))))))))
     )
 
     gamedays = gamedays.assign(
@@ -2017,8 +2020,9 @@ def scrape_espn_ids_single_game(game_date, home_team, away_team):
                     np.where(gamedays.home_team=='GOLDEN', 'VGK',
                     np.where(gamedays.home_team=='KNIGHTS', 'VGK',
                     np.where(gamedays.home_team=='CAPITALS', 'WSH',
-                    np.where(gamedays.home_team=='JETS', 'WPG', 'mistake'
-                            ))))))))))))))))))))))))))))))))))))
+                    np.where(gamedays.home_team=='JETS', 'WPG', 
+                    np.where(gamedays.home_team=='CLUB', 'UTA', 'mistake'
+                            )))))))))))))))))))))))))))))))))))))
     )
     
     gamedays = gamedays[(gamedays.game_date==this_date) & (gamedays.home_team==home_team) & (gamedays.away_team==away_team)] 
@@ -2434,7 +2438,7 @@ def fix_missing(single, event_coords, events):
     
     return(events)
 
-def full_scrape_1by1(game_id_list, shift_to_espn = False):
+def full_scrape_1by1(game_id_list, shift_to_espn = True):
     
     global single
     global event_coords
@@ -2446,12 +2450,13 @@ def full_scrape_1by1(game_id_list, shift_to_espn = False):
     
     i = 0
     
-    while i in range(0, len(game_id_list)):
+    while i in range(0, len(game_id_list)) and len(game_id_list)>0:
        
         # First thing to try: Scraping HTML events
         
         try:
             first_time = time.time()
+            print(game_id_list[i]) 
             game_id = game_id_list[i]
             print('Attempting scrape for: ' + str(game_id))
             season = str(int(str(game_id)[:4])) + str(int(str(game_id)[:4]) + 1)
@@ -2678,6 +2683,11 @@ def full_scrape_1by1(game_id_list, shift_to_espn = False):
             print(str(game_id) + ' has an issue with the HTML Report. Here is the error: ' + str(e))
             i = i + 1
             continue
+
+        except KeyError as k:
+            print(str(game_id) + 'gave some kind of Key Error. Here is the error: ' + str(e))
+            i = i + 1
+            continue
             
         except KeyboardInterrupt:
             print('You manually interrupted the scrape. You will get to keep every game you have already completed scraping after just a bit of post-processing. Good bye.')

{TopDownHockey_Scraper-3.2.7.dist-info → TopDownHockey_Scraper-4.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: TopDownHockey_Scraper
-Version: 3.2.7
+Version: 4.0.0
 Summary: The TopDownHockey Scraper
 Home-page: https://github.com/TopDownHockey/TopDownHockey_Scraper
 Author: Patrick Bacon

TopDownHockey_Scraper-4.0.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+TopDownHockey_Scraper/TopDownHockey_EliteProspects_Scraper.py,sha256=j-7gTk-cp_0LyZihNxm67xH9KdA3Fx4xrFKKu3-9-rU,42245
+TopDownHockey_Scraper/TopDownHockey_NHL_Scraper.py,sha256=jPYnjZMTN6tvQgAvQo9mmFSVQmc4-fEra1jLeuFRkpA,153624
+TopDownHockey_Scraper-4.0.0.dist-info/LICENSE,sha256=2bm9uFabQZ3Ykb_SaSU_uUbAj2-htc6WJQmS_65qD00,1073
+TopDownHockey_Scraper-4.0.0.dist-info/METADATA,sha256=5CD1aQY7EMR8wuQcRmItqnY9Uk9Ews8P0Sx_b3sJFUI,5462
+TopDownHockey_Scraper-4.0.0.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+TopDownHockey_Scraper-4.0.0.dist-info/top_level.txt,sha256=PBd96GLGFq97ZDLd7_4ZCx8_ZFr_wdWKs5SIpGl5xCs,22
+TopDownHockey_Scraper-4.0.0.dist-info/RECORD,,

{TopDownHockey_Scraper-3.2.7.dist-info → TopDownHockey_Scraper-4.0.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

TopDownHockey_Scraper-3.2.7.dist-info/RECORD

@@ -1,7 +0,0 @@
-TopDownHockey_Scraper/TopDownHockey_EliteProspects_Scraper.py,sha256=-EPVHQc06W8OcpVoTQvpUH40sjLj9Nwsv1-y3ANrOiQ,45380
-TopDownHockey_Scraper/TopDownHockey_NHL_Scraper.py,sha256=vumUGPWrtHOTWltSwKZCJNgKzum9UKr_xh7xX0E9_Fo,153213
-TopDownHockey_Scraper-3.2.7.dist-info/LICENSE,sha256=2bm9uFabQZ3Ykb_SaSU_uUbAj2-htc6WJQmS_65qD00,1073
-TopDownHockey_Scraper-3.2.7.dist-info/METADATA,sha256=MhGH9lavzF59_ILyFKpgWil9x8IIGTw4zi_bJ1PB0DU,5462
-TopDownHockey_Scraper-3.2.7.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-TopDownHockey_Scraper-3.2.7.dist-info/top_level.txt,sha256=PBd96GLGFq97ZDLd7_4ZCx8_ZFr_wdWKs5SIpGl5xCs,22
-TopDownHockey_Scraper-3.2.7.dist-info/RECORD,,