scraping-rtn 0.0.5__tar.gz → 0.0.6.0__tar.gz

{scraping_rtn-0.0.5 → scraping_rtn-0.0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scraping_rtn
-Version: 0.0.5
+Version: 0.0.6.0
 Summary: package to scrape gymnastics data from Road To Nationals
 Author-email: Claire Harmon <ceharmon220@gmail.com>
 Project-URL: Homepage, https://github.com/cgn-charmon/scraping_rtn

{scraping_rtn-0.0.5 → scraping_rtn-0.0.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scraping_rtn"
-version = "0.0.5"
+version = "0.0.6.0"
 authors = [
     { name="Claire Harmon", email="ceharmon220@gmail.com" },
 ]

{scraping_rtn-0.0.5 → scraping_rtn-0.0.6.0}/src/scraping_rtn/RtnSingleTeamYear.py

@@ -1,5 +1,5 @@
-from .src import EVENT_MAP, EVENTS, get_data_from_api, fix_opponents, normalize_date, merge_dicts, get_extra_cols, \
-    SCHEDULE_COLS, RESULTS_COLS, IND_RESULTS_COLS, ROSTER_COLS
+from .src import EVENT_MAP, EVENTS, session_setup, get_data_from_api, fix_opponents, normalize_date, merge_dicts, \
+    get_extra_cols, SCHEDULE_COLS, RESULTS_COLS, IND_RESULTS_COLS, ROSTER_COLS
 import pandas as pd
 import numpy as np
 from datetime import datetime
@@ -15,8 +15,14 @@ class RtnSingleTeamYear(object):
             else:
                 self.team_id = team_id
 
+    def connect_session(self):
+        self.session = session_setup()
+
     def get_team_mapping(self):
-        all_teams_data = get_data_from_api(endpoint='gymnasts2', suffix=str(self.year) + '/1').json()
+        if not hasattr(self, 'session'):
+            self.connect_session()
+
+        all_teams_data = get_data_from_api(session=self.session, endpoint='gymnasts2', suffix=str(self.year) + '/1').json()
         return {team['team_name']: team['id'] for team in all_teams_data['teams']}
 
     def get_team_id(self):
@@ -33,10 +39,13 @@ class RtnSingleTeamYear(object):
         #     raise ValueError(f'{self.team_name} does not exist in data for {self.year}')
 
     def _get_raw_roster(self):
+        if not hasattr(self, 'session'):
+            self.connect_session()
+
         rename_map = {'id': 'Gymnast ID', 'hometown': 'Hometown', 'school_year': 'School Year', 'events': 'Events'}
         school_year_map = {'1': 'FR', '2': 'SO', '3': 'JR', '4': 'SR'}
 
-        roster_data = get_data_from_api(endpoint='rostermain', suffix=str(self.year)+'/'+str(self.team_id)+'/1').json()
+        roster_data = get_data_from_api(session=self.session, endpoint='rostermain', suffix=str(self.year)+'/'+str(self.team_id)+'/1').json()
 
         self._raw_roster = [{**{rename_map.get(k, k): v if k != 'school_year' else school_year_map.get(v, '') for k, v in data.items()},
                             **{'Name': data['fname'] + ' ' + data['lname'], 'Team': self.team_name}}
@@ -57,7 +66,10 @@ class RtnSingleTeamYear(object):
         return self.roster
 
     def _get_raw_season_results(self):
-        meets = get_data_from_api(endpoint='dashboard', suffix=str(self.year)+'/'+str(self.team_id)).json()
+        if not hasattr(self, 'session'):
+            self.connect_session()
+
+        meets = get_data_from_api(session=self.session, endpoint='dashboard', suffix=str(self.year)+'/'+str(self.team_id)).json()
         name_map = {'team_id': 'Team ID', 'team_name': 'Team', 'meet_id': 'Team Meet ID',
                     'meet_date': 'Meet Date', 'team_score': 'Score', 'home': 'Home/Away',
                     'opponent': 'Opponents', 'meet_desc': 'Meet Name', 'linked_id': 'Meet ID'}
@@ -110,10 +122,13 @@ class RtnSingleTeamYear(object):
         return self.season_results
 
     def _team_event_scores_by_meet(self):
+        if not hasattr(self, 'session'):
+            self.connect_session()
+
         team_scores_all = []
         for meet_id in [data['Team Meet ID'] for data in self._raw_season_results if data['Meet Date'] <= datetime.now()]:
             try:
-                meet_res = get_data_from_api(endpoint='meetresults', suffix=str(meet_id)).json()
+                meet_res = get_data_from_api(session=self.session, endpoint='meetresults', suffix=str(meet_id)).json()
                 # This API call returns scores from all teams at this meet, not just this team. Need to pick out correct score
                 team_scores = [score for score in meet_res['teams'] if score['tname'] == self.team_name and score['mid'] == str(meet_id)]
                 assert len(team_scores) == 1, 'Multiple team scores??'
@@ -129,7 +144,10 @@ class RtnSingleTeamYear(object):
                 self._raw_season_results[i].update({'VT': np.nan, 'UB': np.nan, 'BB': np.nan, 'FX': np.nan})
 
     def _team_event_scores_team_consistency(self):
-        res = get_data_from_api(endpoint='teamConsistency', suffix=f'{self.year}/{self.team_id}').json()
+        if not hasattr(self, 'session'):
+            self.connect_session()
+
+        res = get_data_from_api(session=self.session, endpoint='teamConsistency', suffix=f'{self.year}/{self.team_id}').json()
         if len(res['labels']) == 0:
             print(f'No team consistency data found for {self.team_name} in {self.year}')
             for i in range(len(self._raw_season_results)):
@@ -173,10 +191,13 @@ class RtnSingleTeamYear(object):
         return self.individual_results
 
     def _individual_scores_by_meet(self):
+        if not hasattr(self, 'session'):
+            self.connect_session()
+
         individual_scores_all = []
         for meet_id in [meet['Team Meet ID'] for meet in self._raw_schedule if meet['Meet Date'] <= datetime.now()]:
             try:
-                meet_res = get_data_from_api(endpoint='meetresults', suffix=str(meet_id)).json()
+                meet_res = get_data_from_api(session=self.session, endpoint='meetresults', suffix=str(meet_id)).json()
                 if len(meet_res) == 0 or len(meet_res['scores']) == 0 or len(meet_res['scores'][0]) == 0:
                     print(f'No data found for meet {meet_id}')
                     continue
@@ -212,10 +233,13 @@ class RtnSingleTeamYear(object):
                                                     'Meet Name', 'Meet ID'])
 
     def _individual_scores_individual_consistency(self):
+        if not hasattr(self, 'session'):
+            self.connect_session()
+
         ind_consistency_all = []
         for gymnast in self._raw_roster:
             try:
-                res = get_data_from_api(endpoint='indConsistency', suffix=f"{self.year}/{gymnast['Gymnast ID']}").json()
+                res = get_data_from_api(session=self.session, endpoint='indConsistency', suffix=f"{self.year}/{gymnast['Gymnast ID']}").json()
                 ind_consistency = [{'Meet Date': normalize_date(res['labels'][i][:7] + str(self.year), dt_format='%b-%d-%Y'),
                                     'VT': round(float(res['vts'][i]), 4) if res['vts'][i] is not None else np.nan,
                                     'UB': round(float(res['ubs'][i]), 4) if res['ubs'][i] is not None else np.nan,
@@ -249,11 +273,14 @@ class RtnSingleTeamYear(object):
             return pd.DataFrame(columns=ROSTER_COLS + EVENTS) # + ['AA'])
 
     def _get_raw_individual_nqs(self):
+        if not hasattr(self, 'session'):
+            self.connect_session()
+
         name_map = {'maxv': 'VT', 'maxub': 'UB', 'maxbb': 'BB', 'maxfx': 'FX',
                     # 'maxaa': 'AA',
                     'gid': 'Gymnast ID'}
 
-        nqsData = get_data_from_api(endpoint='rostermain', suffix=f'{self.year}/{self.team_id}/4').json()
+        nqsData = get_data_from_api(session=self.session, endpoint='rostermain', suffix=f'{self.year}/{self.team_id}/4').json()
         ind_nqs = [{name_map[k]: round(float(v), 4) if k != 'gid' and v != '' else (np.nan if k != 'gid' else v)
                     for k, v in data.items() if k in name_map.keys()} for data in nqsData['ind']]
 
@@ -265,17 +292,24 @@ class RtnSingleTeamYear(object):
             self._raw_individual_nqs = []
 
     def _get_current_week(self):
+        if not hasattr(self, 'session'):
+            self.connect_session()
+
         if not hasattr(self, 'week'):
-            return get_data_from_api(endpoint='currentweek', suffix=str(self.year)).json()['max']
+            week_data = get_data_from_api(session=self.session, endpoint='currentweek', suffix=str(self.year)).json()
+            return min(int(week_data['week']), int(week_data['max']))
 
     def _get_raw_rankings(self, team_vs_ind, event, week):
+        if not hasattr(self, 'session'):
+            self.connect_session()
+
         team_ind_map = {'team': 0, 'ind': 1}
         event_api_map = {'VT': 1, 'UB': 2, 'BB': 3, 'FX': 4, 'AA': 5}
         rename_map = {'rank': 'Rank', 'gid': 'Gymnast ID', 'team': 'Team', 'tid': 'Team ID',
                       'rqs': 'NQS', 'reg': 'Region', 'con': 'Conference', 'div': 'Division',
                       'usag': 'USAG', 'ave': 'Average', 'high': 'High', 'name': 'Team'}
 
-        res = get_data_from_api(endpoint='results', suffix=f'{self.year}/{week}/{team_ind_map[team_vs_ind]}/{event_api_map[event]}').json()
+        res = get_data_from_api(session=self.session, endpoint='results', suffix=f'{self.year}/{week}/{team_ind_map[team_vs_ind]}/{event_api_map[event]}').json()
         if team_vs_ind == 'ind':
             self._raw_rankings[team_vs_ind][event] = [{**{rename_map.get(k): float(v) if k in ['rqs', 'ave', 'high'] else v for k, v in data.items() if k in rename_map},
                                                        **{'Name': data['fname'] + ' ' + data['lname'], 'Event': event}}

{scraping_rtn-0.0.5 → scraping_rtn-0.0.6.0}/src/scraping_rtn/src.py

@@ -1,4 +1,6 @@
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 from functools import lru_cache
 from datetime import datetime
 
@@ -19,12 +21,22 @@ def validate_input(teams):
     return teams
 
 
+def session_setup():
+    session = requests.Session()
+    retry = Retry(connect=3, backoff_factor=0.5)
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+
+    return session
+
+
 @lru_cache(maxsize=1000000)
-def get_data_from_api(endpoint, suffix):
+def get_data_from_api(session, endpoint, suffix):
     url = 'https://www.roadtonationals.com/api/women/' + endpoint
     if suffix:
         url += '/' + suffix
-    return requests.get(url)
+    return session.get(url)
 
 
 def fix_opponents(ops):

{scraping_rtn-0.0.5 → scraping_rtn-0.0.6.0}/src/scraping_rtn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scraping_rtn
-Version: 0.0.5
+Version: 0.0.6.0
 Summary: package to scrape gymnastics data from Road To Nationals
 Author-email: Claire Harmon <ceharmon220@gmail.com>
 Project-URL: Homepage, https://github.com/cgn-charmon/scraping_rtn