scraping-rtn 0.0.6.0__tar.gz → 0.0.6.1__tar.gz

{scraping_rtn-0.0.6.0 → scraping_rtn-0.0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scraping_rtn
-Version: 0.0.6.0
+Version: 0.0.6.1
 Summary: package to scrape gymnastics data from Road To Nationals
 Author-email: Claire Harmon <ceharmon220@gmail.com>
 Project-URL: Homepage, https://github.com/cgn-charmon/scraping_rtn

{scraping_rtn-0.0.6.0 → scraping_rtn-0.0.6.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scraping_rtn"
-version = "0.0.6.0"
+version = "0.0.6.1"
 authors = [
     { name="Claire Harmon", email="ceharmon220@gmail.com" },
 ]

{scraping_rtn-0.0.6.0 → scraping_rtn-0.0.6.1}/src/scraping_rtn/RtnSingleTeamYear.py

@@ -1,4 +1,4 @@
-from .src import EVENT_MAP, EVENTS, session_setup, get_data_from_api, fix_opponents, normalize_date, merge_dicts, \
+from .src import EVENT_MAP, EVENTS, get_data_from_api, fix_opponents, normalize_date, merge_dicts, \
     get_extra_cols, SCHEDULE_COLS, RESULTS_COLS, IND_RESULTS_COLS, ROSTER_COLS
 import pandas as pd
 import numpy as np
@@ -15,14 +15,8 @@ class RtnSingleTeamYear(object):
             else:
                 self.team_id = team_id
 
-    def connect_session(self):
-        self.session = session_setup()
-
     def get_team_mapping(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
-        all_teams_data = get_data_from_api(session=self.session, endpoint='gymnasts2', suffix=str(self.year) + '/1').json()
+        all_teams_data = get_data_from_api(endpoint='gymnasts2', suffix=str(self.year) + '/1').json()
         return {team['team_name']: team['id'] for team in all_teams_data['teams']}
 
     def get_team_id(self):
@@ -39,13 +33,10 @@ class RtnSingleTeamYear(object):
         #     raise ValueError(f'{self.team_name} does not exist in data for {self.year}')
 
     def _get_raw_roster(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
         rename_map = {'id': 'Gymnast ID', 'hometown': 'Hometown', 'school_year': 'School Year', 'events': 'Events'}
         school_year_map = {'1': 'FR', '2': 'SO', '3': 'JR', '4': 'SR'}
 
-        roster_data = get_data_from_api(session=self.session, endpoint='rostermain', suffix=str(self.year)+'/'+str(self.team_id)+'/1').json()
+        roster_data = get_data_from_api(endpoint='rostermain', suffix=str(self.year)+'/'+str(self.team_id)+'/1').json()
 
         self._raw_roster = [{**{rename_map.get(k, k): v if k != 'school_year' else school_year_map.get(v, '') for k, v in data.items()},
                             **{'Name': data['fname'] + ' ' + data['lname'], 'Team': self.team_name}}
@@ -66,10 +57,7 @@ class RtnSingleTeamYear(object):
         return self.roster
 
     def _get_raw_season_results(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
-        meets = get_data_from_api(session=self.session, endpoint='dashboard', suffix=str(self.year)+'/'+str(self.team_id)).json()
+        meets = get_data_from_api(endpoint='dashboard', suffix=str(self.year)+'/'+str(self.team_id)).json()
         name_map = {'team_id': 'Team ID', 'team_name': 'Team', 'meet_id': 'Team Meet ID',
                     'meet_date': 'Meet Date', 'team_score': 'Score', 'home': 'Home/Away',
                     'opponent': 'Opponents', 'meet_desc': 'Meet Name', 'linked_id': 'Meet ID'}
@@ -122,13 +110,10 @@ class RtnSingleTeamYear(object):
         return self.season_results
 
     def _team_event_scores_by_meet(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
         team_scores_all = []
         for meet_id in [data['Team Meet ID'] for data in self._raw_season_results if data['Meet Date'] <= datetime.now()]:
             try:
-                meet_res = get_data_from_api(session=self.session, endpoint='meetresults', suffix=str(meet_id)).json()
+                meet_res = get_data_from_api(endpoint='meetresults', suffix=str(meet_id)).json()
                 # This API call returns scores from all teams at this meet, not just this team. Need to pick out correct score
                 team_scores = [score for score in meet_res['teams'] if score['tname'] == self.team_name and score['mid'] == str(meet_id)]
                 assert len(team_scores) == 1, 'Multiple team scores??'
@@ -144,10 +129,7 @@ class RtnSingleTeamYear(object):
                 self._raw_season_results[i].update({'VT': np.nan, 'UB': np.nan, 'BB': np.nan, 'FX': np.nan})
 
     def _team_event_scores_team_consistency(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
-        res = get_data_from_api(session=self.session, endpoint='teamConsistency', suffix=f'{self.year}/{self.team_id}').json()
+        res = get_data_from_api(endpoint='teamConsistency', suffix=f'{self.year}/{self.team_id}').json()
         if len(res['labels']) == 0:
             print(f'No team consistency data found for {self.team_name} in {self.year}')
             for i in range(len(self._raw_season_results)):
@@ -191,13 +173,10 @@ class RtnSingleTeamYear(object):
         return self.individual_results
 
     def _individual_scores_by_meet(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
         individual_scores_all = []
         for meet_id in [meet['Team Meet ID'] for meet in self._raw_schedule if meet['Meet Date'] <= datetime.now()]:
             try:
-                meet_res = get_data_from_api(session=self.session, endpoint='meetresults', suffix=str(meet_id)).json()
+                meet_res = get_data_from_api(endpoint='meetresults', suffix=str(meet_id)).json()
                 if len(meet_res) == 0 or len(meet_res['scores']) == 0 or len(meet_res['scores'][0]) == 0:
                     print(f'No data found for meet {meet_id}')
                     continue
@@ -233,13 +212,10 @@ class RtnSingleTeamYear(object):
                                                     'Meet Name', 'Meet ID'])
 
     def _individual_scores_individual_consistency(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
         ind_consistency_all = []
         for gymnast in self._raw_roster:
             try:
-                res = get_data_from_api(session=self.session, endpoint='indConsistency', suffix=f"{self.year}/{gymnast['Gymnast ID']}").json()
+                res = get_data_from_api(endpoint='indConsistency', suffix=f"{self.year}/{gymnast['Gymnast ID']}").json()
                 ind_consistency = [{'Meet Date': normalize_date(res['labels'][i][:7] + str(self.year), dt_format='%b-%d-%Y'),
                                     'VT': round(float(res['vts'][i]), 4) if res['vts'][i] is not None else np.nan,
                                     'UB': round(float(res['ubs'][i]), 4) if res['ubs'][i] is not None else np.nan,
@@ -273,14 +249,11 @@ class RtnSingleTeamYear(object):
             return pd.DataFrame(columns=ROSTER_COLS + EVENTS) # + ['AA'])
 
     def _get_raw_individual_nqs(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
         name_map = {'maxv': 'VT', 'maxub': 'UB', 'maxbb': 'BB', 'maxfx': 'FX',
                     # 'maxaa': 'AA',
                     'gid': 'Gymnast ID'}
 
-        nqsData = get_data_from_api(session=self.session, endpoint='rostermain', suffix=f'{self.year}/{self.team_id}/4').json()
+        nqsData = get_data_from_api(endpoint='rostermain', suffix=f'{self.year}/{self.team_id}/4').json()
         ind_nqs = [{name_map[k]: round(float(v), 4) if k != 'gid' and v != '' else (np.nan if k != 'gid' else v)
                     for k, v in data.items() if k in name_map.keys()} for data in nqsData['ind']]
 
@@ -292,24 +265,18 @@ class RtnSingleTeamYear(object):
             self._raw_individual_nqs = []
 
     def _get_current_week(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
         if not hasattr(self, 'week'):
-            week_data = get_data_from_api(session=self.session, endpoint='currentweek', suffix=str(self.year)).json()
+            week_data = get_data_from_api(endpoint='currentweek', suffix=str(self.year)).json()
             return min(int(week_data['week']), int(week_data['max']))
 
     def _get_raw_rankings(self, team_vs_ind, event, week):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
         team_ind_map = {'team': 0, 'ind': 1}
         event_api_map = {'VT': 1, 'UB': 2, 'BB': 3, 'FX': 4, 'AA': 5}
         rename_map = {'rank': 'Rank', 'gid': 'Gymnast ID', 'team': 'Team', 'tid': 'Team ID',
                       'rqs': 'NQS', 'reg': 'Region', 'con': 'Conference', 'div': 'Division',
                       'usag': 'USAG', 'ave': 'Average', 'high': 'High', 'name': 'Team'}
 
-        res = get_data_from_api(session=self.session, endpoint='results', suffix=f'{self.year}/{week}/{team_ind_map[team_vs_ind]}/{event_api_map[event]}').json()
+        res = get_data_from_api(endpoint='results', suffix=f'{self.year}/{week}/{team_ind_map[team_vs_ind]}/{event_api_map[event]}').json()
         if team_vs_ind == 'ind':
             self._raw_rankings[team_vs_ind][event] = [{**{rename_map.get(k): float(v) if k in ['rqs', 'ave', 'high'] else v for k, v in data.items() if k in rename_map},
                                                        **{'Name': data['fname'] + ' ' + data['lname'], 'Event': event}}

{scraping_rtn-0.0.6.0 → scraping_rtn-0.0.6.1}/src/scraping_rtn/src.py

@@ -21,21 +21,18 @@ def validate_input(teams):
     return teams
 
 
-def session_setup():
+@lru_cache(maxsize=1000000)
+def get_data_from_api(endpoint, suffix):
     session = requests.Session()
     retry = Retry(connect=3, backoff_factor=0.5)
     adapter = HTTPAdapter(max_retries=retry)
     session.mount('http://', adapter)
     session.mount('https://', adapter)
 
-    return session
-
-
-@lru_cache(maxsize=1000000)
-def get_data_from_api(session, endpoint, suffix):
     url = 'https://www.roadtonationals.com/api/women/' + endpoint
     if suffix:
         url += '/' + suffix
+
     return session.get(url)
 
 

{scraping_rtn-0.0.6.0 → scraping_rtn-0.0.6.1}/src/scraping_rtn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scraping_rtn
-Version: 0.0.6.0
+Version: 0.0.6.1
 Summary: package to scrape gymnastics data from Road To Nationals
 Author-email: Claire Harmon <ceharmon220@gmail.com>
 Project-URL: Homepage, https://github.com/cgn-charmon/scraping_rtn