scraping-rtn 0.0.6.0__tar.gz → 0.0.7__tar.gz

{scraping_rtn-0.0.6.0 → scraping_rtn-0.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: scraping_rtn
-Version: 0.0.6.0
+Version: 0.0.7
 Summary: package to scrape gymnastics data from Road To Nationals
 Author-email: Claire Harmon <ceharmon220@gmail.com>
 Project-URL: Homepage, https://github.com/cgn-charmon/scraping_rtn

{scraping_rtn-0.0.6.0 → scraping_rtn-0.0.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scraping_rtn"
-version = "0.0.6.0"
+version = "0.0.7"
 authors = [
     { name="Claire Harmon", email="ceharmon220@gmail.com" },
 ]

{scraping_rtn-0.0.6.0 → scraping_rtn-0.0.7}/src/scraping_rtn/RtnSingleTeamYear.py

@@ -1,4 +1,4 @@
-from .src import EVENT_MAP, EVENTS, session_setup, get_data_from_api, fix_opponents, normalize_date, merge_dicts, \
+from .src import EVENT_MAP, EVENTS, get_data_from_api, fix_opponents, normalize_date, merge_dicts, \
     get_extra_cols, SCHEDULE_COLS, RESULTS_COLS, IND_RESULTS_COLS, ROSTER_COLS
 import pandas as pd
 import numpy as np
@@ -15,14 +15,11 @@ class RtnSingleTeamYear(object):
             else:
                 self.team_id = team_id
 
-    def connect_session(self):
-        self.session = session_setup()
+    def get_team_mapping(self, force_update=False):
+        if force_update:
+            get_data_from_api.clear_cache()
 
-    def get_team_mapping(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
-        all_teams_data = get_data_from_api(session=self.session, endpoint='gymnasts2', suffix=str(self.year) + '/1').json()
+        all_teams_data = get_data_from_api(endpoint='gymnasts2', suffix=str(self.year) + '/1').json()
         return {team['team_name']: team['id'] for team in all_teams_data['teams']}
 
     def get_team_id(self):
@@ -38,22 +35,22 @@ class RtnSingleTeamYear(object):
         # else:
         #     raise ValueError(f'{self.team_name} does not exist in data for {self.year}')
 
-    def _get_raw_roster(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
+    def _get_raw_roster(self, force_update=False):
         rename_map = {'id': 'Gymnast ID', 'hometown': 'Hometown', 'school_year': 'School Year', 'events': 'Events'}
         school_year_map = {'1': 'FR', '2': 'SO', '3': 'JR', '4': 'SR'}
 
-        roster_data = get_data_from_api(session=self.session, endpoint='rostermain', suffix=str(self.year)+'/'+str(self.team_id)+'/1').json()
+        if force_update:
+            get_data_from_api.clear_cache()
+
+        roster_data = get_data_from_api(endpoint='rostermain', suffix=str(self.year)+'/'+str(self.team_id)+'/1').json()
 
         self._raw_roster = [{**{rename_map.get(k, k): v if k != 'school_year' else school_year_map.get(v, '') for k, v in data.items()},
                             **{'Name': data['fname'] + ' ' + data['lname'], 'Team': self.team_name}}
                             for data in roster_data]
 
-    def get_roster(self, include_hometowns=False, include_class=False, include_events=False):
+    def get_roster(self, include_hometowns=False, include_class=False, include_events=False, force_update=False):
         if not hasattr(self, 'raw_roster'):
-            self._get_raw_roster()
+            self._get_raw_roster(force_update=force_update)
 
         extra_cols = get_extra_cols(include_hometowns=include_hometowns, include_class=include_class, include_events=include_events)
 
@@ -65,11 +62,11 @@ class RtnSingleTeamYear(object):
 
         return self.roster
 
-    def _get_raw_season_results(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
+    def _get_raw_season_results(self, force_update=False):
+        if force_update:
+            get_data_from_api.clear_cache()
 
-        meets = get_data_from_api(session=self.session, endpoint='dashboard', suffix=str(self.year)+'/'+str(self.team_id)).json()
+        meets = get_data_from_api(endpoint='dashboard', suffix=str(self.year)+'/'+str(self.team_id)).json()
         name_map = {'team_id': 'Team ID', 'team_name': 'Team', 'meet_id': 'Team Meet ID',
                     'meet_date': 'Meet Date', 'team_score': 'Score', 'home': 'Home/Away',
                     'opponent': 'Opponents', 'meet_desc': 'Meet Name', 'linked_id': 'Meet ID'}
@@ -79,9 +76,9 @@ class RtnSingleTeamYear(object):
                                      for k, v in data.items() if k != 'jas'} for data in meets['meets'] if data['team_name'] == self.team_name]
         self._raw_schedule = [{k: v for k, v in data.items() if k not in ('Score', 'VT', 'UB', 'BB', 'FX')} for data in self._raw_season_results]
 
-    def get_schedule(self):
+    def get_schedule(self, force_update=False):
         if not hasattr(self, '_raw_schedule'):
-            self._get_raw_season_results()
+            self._get_raw_season_results(force_update=force_update)
 
         if len(self._raw_schedule) > 0:
             return pd.DataFrame(self._raw_schedule)
@@ -103,14 +100,14 @@ class RtnSingleTeamYear(object):
             * Uses team meet id to join back to meet info, such as opponent, etc.
         """
         if not hasattr(self, '_raw_season_results'):
-            self._get_raw_season_results()
+            self._get_raw_season_results(force_update=force_update)
 
         if len(self._raw_season_results) > 0:
             if (len({'VT', 'UB', 'BB', 'FX'}.intersection(self._raw_season_results[0].keys())) != 4 or force_update):
                 if method == 'team_consistency':
-                    self._team_event_scores_team_consistency()
+                    self._team_event_scores_team_consistency(force_update=force_update)
                 elif method == 'by_meet':
-                    self._team_event_scores_by_meet()
+                    self._team_event_scores_by_meet(force_update=force_update)
                 else:
                     raise ValueError('Method must be "team_consistency" or "by_meet"')
 
@@ -121,14 +118,14 @@ class RtnSingleTeamYear(object):
 
         return self.season_results
 
-    def _team_event_scores_by_meet(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
+    def _team_event_scores_by_meet(self, force_update=False):
         team_scores_all = []
         for meet_id in [data['Team Meet ID'] for data in self._raw_season_results if data['Meet Date'] <= datetime.now()]:
             try:
-                meet_res = get_data_from_api(session=self.session, endpoint='meetresults', suffix=str(meet_id)).json()
+                if force_update:
+                    get_data_from_api.clear_cache()
+
+                meet_res = get_data_from_api(endpoint='meetresults', suffix=str(meet_id)).json()
                 # This API call returns scores from all teams at this meet, not just this team. Need to pick out correct score
                 team_scores = [score for score in meet_res['teams'] if score['tname'] == self.team_name and score['mid'] == str(meet_id)]
                 assert len(team_scores) == 1, 'Multiple team scores??'
@@ -143,11 +140,11 @@ class RtnSingleTeamYear(object):
             for i in range(len(self._raw_season_results)):
                 self._raw_season_results[i].update({'VT': np.nan, 'UB': np.nan, 'BB': np.nan, 'FX': np.nan})
 
-    def _team_event_scores_team_consistency(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
+    def _team_event_scores_team_consistency(self, force_update=False):
+        if force_update:
+            get_data_from_api.clear_cache()
 
-        res = get_data_from_api(session=self.session, endpoint='teamConsistency', suffix=f'{self.year}/{self.team_id}').json()
+        res = get_data_from_api(endpoint='teamConsistency', suffix=f'{self.year}/{self.team_id}').json()
         if len(res['labels']) == 0:
             print(f'No team consistency data found for {self.team_name} in {self.year}')
             for i in range(len(self._raw_season_results)):
@@ -182,22 +179,22 @@ class RtnSingleTeamYear(object):
                 if not hasattr(self, '_raw_roster'):
                     self.get_roster()
 
-                self._individual_scores_individual_consistency()
+                self._individual_scores_individual_consistency(force_update=force_update)
             elif method == 'by_meet':
-                self._individual_scores_by_meet()
+                self._individual_scores_by_meet(force_update=force_update)
             else:
                 raise ValueError('Method must be "individual_consistency" or "by_meet"')
 
         return self.individual_results
 
-    def _individual_scores_by_meet(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
+    def _individual_scores_by_meet(self, force_update=False):
         individual_scores_all = []
         for meet_id in [meet['Team Meet ID'] for meet in self._raw_schedule if meet['Meet Date'] <= datetime.now()]:
             try:
-                meet_res = get_data_from_api(session=self.session, endpoint='meetresults', suffix=str(meet_id)).json()
+                if force_update:
+                    get_data_from_api.clear_cache()
+
+                meet_res = get_data_from_api(endpoint='meetresults', suffix=str(meet_id)).json()
                 if len(meet_res) == 0 or len(meet_res['scores']) == 0 or len(meet_res['scores'][0]) == 0:
                     print(f'No data found for meet {meet_id}')
                     continue
@@ -232,14 +229,14 @@ class RtnSingleTeamYear(object):
                                                     'Team ID', 'Team', 'Team Meet ID', 'Home/Away', 'Opponents',
                                                     'Meet Name', 'Meet ID'])
 
-    def _individual_scores_individual_consistency(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
+    def _individual_scores_individual_consistency(self, force_update=False):
         ind_consistency_all = []
         for gymnast in self._raw_roster:
             try:
-                res = get_data_from_api(session=self.session, endpoint='indConsistency', suffix=f"{self.year}/{gymnast['Gymnast ID']}").json()
+                if force_update:
+                    get_data_from_api.clear_cache()
+
+                res = get_data_from_api(endpoint='indConsistency', suffix=f"{self.year}/{gymnast['Gymnast ID']}").json()
                 ind_consistency = [{'Meet Date': normalize_date(res['labels'][i][:7] + str(self.year), dt_format='%b-%d-%Y'),
                                     'VT': round(float(res['vts'][i]), 4) if res['vts'][i] is not None else np.nan,
                                     'UB': round(float(res['ubs'][i]), 4) if res['ubs'][i] is not None else np.nan,
@@ -260,27 +257,26 @@ class RtnSingleTeamYear(object):
         else:
             self.individual_results = pd.DataFrame(columns=SCHEDULE_COLS + IND_RESULTS_COLS)
 
-    def get_individual_nqs(self):
+    def get_individual_nqs(self, force_update=False):
         if not hasattr(self, '_raw_roster'):
-            self._get_raw_roster()
+            self._get_raw_roster(force_update=force_update)
 
         if not hasattr(self, '_raw_individual_nqs'):
-            self._get_raw_individual_nqs()
+            self._get_raw_individual_nqs(force_update=force_update)
 
         if len(self._raw_individual_nqs) > 0:
             return pd.DataFrame(self._raw_individual_nqs)
         else:
             return pd.DataFrame(columns=ROSTER_COLS + EVENTS) # + ['AA'])
 
-    def _get_raw_individual_nqs(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
+    def _get_raw_individual_nqs(self, force_update=False):
         name_map = {'maxv': 'VT', 'maxub': 'UB', 'maxbb': 'BB', 'maxfx': 'FX',
                     # 'maxaa': 'AA',
                     'gid': 'Gymnast ID'}
+        if force_update:
+            get_data_from_api.clear_cache()
 
-        nqsData = get_data_from_api(session=self.session, endpoint='rostermain', suffix=f'{self.year}/{self.team_id}/4').json()
+        nqsData = get_data_from_api(endpoint='rostermain', suffix=f'{self.year}/{self.team_id}/4').json()
         ind_nqs = [{name_map[k]: round(float(v), 4) if k != 'gid' and v != '' else (np.nan if k != 'gid' else v)
                     for k, v in data.items() if k in name_map.keys()} for data in nqsData['ind']]
 
@@ -291,25 +287,25 @@ class RtnSingleTeamYear(object):
         else:
             self._raw_individual_nqs = []
 
-    def _get_current_week(self):
-        if not hasattr(self, 'session'):
-            self.connect_session()
-
+    def _get_current_week(self, force_update=False):
         if not hasattr(self, 'week'):
-            week_data = get_data_from_api(session=self.session, endpoint='currentweek', suffix=str(self.year)).json()
-            return min(int(week_data['week']), int(week_data['max']))
+            if force_update:
+                get_data_from_api.clear_cache()
 
-    def _get_raw_rankings(self, team_vs_ind, event, week):
-        if not hasattr(self, 'session'):
-            self.connect_session()
+            week_data = get_data_from_api(endpoint='currentweek', suffix=str(self.year)).json()
+            return min(int(week_data['week']), int(week_data['max']))
 
+    def _get_raw_rankings(self, team_vs_ind, event, week, force_update=False):
         team_ind_map = {'team': 0, 'ind': 1}
         event_api_map = {'VT': 1, 'UB': 2, 'BB': 3, 'FX': 4, 'AA': 5}
         rename_map = {'rank': 'Rank', 'gid': 'Gymnast ID', 'team': 'Team', 'tid': 'Team ID',
                       'rqs': 'NQS', 'reg': 'Region', 'con': 'Conference', 'div': 'Division',
                       'usag': 'USAG', 'ave': 'Average', 'high': 'High', 'name': 'Team'}
 
-        res = get_data_from_api(session=self.session, endpoint='results', suffix=f'{self.year}/{week}/{team_ind_map[team_vs_ind]}/{event_api_map[event]}').json()
+        if force_update:
+            get_data_from_api.clear_cache()
+
+        res = get_data_from_api(endpoint='results', suffix=f'{self.year}/{week}/{team_ind_map[team_vs_ind]}/{event_api_map[event]}').json()
         if team_vs_ind == 'ind':
             self._raw_rankings[team_vs_ind][event] = [{**{rename_map.get(k): float(v) if k in ['rqs', 'ave', 'high'] else v for k, v in data.items() if k in rename_map},
                                                        **{'Name': data['fname'] + ' ' + data['lname'], 'Event': event}}
@@ -319,9 +315,9 @@ class RtnSingleTeamYear(object):
                                                        **{'Event': event}}
                                                       for data in res['data']]
 
-    def get_overall_rankings(self, team_vs_ind='team', event='AA', week=None):
+    def get_overall_rankings(self, team_vs_ind='team', event='AA', week=None, force_update=False):
         if not week:
-            week = self._get_current_week()
+            week = self._get_current_week(force_update=force_update)
 
         if not hasattr(self, '_raw_rankings'):
             self._raw_rankings = {'team': {event: None for event in EVENT_MAP.values()},
@@ -333,7 +329,7 @@ class RtnSingleTeamYear(object):
                                'Division', 'Conference', 'Region', 'USAG']}
 
         if self._raw_rankings[team_vs_ind][event] is None:
-            self._get_raw_rankings(team_vs_ind=team_vs_ind, event=event, week=week)
+            self._get_raw_rankings(team_vs_ind=team_vs_ind, event=event, week=week, force_update=force_update)
 
         return pd.DataFrame(self._raw_rankings[team_vs_ind][event])[col_orders[team_vs_ind]]
 

{scraping_rtn-0.0.6.0 → scraping_rtn-0.0.7}/src/scraping_rtn/__init__.py

@@ -7,12 +7,12 @@ def save(df, filename):
     df.to_csv(filename, index=False)
 
 
-def all_teams(year):
+def all_teams(year, force_update=False):
     rtn = RtnSingleTeamYear(year=year, team_name=None)
-    return list(rtn.get_team_mapping().keys())
+    return list(rtn.get_team_mapping(force_update=force_update).keys())
 
 
-def roster(year, teams, include_hometowns=False, include_class=False, include_events=False, verbose=False):
+def roster(year, teams, include_hometowns=False, include_class=False, include_events=False, verbose=False, force_update=False):
     teams = validate_input(teams)
 
     all_rosters = []
@@ -21,7 +21,7 @@ def roster(year, teams, include_hometowns=False, include_class=False, include_ev
             print(f'Getting roster for {team}                              ', end='\r' if team != teams[-1] else None)
         rtn = RtnSingleTeamYear(year=year, team_name=team)
         res = rtn.get_roster(include_hometowns=include_hometowns, include_class=include_class,
-                             include_events=include_events)
+                             include_events=include_events, force_update=force_update)
         if verbose and len(res) == 0:
             print(f'\tNo roster found for {team}')
         all_rosters.append(res)
@@ -30,7 +30,7 @@ def roster(year, teams, include_hometowns=False, include_class=False, include_ev
     return pd.concat(all_rosters)[ROSTER_COLS + extra_cols]
 
 
-def schedule(year, teams, verbose=False):
+def schedule(year, teams, verbose=False, force_update=False):
     teams = validate_input(teams)
 
     all_schedules = []
@@ -38,7 +38,7 @@ def schedule(year, teams, verbose=False):
         if verbose:
             print(f'Getting schedule for {team}                              ', end='\r' if team != teams[-1] else None)
         rtn = RtnSingleTeamYear(year=year, team_name=team)
-        res = rtn.get_schedule()
+        res = rtn.get_schedule(force_update=force_update)
         if verbose and len(res) == 0:
             print(f'\tNo schedule found for {team}')
         all_schedules.append(res)
@@ -54,7 +54,7 @@ def team_results(year, teams, method='team_consistency', force_update=False, ver
         if verbose:
             print(f'Getting schedule and results for {team}                              ', end='\r' if team != teams[-1] else None)
         rtn = RtnSingleTeamYear(year=year, team_name=team)
-        res = rtn.get_team_scores(method=method,force_update=force_update)
+        res = rtn.get_team_scores(method=method, force_update=force_update)
         if verbose and len(res) == 0:
             print(f'\tNo schedule and results found for {team}')
         all_results.append(res)
@@ -78,7 +78,7 @@ def individual_results(year, teams, method='by_meet', force_update=False, verbos
     return pd.concat(all_scores)[SCHEDULE_COLS + IND_RESULTS_COLS]
 
 
-def individual_nqs(year, teams, verbose=False):
+def individual_nqs(year, teams, verbose=False, force_update=False):
     teams = validate_input(teams)
 
     all_nqs = []
@@ -86,7 +86,7 @@ def individual_nqs(year, teams, verbose=False):
         if verbose:
             print(f'Getting individual NQS for {team}                              ', end='\r' if team != teams[-1] else None)
         rtn = RtnSingleTeamYear(year=year, team_name=team)
-        res = rtn.get_individual_nqs()
+        res = rtn.get_individual_nqs(force_update=force_update)
         if verbose and len(res) == 0:
             print(f'\tNo individual NQS found for {team}')
         all_nqs.append(res)
@@ -94,6 +94,6 @@ def individual_nqs(year, teams, verbose=False):
     return pd.concat(all_nqs)[ROSTER_COLS + EVENTS] # + ['AA']]
 
 
-def rankings(year, team_vs_ind='team', event='AA', week=None):
+def rankings(year, team_vs_ind='team', event='AA', week=None, force_update=False):
     rtn = RtnSingleTeamYear(year=year, team_name=None)
-    return rtn.get_overall_rankings(team_vs_ind=team_vs_ind, event=event, week=week)
+    return rtn.get_overall_rankings(team_vs_ind=team_vs_ind, event=event, week=week, force_update=force_update)

{scraping_rtn-0.0.6.0 → scraping_rtn-0.0.7}/src/scraping_rtn/src.py

@@ -21,21 +21,18 @@ def validate_input(teams):
     return teams
 
 
-def session_setup():
+@lru_cache(maxsize=1000000)
+def get_data_from_api(endpoint, suffix):
     session = requests.Session()
     retry = Retry(connect=3, backoff_factor=0.5)
     adapter = HTTPAdapter(max_retries=retry)
     session.mount('http://', adapter)
     session.mount('https://', adapter)
 
-    return session
-
-
-@lru_cache(maxsize=1000000)
-def get_data_from_api(session, endpoint, suffix):
     url = 'https://www.roadtonationals.com/api/women/' + endpoint
     if suffix:
         url += '/' + suffix
+
     return session.get(url)
 
 

{scraping_rtn-0.0.6.0 → scraping_rtn-0.0.7}/src/scraping_rtn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: scraping_rtn
-Version: 0.0.6.0
+Version: 0.0.7
 Summary: package to scrape gymnastics data from Road To Nationals
 Author-email: Claire Harmon <ceharmon220@gmail.com>
 Project-URL: Homepage, https://github.com/cgn-charmon/scraping_rtn