scraping-rtn 0.0.1__tar.gz → 0.0.8.0__tar.gz

{scraping_rtn-0.0.1 → scraping_rtn-0.0.8.0}/PKG-INFO

@@ -1,15 +1,15 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: scraping_rtn
-Version: 0.0.1
+Version: 0.0.8.0
 Summary: package to scrape gymnastics data from Road To Nationals
 Author-email: Claire Harmon <ceharmon220@gmail.com>
-Project-URL: Homepage, https://github.com/cgn-charmon/scraping_rtn
+License: MIT
+Project-URL: Homepage, https://github.com/ceharmon/scraping_rtn
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-License-File: LICENSE
 Requires-Dist: pandas>=1.5.3
 Requires-Dist: numpy>=1.23.5
 Requires-Dist: requests>=2.28.1

{scraping_rtn-0.0.1 → scraping_rtn-0.0.8.0}/pyproject.toml

@@ -1,21 +1,25 @@
 [build-system]
-requires = ["setuptools>=61.0"]
+requires = ["setuptools>=69.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "scraping_rtn"
-version = "0.0.1"
+version = "0.0.8.0"
 authors = [
-    { name="Claire Harmon", email="ceharmon220@gmail.com" },
+    { name = "Claire Harmon", email = "ceharmon220@gmail.com" },
 ]
 description = "package to scrape gymnastics data from Road To Nationals"
 readme = "README.md"
 requires-python = ">=3.9"
+
+license = { text = "MIT" }
+
 dependencies = [
     "pandas >=1.5.3",
     "numpy >=1.23.5",
     "requests >=2.28.1"
 ]
+
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",
@@ -23,4 +27,7 @@ classifiers = [
 ]
 
 [project.urls]
-"Homepage" = "https://github.com/cgn-charmon/scraping_rtn"
+Homepage = "https://github.com/ceharmon/scraping_rtn"
+
+[tool.setuptools]
+license-files = []

{scraping_rtn-0.0.1 → scraping_rtn-0.0.8.0}/src/scraping_rtn/RtnSingleTeamYear.py

@@ -1,11 +1,12 @@
-from .src import EVENT_MAP, EVENTS, get_data_from_api, fix_opponents, normalize_date, merge_dicts, get_extra_cols, \
-    SCHEDULE_COLS, RESULTS_COLS, IND_RESULTS_COLS, ROSTER_COLS
+from .src import EVENT_MAP, EVENTS, get_session, get_data_from_api, fix_opponents, normalize_date, merge_dicts, \
+    get_extra_cols, SCHEDULE_COLS, RESULTS_COLS, IND_RESULTS_COLS, ROSTER_COLS
 import pandas as pd
 import numpy as np
+from datetime import datetime
 
 
 class RtnSingleTeamYear(object):
-    def __init__(self, year, team_name, team_id=None):
+    def __init__(self, year, team_name, team_id=None, session=None):
         self.year = year
         if team_name is not None:
             self.team_name = team_name
@@ -14,33 +15,47 @@ class RtnSingleTeamYear(object):
             else:
                 self.team_id = team_id
 
-    def get_team_mapping(self):
-        all_teams_data = get_data_from_api(endpoint='gymnasts2', suffix=str(self.year) + '/1').json()
+        if session is None:
+            self.session = get_session()
+        else:
+            self.session = session
+
+    def get_team_mapping(self, force_update=False):
+        if force_update:
+            get_data_from_api.cache_clear()
+
+        all_teams_data = get_data_from_api(endpoint='gymnasts2', suffix=str(self.year) + '/1', session=self.session).json()
         return {team['team_name']: team['id'] for team in all_teams_data['teams']}
 
     def get_team_id(self):
         if not hasattr(self, 'team_id_map'):
             self.team_id_map = self.get_team_mapping()
 
+        if self.team_name and self.team_name not in self.team_id_map.keys():
+            raise ValueError(f'Unknown team name: {self.team_name}')
+
         return self.team_id_map.get(self.team_name, -1)
         # if self.team_name in self.team_id_map.keys():
         #     return self.team_id_map[self.team_name]
         # else:
         #     raise ValueError(f'{self.team_name} does not exist in data for {self.year}')
 
-    def _get_raw_roster(self):
+    def _get_raw_roster(self, force_update=False):
         rename_map = {'id': 'Gymnast ID', 'hometown': 'Hometown', 'school_year': 'School Year', 'events': 'Events'}
         school_year_map = {'1': 'FR', '2': 'SO', '3': 'JR', '4': 'SR'}
 
-        roster_data = get_data_from_api(endpoint='rostermain', suffix=str(self.year)+'/'+str(self.team_id)+'/1').json()
+        if force_update:
+            get_data_from_api.cache_clear()
 
-        self._raw_roster = [{**{rename_map.get(k, k): v if k != 'school_year' else school_year_map[v] for k, v in data.items()},
+        roster_data = get_data_from_api(endpoint='rostermain', suffix=str(self.year)+'/'+str(self.team_id)+'/1', session=self.session).json()
+
+        self._raw_roster = [{**{rename_map.get(k, k): v if k != 'school_year' else school_year_map.get(v, '') for k, v in data.items()},
                             **{'Name': data['fname'] + ' ' + data['lname'], 'Team': self.team_name}}
                             for data in roster_data]
 
-    def get_roster(self, include_hometowns=False, include_class=False, include_events=False):
+    def get_roster(self, include_hometowns=False, include_class=False, include_events=False, force_update=False):
         if not hasattr(self, 'raw_roster'):
-            self._get_raw_roster()
+            self._get_raw_roster(force_update=force_update)
 
         extra_cols = get_extra_cols(include_hometowns=include_hometowns, include_class=include_class, include_events=include_events)
 
@@ -52,20 +67,23 @@ class RtnSingleTeamYear(object):
 
         return self.roster
 
-    def _get_raw_season_results(self):
-        meets = get_data_from_api(endpoint='dashboard', suffix=str(self.year)+'/'+str(self.team_id)).json()
+    def _get_raw_season_results(self, force_update=False):
+        if force_update:
+            get_data_from_api.cache_clear()
+
+        meets = get_data_from_api(endpoint='dashboard', suffix=str(self.year)+'/'+str(self.team_id), session=self.session).json()
         name_map = {'team_id': 'Team ID', 'team_name': 'Team', 'meet_id': 'Team Meet ID',
                     'meet_date': 'Meet Date', 'team_score': 'Score', 'home': 'Home/Away',
                     'opponent': 'Opponents', 'meet_desc': 'Meet Name', 'linked_id': 'Meet ID'}
 
         self._raw_season_results = [{name_map.get(k, k): fix_opponents(v) if k == 'opponent'
                                     else (normalize_date(v) if k == 'meet_date' else v)
-                                     for k, v in data.items() if k != 'jas'} for data in meets['meets']]
+                                     for k, v in data.items() if k != 'jas'} for data in meets['meets'] if data['team_name'] == self.team_name]
         self._raw_schedule = [{k: v for k, v in data.items() if k not in ('Score', 'VT', 'UB', 'BB', 'FX')} for data in self._raw_season_results]
 
-    def get_schedule(self):
+    def get_schedule(self, force_update=False):
         if not hasattr(self, '_raw_schedule'):
-            self._get_raw_season_results()
+            self._get_raw_season_results(force_update=force_update)
 
         if len(self._raw_schedule) > 0:
             return pd.DataFrame(self._raw_schedule)
@@ -87,29 +105,32 @@ class RtnSingleTeamYear(object):
             * Uses team meet id to join back to meet info, such as opponent, etc.
         """
         if not hasattr(self, '_raw_season_results'):
-            self._get_raw_season_results()
+            self._get_raw_season_results(force_update=force_update)
 
         if len(self._raw_season_results) > 0:
             if (len({'VT', 'UB', 'BB', 'FX'}.intersection(self._raw_season_results[0].keys())) != 4 or force_update):
                 if method == 'team_consistency':
-                    self._team_event_scores_team_consistency()
+                    self._team_event_scores_team_consistency(force_update=force_update)
                 elif method == 'by_meet':
-                    self._team_event_scores_by_meet()
+                    self._team_event_scores_by_meet(force_update=force_update)
                 else:
                     raise ValueError('Method must be "team_consistency" or "by_meet"')
 
             # TODO: different way to drop duplicates?
-            self.season_results = pd.DataFrame(self._raw_season_results).drop_duplicates()
+            self.season_results = pd.DataFrame(self._raw_season_results).dropna(subset=['Score']).drop_duplicates()
         else:
             self.season_results = pd.DataFrame(columns=SCHEDULE_COLS + RESULTS_COLS)
 
         return self.season_results
 
-    def _team_event_scores_by_meet(self):
+    def _team_event_scores_by_meet(self, force_update=False):
         team_scores_all = []
-        for meet_id in [data['Team Meet ID'] for data in self._raw_season_results]:
+        for meet_id in [data['Team Meet ID'] for data in self._raw_season_results if data['Meet Date'] <= datetime.now()]:
             try:
-                meet_res = get_data_from_api(endpoint='meetresults', suffix=str(meet_id)).json()
+                if force_update:
+                    get_data_from_api.cache_clear()
+
+                meet_res = get_data_from_api(endpoint='meetresults', suffix=str(meet_id), session=self.session).json()
                 # This API call returns scores from all teams at this meet, not just this team. Need to pick out correct score
                 team_scores = [score for score in meet_res['teams'] if score['tname'] == self.team_name and score['mid'] == str(meet_id)]
                 assert len(team_scores) == 1, 'Multiple team scores??'
@@ -124,10 +145,13 @@ class RtnSingleTeamYear(object):
             for i in range(len(self._raw_season_results)):
                 self._raw_season_results[i].update({'VT': np.nan, 'UB': np.nan, 'BB': np.nan, 'FX': np.nan})
 
-    def _team_event_scores_team_consistency(self):
-        res = get_data_from_api(endpoint='teamConsistency', suffix=f'{self.year}/{self.team_id}').json()
+    def _team_event_scores_team_consistency(self, force_update=False):
+        if force_update:
+            get_data_from_api.cache_clear()
+
+        res = get_data_from_api(endpoint='teamConsistency', suffix=f'{self.year}/{self.team_id}', session=self.session).json()
         if len(res['labels']) == 0:
-            print(f'No team consistency data found for year {self.year}')
+            print(f'No team consistency data found for {self.team_name} in {self.year}')
             for i in range(len(self._raw_season_results)):
                 self._raw_season_results[i].update({'VT': np.nan, 'UB': np.nan, 'BB': np.nan, 'FX': np.nan})
         else:
@@ -145,12 +169,10 @@ class RtnSingleTeamYear(object):
         Methods:
         * Individual Consistency - Uses Individual Consistency tab from RTN
             * Tends to have more complete data, especially for older years
-            * Requires summing of all events to get AA (code does this for you)
             * Relies on date to join back to meet info, such as opponent, etc.
             * One API call per gymnast, relative speed depends on number of meets vs number of gymnasts
         * By Meet - loops through each meet to get scores
             * Older meets tend to be missing
-            * Includes AA scores in the response
             * Uses team meet id to join back to meet info, such as opponent, etc.
             * One API call per meet, relative speed depends on number of meets vs number of gymnasts
         """
@@ -162,20 +184,31 @@ class RtnSingleTeamYear(object):
                 if not hasattr(self, '_raw_roster'):
                     self.get_roster()
 
-                self._individual_scores_individual_consistency()
+                self._individual_scores_individual_consistency(force_update=force_update)
             elif method == 'by_meet':
-                self._individual_scores_by_meet()
+                self._individual_scores_by_meet(force_update=force_update)
             else:
                 raise ValueError('Method must be "individual_consistency" or "by_meet"')
 
         return self.individual_results
 
-    def _individual_scores_by_meet(self):
+    def _individual_scores_by_meet(self, force_update=False):
         individual_scores_all = []
-        for meet_id in [meet['Team Meet ID'] for meet in self._raw_schedule]:
+        for meet_id in [meet['Team Meet ID'] for meet in self._raw_schedule if meet['Meet Date'] <= datetime.now()]:
             try:
-                meet_res = get_data_from_api(endpoint='meetresults', suffix=str(meet_id)).json()
-                team_inds = [ind for ind, scores in enumerate(meet_res['scores']) if len(scores) > 0 and scores[0]['team_name'] == self.team_name]
+                if force_update:
+                    get_data_from_api.cache_clear()
+
+                meet_res = get_data_from_api(endpoint='meetresults', suffix=str(meet_id), session=self.session).json()
+                if len(meet_res) == 0 or len(meet_res['scores']) == 0 or len(meet_res['scores'][0]) == 0:
+                    print(f'No data found for meet {meet_id}')
+                    continue
+
+                if 'team_name' in meet_res['scores'][0][0]:
+                    team_inds = [ind for ind, scores in enumerate(meet_res['scores']) if len(scores) > 0 and scores[0]['team_name'] == self.team_name]
+                else:
+                    raise ValueError('Key not found')
+
                 if len(team_inds) == 0:
                     print(f'No scores found at meet {meet_id}')
                     continue
@@ -195,16 +228,20 @@ class RtnSingleTeamYear(object):
         if len(individual_scores_all) > 0:
             merge_dicts(dict1=individual_scores_all, dict2=self._raw_schedule, merge_field='Team Meet ID')
             self.individual_results = pd.DataFrame(individual_scores_all)
+            self.individual_results['AA'] = self.individual_results[['VT', 'UB', 'BB', 'FX']].dropna(how='any').astype(float).T.sum().round(4)
         else:
             self.individual_results = pd.DataFrame(columns=['Meet Date', 'VT', 'UB', 'BB', 'FX', 'AA', 'Gymnast ID', 'Name',
                                                     'Team ID', 'Team', 'Team Meet ID', 'Home/Away', 'Opponents',
                                                     'Meet Name', 'Meet ID'])
 
-    def _individual_scores_individual_consistency(self):
+    def _individual_scores_individual_consistency(self, force_update=False):
         ind_consistency_all = []
         for gymnast in self._raw_roster:
             try:
-                res = get_data_from_api(endpoint='indConsistency', suffix=f"{self.year}/{gymnast['Gymnast ID']}").json()
+                if force_update:
+                    get_data_from_api.cache_clear()
+
+                res = get_data_from_api(endpoint='indConsistency', suffix=f"{self.year}/{gymnast['Gymnast ID']}", session=self.session).json()
                 ind_consistency = [{'Meet Date': normalize_date(res['labels'][i][:7] + str(self.year), dt_format='%b-%d-%Y'),
                                     'VT': round(float(res['vts'][i]), 4) if res['vts'][i] is not None else np.nan,
                                     'UB': round(float(res['ubs'][i]), 4) if res['ubs'][i] is not None else np.nan,
@@ -225,24 +262,26 @@ class RtnSingleTeamYear(object):
         else:
             self.individual_results = pd.DataFrame(columns=SCHEDULE_COLS + IND_RESULTS_COLS)
 
-    def get_individual_nqs(self):
+    def get_individual_nqs(self, force_update=False):
         if not hasattr(self, '_raw_roster'):
-            self._get_raw_roster()
+            self._get_raw_roster(force_update=force_update)
 
         if not hasattr(self, '_raw_individual_nqs'):
-            self._get_raw_individual_nqs()
+            self._get_raw_individual_nqs(force_update=force_update)
 
         if len(self._raw_individual_nqs) > 0:
             return pd.DataFrame(self._raw_individual_nqs)
         else:
             return pd.DataFrame(columns=ROSTER_COLS + EVENTS) # + ['AA'])
 
-    def _get_raw_individual_nqs(self):
+    def _get_raw_individual_nqs(self, force_update=False):
         name_map = {'maxv': 'VT', 'maxub': 'UB', 'maxbb': 'BB', 'maxfx': 'FX',
                     # 'maxaa': 'AA',
                     'gid': 'Gymnast ID'}
+        if force_update:
+            get_data_from_api.cache_clear()
 
-        nqsData = get_data_from_api(endpoint='rostermain', suffix=f'{self.year}/{self.team_id}/4').json()
+        nqsData = get_data_from_api(endpoint='rostermain', suffix=f'{self.year}/{self.team_id}/4', session=self.session).json()
         ind_nqs = [{name_map[k]: round(float(v), 4) if k != 'gid' and v != '' else (np.nan if k != 'gid' else v)
                     for k, v in data.items() if k in name_map.keys()} for data in nqsData['ind']]
 
@@ -253,18 +292,25 @@ class RtnSingleTeamYear(object):
         else:
             self._raw_individual_nqs = []
 
-    def _get_current_week(self):
+    def _get_current_week(self, force_update=False):
         if not hasattr(self, 'week'):
-            return get_data_from_api(endpoint='currentweek', suffix=str(self.year)).json()['max']
+            if force_update:
+                get_data_from_api.cache_clear()
 
-    def _get_raw_rankings(self, team_vs_ind, event, week):
+            week_data = get_data_from_api(endpoint='currentweek', suffix=str(self.year), session=self.session).json()
+            return min(int(week_data['week']), int(week_data['max']))
+
+    def _get_raw_rankings(self, team_vs_ind, event, week, force_update=False):
         team_ind_map = {'team': 0, 'ind': 1}
         event_api_map = {'VT': 1, 'UB': 2, 'BB': 3, 'FX': 4, 'AA': 5}
         rename_map = {'rank': 'Rank', 'gid': 'Gymnast ID', 'team': 'Team', 'tid': 'Team ID',
                       'rqs': 'NQS', 'reg': 'Region', 'con': 'Conference', 'div': 'Division',
                       'usag': 'USAG', 'ave': 'Average', 'high': 'High', 'name': 'Team'}
 
-        res = get_data_from_api(endpoint='results', suffix=f'{self.year}/{week}/{team_ind_map[team_vs_ind]}/{event_api_map[event]}').json()
+        if force_update:
+            get_data_from_api.cache_clear()
+
+        res = get_data_from_api(endpoint='results', suffix=f'{self.year}/{week}/{team_ind_map[team_vs_ind]}/{event_api_map[event]}', session=self.session).json()
         if team_vs_ind == 'ind':
             self._raw_rankings[team_vs_ind][event] = [{**{rename_map.get(k): float(v) if k in ['rqs', 'ave', 'high'] else v for k, v in data.items() if k in rename_map},
                                                        **{'Name': data['fname'] + ' ' + data['lname'], 'Event': event}}
@@ -274,9 +320,9 @@ class RtnSingleTeamYear(object):
                                                        **{'Event': event}}
                                                       for data in res['data']]
 
-    def get_overall_rankings(self, team_vs_ind='team', event='AA', week=None):
+    def get_overall_rankings(self, team_vs_ind='team', event='AA', week=None, force_update=False):
         if not week:
-            week = self._get_current_week()
+            week = self._get_current_week(force_update=force_update)
 
         if not hasattr(self, '_raw_rankings'):
             self._raw_rankings = {'team': {event: None for event in EVENT_MAP.values()},
@@ -288,7 +334,7 @@ class RtnSingleTeamYear(object):
                                'Division', 'Conference', 'Region', 'USAG']}
 
         if self._raw_rankings[team_vs_ind][event] is None:
-            self._get_raw_rankings(team_vs_ind=team_vs_ind, event=event, week=week)
+            self._get_raw_rankings(team_vs_ind=team_vs_ind, event=event, week=week, force_update=force_update)
 
         return pd.DataFrame(self._raw_rankings[team_vs_ind][event])[col_orders[team_vs_ind]]
 

scraping_rtn-0.0.8.0/src/scraping_rtn/__init__.py

@@ -0,0 +1,107 @@
+from .RtnSingleTeamYear import RtnSingleTeamYear
+from .src import get_session, validate_input, get_extra_cols, SCHEDULE_COLS, RESULTS_COLS, IND_RESULTS_COLS, EVENTS, ROSTER_COLS
+import pandas as pd
+
+BLANK_SPACES = ' '*30
+
+
+def save(df, filename):
+    df.to_csv(filename, index=False)
+
+
+def all_teams(year, force_update=False):
+    rtn = RtnSingleTeamYear(year=year, team_name=None)
+    return list(rtn.get_team_mapping(force_update=force_update).keys())
+
+
+def roster(year, teams, include_hometowns=False, include_class=False, include_events=False, verbose=False, force_update=False):
+    teams = validate_input(teams)
+    session = get_session()
+
+    all_rosters = []
+    for i, team in enumerate(teams):
+        if verbose:
+            print(f'Getting roster for {team}{BLANK_SPACES}', end='\r' if team != teams[-1] else None)
+        rtn = RtnSingleTeamYear(year=year, team_name=team, session=session)
+        res = rtn.get_roster(include_hometowns=include_hometowns, include_class=include_class,
+                             include_events=include_events, force_update=force_update if i == 0 else False)
+        if verbose and len(res) == 0:
+            print(f'\tNo roster found for {team}')
+        all_rosters.append(res)
+
+    extra_cols = get_extra_cols(include_hometowns=include_hometowns, include_class=include_class, include_events=include_events)
+    return pd.concat(all_rosters)[ROSTER_COLS + extra_cols]
+
+
+def schedule(year, teams, verbose=False, force_update=False):
+    teams = validate_input(teams)
+    session = get_session()
+
+    all_schedules = []
+    for i, team in enumerate(teams):
+        if verbose:
+            print(f'Getting schedule for {team}{BLANK_SPACES}', end='\r' if team != teams[-1] else None)
+        rtn = RtnSingleTeamYear(year=year, team_name=team, session=session)
+        res = rtn.get_schedule(force_update=force_update if i == 0 else False)
+        if verbose and len(res) == 0:
+            print(f'\tNo schedule found for {team}')
+        all_schedules.append(res)
+
+    return pd.concat(all_schedules)[SCHEDULE_COLS]
+
+
+def team_results(year, teams, method='team_consistency', force_update=False, verbose=False):
+    teams = validate_input(teams)
+    session = get_session()
+    
+    all_results = []
+    for i, team in enumerate(teams):
+        if verbose:
+            print(f'Getting schedule and results for {team}{BLANK_SPACES}', end='\r' if team != teams[-1] else None)
+        rtn = RtnSingleTeamYear(year=year, team_name=team, session=session)
+        res = rtn.get_team_scores(method=method, force_update=force_update if i == 0 else False)
+        if verbose and len(res) == 0:
+            print(f'\tNo schedule and results found for {team}')
+        all_results.append(res)
+
+    return pd.concat(all_results)[SCHEDULE_COLS + RESULTS_COLS]
+
+
+def individual_results(year, teams, method='by_meet', force_update=False, verbose=False):
+    teams = validate_input(teams)
+    session = get_session()
+    
+    all_scores = []
+    for i, team in enumerate(teams):
+        if verbose:
+            print(f'Getting scores for {team}{BLANK_SPACES}', end='\r' if team != teams[-1] else None)
+        rtn = RtnSingleTeamYear(year=year, team_name=team, session=session)
+        res = rtn.get_individual_scores(method=method, force_update=force_update if i == 0 else False)
+        if verbose and len(res) == 0:
+            print(f'\tNo scores found for {team}')
+        all_scores.append(res)
+
+    return pd.concat(all_scores)[SCHEDULE_COLS + IND_RESULTS_COLS]
+
+
+def individual_nqs(year, teams, verbose=False, force_update=False):
+    teams = validate_input(teams)
+    session = get_session()
+    
+    all_nqs = []
+    for i, team in enumerate(teams):
+        if verbose:
+            print(f'Getting individual NQS for {team}{BLANK_SPACES}', end='\r' if team != teams[-1] else None)
+        rtn = RtnSingleTeamYear(year=year, team_name=team, session=session)
+        res = rtn.get_individual_nqs(force_update=force_update if i == 0 else False)
+        if verbose and len(res) == 0:
+            print(f'\tNo individual NQS found for {team}')
+        all_nqs.append(res)
+
+    return pd.concat(all_nqs)[ROSTER_COLS + EVENTS] # + ['AA']]
+
+
+def rankings(year, team_vs_ind='team', event='AA', week=None, force_update=False):
+    session = get_session()
+    rtn = RtnSingleTeamYear(year=year, team_name=None, session=session)
+    return rtn.get_overall_rankings(team_vs_ind=team_vs_ind, event=event, week=week, force_update=force_update)

{scraping_rtn-0.0.1 → scraping_rtn-0.0.8.0}/src/scraping_rtn/src.py

@@ -1,4 +1,6 @@
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 from functools import lru_cache
 from datetime import datetime
 
@@ -19,16 +21,32 @@ def validate_input(teams):
     return teams
 
 
+def get_session():
+    session = requests.Session()
+    retry = Retry(connect=3, backoff_factor=0.5)
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+    return session
+
+
 @lru_cache(maxsize=1000000)
-def get_data_from_api(endpoint, suffix):
+def get_data_from_api(endpoint, suffix, session=None):
+    if not session:    
+        session = get_session()
+
     url = 'https://www.roadtonationals.com/api/women/' + endpoint
     if suffix:
         url += '/' + suffix
-    return requests.get(url)
+
+    return session.get(url)
 
 
 def fix_opponents(ops):
-    ops = (ops.replace(', ', '/').replace(' and ', '/').replace(' @ ', '/').replace(' w/ ', '/').replace(' with ', '/'))
+    if not isinstance(ops, str):
+        return ops
+
+    ops = (ops.replace(', ', '/').replace(',','/').replace(' and ', '/').replace(' @ ', '/').replace(' w/ ', '/').replace(' with ', '/'))
 
     if 'william & mary' in ops.lower():
         # Todo: title case is going to mess with other opponents here, full mapping might fix that

{scraping_rtn-0.0.1 → scraping_rtn-0.0.8.0}/src/scraping_rtn.egg-info/PKG-INFO

@@ -1,15 +1,15 @@
-Metadata-Version: 2.1
-Name: scraping-rtn
-Version: 0.0.1
+Metadata-Version: 2.4
+Name: scraping_rtn
+Version: 0.0.8.0
 Summary: package to scrape gymnastics data from Road To Nationals
 Author-email: Claire Harmon <ceharmon220@gmail.com>
-Project-URL: Homepage, https://github.com/cgn-charmon/scraping_rtn
+License: MIT
+Project-URL: Homepage, https://github.com/ceharmon/scraping_rtn
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-License-File: LICENSE
 Requires-Dist: pandas>=1.5.3
 Requires-Dist: numpy>=1.23.5
 Requires-Dist: requests>=2.28.1

{scraping_rtn-0.0.1 → scraping_rtn-0.0.8.0}/src/scraping_rtn.egg-info/SOURCES.txt

@@ -1,4 +1,3 @@
-LICENSE
 README.md
 pyproject.toml
 src/scraping_rtn/RtnSingleTeamYear.py

scraping_rtn-0.0.1/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2023 Claire Harmon
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

scraping_rtn-0.0.1/src/scraping_rtn/__init__.py

@@ -1,99 +0,0 @@
-from .RtnSingleTeamYear import RtnSingleTeamYear
-from .src import validate_input, get_extra_cols, SCHEDULE_COLS, RESULTS_COLS, IND_RESULTS_COLS, EVENTS, ROSTER_COLS
-import pandas as pd
-
-
-def save(df, filename):
-    df.to_csv(filename, index=False)
-
-
-def all_teams(year):
-    rtn = RtnSingleTeamYear(year=year, team_name=None)
-    return list(rtn.get_team_mapping().keys())
-
-
-def roster(year, teams, include_hometowns=False, include_class=False, include_events=False, verbose=False):
-    teams = validate_input(teams)
-
-    all_rosters = []
-    for team in teams:
-        if verbose:
-            print(f'Getting roster for {team}                              ', end='\r')
-        rtn = RtnSingleTeamYear(year=year, team_name=team)
-        res = rtn.get_roster(include_hometowns=include_hometowns, include_class=include_class,
-                             include_events=include_events)
-        if verbose and len(res) == 0:
-            print(f'\tNo roster found for {team}')
-        all_rosters.append(res)
-
-    extra_cols = get_extra_cols(include_hometowns=include_hometowns, include_class=include_class, include_events=include_events)
-    return pd.concat(all_rosters)[ROSTER_COLS + extra_cols]
-
-
-def schedule(year, teams, verbose=False):
-    teams = validate_input(teams)
-
-    all_schedules = []
-    for team in teams:
-        if verbose:
-            print(f'Getting schedule for {team}                              ', end='\r')
-        rtn = RtnSingleTeamYear(year=year, team_name=team)
-        res = rtn.get_schedule()
-        if verbose and len(res) == 0:
-            print(f'\tNo schedule found for {team}')
-        all_schedules.append(res)
-
-    return pd.concat(all_schedules)[SCHEDULE_COLS]
-
-
-def team_results(year, teams, method='team_consistency', force_update=False, verbose=False):
-    teams = validate_input(teams)
-
-    all_results = []
-    for team in teams:
-        if verbose:
-            print(f'Getting schedule and results for {team}                              ', end='\r')
-        rtn = RtnSingleTeamYear(year=year, team_name=team)
-        res = rtn.get_team_scores(method=method,force_update=force_update)
-        if verbose and len(res) == 0:
-            print(f'\tNo schedule and results found for {team}')
-        all_results.append(res)
-
-    return pd.concat(all_results)[SCHEDULE_COLS + RESULTS_COLS]
-
-
-def individual_results(year, teams, method='by_meet', force_update=False, verbose=False):
-    teams = validate_input(teams)
-
-    all_scores = []
-    for team in teams:
-        if verbose:
-            print(f'Getting scores for {team}                              ', end='\r')
-        rtn = RtnSingleTeamYear(year=year, team_name=team)
-        res = rtn.get_individual_scores(method=method, force_update=force_update)
-        if verbose and len(res) == 0:
-            print(f'\tNo scores found for {team}')
-        all_scores.append(res)
-
-    return pd.concat(all_scores)[SCHEDULE_COLS + IND_RESULTS_COLS]
-
-
-def individual_nqs(year, teams, verbose=False):
-    teams = validate_input(teams)
-
-    all_nqs = []
-    for team in teams:
-        if verbose:
-            print(f'Getting individual NQS for {team}                              ', end='\r')
-        rtn = RtnSingleTeamYear(year=year, team_name=team)
-        res = rtn.get_individual_nqs()
-        if verbose and len(res) == 0:
-            print(f'\tNo individual NQS found for {team}')
-        all_nqs.append(res)
-
-    return pd.concat(all_nqs)[ROSTER_COLS + EVENTS] # + ['AA']]
-
-
-def rankings(year, team_vs_ind='team', event='AA', week=None):
-    rtn = RtnSingleTeamYear(year=year, team_name=None)
-    return rtn.get_overall_rankings(team_vs_ind=team_vs_ind, event=event, week=week)