GetFootballData 0.1.0__py3-none-any.whl

GetFootballData/transfermarkt_scraper.py

@@ -0,0 +1,288 @@
+import pandas as pd
+import numpy as np
+import requests
+import time
+import random
+from bs4 import BeautifulSoup
+from curl_cffi import requests as cffi_requests
+from GetFootballData.scraper_utilities.league_to_tm_map import league_to_tm_dict
+
+pd.set_option('display.max_columns', None)
+pd.set_option('display.max_rows', None)
+
+root_site = 'www.transfermarkt.com'
+
+class TransfermarktScraper:
+
+    def __init__(self):
+
+        self.session = cffi_requests.Session(impersonate='chrome124')
+
+        self.session.headers.update({"accept": "application/json", "referer": f"https://{root_site}/"})
+
+        self.warm_up()
+
+    def warm_up(self):
+        """Request fotmob page to familiarise session with fotmob."""
+        try:
+            self.session.get(f"https://{root_site}/", timeout=5)
+            time.sleep(random.uniform(1.5, 3.0))
+        except Exception as e:
+            print(f"Warm-up handshake warning: {e}")
+
+    def __get_player_id_df(self, year: int, league: str) -> pd.DataFrame:
+
+        '''
+        This method scrapes the transfermarkt ids, nationality and club for every player in the given league
+
+        :param year: starting year of the league (ex. 2025)
+        :param league: league name according to the package's naming conventions (ex. 'ENG-Premier League', 'FRA-Ligue 1')
+        :return: a dataframe containing: player_name, player_id, club_name, nation. If data can't be found, an empty dataframe is returned.
+        '''
+
+        league_dict = league_to_tm_dict[league]
+        league_name = league_dict[0]
+        league_code = league_dict[1]
+
+        url = f'https://{root_site}/{league_name}/tabelle/wettbewerb/{league_code}/saison_id/{str(year)}'
+
+        r = self.session.get(url)
+
+        if not r:
+            return pd.DataFrame()
+
+        soup = BeautifulSoup(r.text, "html.parser")
+
+        club_acronym_list = []
+
+        table = soup.find('table', {'class': 'items'})
+
+        for row in table.find_all('tr'):
+            link = row.find('a', href=True)
+            if not link:
+                continue
+            href = link['href']
+            club_acronym = href.split('/')[1]
+            club_id = href.split('/')[4]
+            club_name = link['title']
+            club_acronym_list.append([club_acronym, club_id, club_name])
+
+        name_id_list = []
+
+        club_counter = 0
+
+        for club_acronym in club_acronym_list:
+
+            club_name = club_acronym[2]
+            #squad_url = f'https://www.transfermarkt.com/{club_acronym[0]}/kader/verein/{club_acronym[1]}/plus/0/galerie/0?saison_id={year}'
+            squad_url = f'https://{root_site}/{club_acronym[0]}/kader/verein/{club_acronym[1]}/saison_id/{year}'
+            r = self.session.get(squad_url)
+            soup = BeautifulSoup(r.text, "html.parser")
+            table = soup.find('table', {'class': 'items'})
+
+            for row in table.find_all('table', {'class': 'inline-table'}):
+                link_area = row.find('td', {'class': 'hauptlink'})
+                if not link_area:
+                    continue
+                link = link_area.find('a', href=True)
+                href = link['href']
+                player_id = href.split('/')[4]
+                player_name = link_area.get_text(strip=True)
+
+                parent_row = row.find_parent('tr')
+
+                nation = None
+                for area in parent_row.find_all('td', {'class': 'zentriert'}):
+                    nation_image = area.find('img', class_='flaggenrahmen')
+                    if not nation_image:
+                        continue
+                    nation = nation_image['title']
+
+                name_id_list.append((player_name, player_id, club_name, nation))
+
+            club_counter += 1
+            print(f'{club_counter}/{len(club_acronym_list)} teams complete.')
+
+        name_id_df = pd.DataFrame(name_id_list, columns=['player_name', 'player_id', 'club_name', 'nation']).set_index('player_name')
+
+        return name_id_df
+
+
+
+    def get_player_overview_df(self, year: int, league: str) -> pd.DataFrame:
+
+        dataframe = self.__get_player_id_df(year, league)
+
+        base_players_url = f'https://tmapi.transfermarkt.technology/players?'
+
+        team_list = dataframe['club_name'].unique().tolist()
+
+        final_player_dataframe = pd.DataFrame()
+
+        for team in team_list:
+
+            team_player_ids = dataframe[dataframe['club_name'] == team]['player_id'].tolist()
+
+            team_players_url = base_players_url
+
+            for player_id in team_player_ids:
+
+                team_players_url = team_players_url + f'ids[]={player_id}&'
+
+            team_players_url = team_players_url[:-1]
+
+            if team_players_url == base_players_url:
+                raise Exception(f'No team player ids found for {team}')
+
+            r = self.session.get(team_players_url)
+
+            player_data = r.json()['data']
+
+            team_data_list = []
+
+            for player in player_data:
+
+                club_assignment_list = [{}, {}]
+                for index, club_dict in enumerate(player['clubAssignments'][:2]): # Only go through first two assignments
+                    club_assignment_list[index] = club_dict
+
+                data_dict = {
+                    'player_name': player['name'],
+                    'player_id': player['id'],
+                    'age': player['lifeDates'].get('age', None),
+                    'date_of_birth': player['lifeDates'].get('dateOfBirth', None),
+                    'birth_place': player['birthPlaceDetails'].get('placeOfBirth', None),
+                    'position': player['attributes'].get('position', {}).get('name', None),
+                    'team': team,
+                    'general_position': player['attributes'].get('position', {}).get('category', None),
+                    'preferred_foot': player['attributes'].get('preferredFoot', {}).get('name', None),
+                    'height': player['attributes'].get('height', None),
+                    'contractEnd':player['attributes'].get('contractUntil', None),
+                    'sponsor': player['attributes'].get('outfitter', {}).get('name', None),
+                    'agent': player['attributes'].get('consultantAgency', {}).get('name', None),
+                    'market_value': player.get('marketValueDetails', {}).get('current', {}).get('value', None),
+                    'previous_market_value': player.get('marketValueDetails', {}).get('previous', {}).get('value', None),
+                    'highest_market_value': player.get('marketValueDetails', {}).get('highest', {}).get('value', None),
+                    'shirt_number': club_assignment_list[0].get('shirtNumber', None),
+                    'debut': club_assignment_list[0].get('debut', None),
+                    'is_captain': club_assignment_list[0].get('isCaptain', None),
+                    'nt_shirt_number': club_assignment_list[1].get('shirtNumber', None),
+                    'nt_debut': club_assignment_list[1].get('debut', None),
+                    'is_nt_captain': club_assignment_list[1].get('isCaptain', None),
+                }
+
+                team_data_list.append(data_dict)
+
+            team_stats_df = pd.DataFrame(team_data_list)
+            final_player_dataframe = pd.concat([final_player_dataframe, team_stats_df])
+
+        final_player_dataframe.set_index('player_id', inplace=True)
+
+        #Set Nationality
+        final_player_dataframe = final_player_dataframe.merge(dataframe.reset_index()[['player_id', 'nation']], on='player_id', how='left')
+
+        return final_player_dataframe
+
+
+    def get_player_transfer_history(self, player_id: str) -> pd.DataFrame:
+
+        '''
+        :param player_id: transfermarkt's player id (as seen in the url of a player's profile)
+        :return: A pandas dataframe containing a row for every recorded transfer a given player has had according to transfermarkt. If there is no history, an empty dataframe is returned. Rows include;
+
+
+        Columns:
+        - source_name: source club
+        - destination_name: destination club
+        - source_club_id: source club transfermarkt id
+        - destination_club_id: destination club transfermarkt id
+        - source_competition: league code of source team
+        - destination_competition: league code of destination team
+        - contract_start_date: start of contract date
+        - contract_end_date: end of contract date
+        - season: year of the beginning of the season
+        - market_value_at_time: market value at the time of the transfer
+        - age: age of the player at the time of the transfer
+        - fee: fee of the transfer in euros (€)
+        - transfer_type: description of the nature of the transfer ex. loan. Column will not contain data if it is a normal transfer.
+        - fee_description: Extra description of the nature of the attached fee, ex. loan fee. Column will not contain data if it is a normal transfer.
+        '''
+
+        url = f"https://tmapi.transfermarkt.technology/transfer/history/player/{player_id}"
+
+        r = self.session.get(url) #Fetch transfer history page via URL
+
+        print(player_id)
+
+        try:
+            transfer_history = r.json()["data"]['history']['terminated']
+        except:
+            return pd.DataFrame()
+
+
+
+        club_id_set = set()
+
+        source_data = transfer_history[0]["transferSource"]
+        club_id_set.add(source_data["clubId"])
+
+        for row in transfer_history:
+            source_data = row["transferSource"]
+            destination_data = row["transferDestination"]
+            club_id_set.add(source_data["clubId"])
+            club_id_set.add(destination_data["clubId"])
+
+        club_url = f'https://tmapi.transfermarkt.technology/clubs?'
+        for club_id in club_id_set:
+            club_url = club_url + f'ids[]={club_id}&'
+        club_url = club_url[:-1]
+
+
+        for i in range(1,5):
+            try:
+                club_r = self.session.get(club_url)  # Fetch club name data from transfermarkt api
+                club_data = club_r.json()["data"]
+                break
+            except requests.exceptions.RequestException as e:
+                print(e)
+                if i == 5:
+                    print(f'Could not access transfermarkt club data from api.')
+                    exit()
+                print(f'Request failed, retrying {i+1}/5 after 10 seconds')
+                time.sleep(10)
+
+
+
+        id_to_club_dict = {}
+        for row in club_data:
+            id_to_club_dict.update({row['id']:row['name']})
+
+        transfer_history_list = []
+
+        for row in transfer_history:
+            source_data = row["transferSource"]
+            destination_data = row["transferDestination"]
+            details_data = row["details"]
+            transfer_type_data = row['typeDetails']
+            entry = {'player_id': player_id,
+                     'source_name': id_to_club_dict[source_data['clubId']],
+                     'destination_name': id_to_club_dict[destination_data['clubId']],
+                     'source_club_id': source_data['clubId'],
+                     'destination_club_id': destination_data['clubId'],
+                     'source_competition': source_data['competitionId'],
+                     'destination_competition': destination_data['competitionId'],
+                     'contract_start_date': details_data["date"],
+                     'contract_end_date': details_data["contractUntilDate"],
+                     'season': details_data.get("season").get('id'),
+                     'market_value_at_time': details_data.get('marketValue', {}).get('value', np.nan),
+                     'age': details_data["age"],
+                     'fee': details_data.get('fee', {}).get('value', np.nan),
+                     'transfer_type': transfer_type_data.get('name', np.nan),
+                     'fee_description': transfer_type_data.get('feeDescription', np.nan),
+                     }
+
+            transfer_history_list.append(entry)
+
+        transfer_history_df = pd.DataFrame(transfer_history_list).set_index('player_id')
+
+        return transfer_history_df

getfootballdata-0.1.0.dist-info/METADATA

@@ -0,0 +1,110 @@
+Metadata-Version: 2.4
+Name: GetFootballData
+Version: 0.1.0
+Summary: Scrapers for FotMob, SofaScore and Transfermarkt
+Author: Max Newby
+License-Expression: AGPL-3.0-or-later
+Project-URL: Homepage, https://github.com/MaxNewby/GetFootballData
+Keywords: football,soccer,scraper,analytics,datasets
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: beautifulsoup4>=4.15
+Requires-Dist: curl_cffi>=0.16
+Requires-Dist: numpy>=2.4
+Requires-Dist: pandas>=3.0
+Requires-Dist: requests>=2.34
+Dynamic: license-file
+
+# GetFootballData
+
+This package contains classes to read data from Sofascore, FotMob and Transfermarkt.
+
+To make it easier league naming conventions are derived from other packages such as ScraperFC to make this package more compatible with others.
+
+Created by Max Newby
+
+>[NOTE] about using the 'league_name' parameter
+>
+>
+>To see the format in which the league_name parameter should be entered in, check the dictionary keys in any _map.py file inside scraper_utilities.
+> Examples include; 'ENG-Premier League', 'ESP-La Liga', 'GER-Bundesliga'.
+
+## <u>Using the Package:</u>
+
+### Sofascore_scraper.py:
+
+A SofascoreScraper object comes with the following methods:
+
+`get_player_id_dicts(self, year: int, league_name: str) -> dict[int, dict[str, str | dict[int, str]]]`
+
+This method returns a dictionary containing {team_id: team_data} pairs. Inside team_data, is the 'team_name' and another dictionary 'player_id_dict' that contains the {player_id, player_name} pairs.
+
+    year: Takes the starting year of the given season.
+    league_name: Takes the name of the given league in the given format.
+    
+    return: a dictionary of team names mapping to dictionaries containing player ids and names in the format {team_id: {name: team_name, player_dict: {player_id: player_name, ...}}, ...}.
+
+
+`get_player_metadata_df(self, year: int, league_name: str) -> pd.Dataframe`
+
+This method returns a dataframe containing general data about each player in the league for a given season.
+
+    :param year: starting year of the given season.
+    :param league_name: name of the league according to the given format.
+
+    :return: a pd.DataFrame containing general data about each player in a given league (height, preferred foot, etc.).
+
+`get_player_stat_df(self, year: int, league_name: str, stat_type='total', specific_stats=None, position_list=None, nationality_code_list=None, team_id_list=None, min_appearances=None, age=None, age_bound='EQ', home_or_away_only=None, preferred_foot=None) -> pd.DataFrame:`
+
+This method will return a dataframe containing statistical data for every player in the league for that given season. Responses can be filtered using the parameters.
+
+    year: starting year of the league in the given format
+    league_name: name of the league in the wanted format
+    stat_type: 'total', 'per90', 'perGame'
+    specific_stats: accepts a list of specific stats to return. Stats can be listed as seen on sofascore, in a league->stats->detailed tab
+    position_list: accepts a list of positions to return. options ['G', 'D', 'M', 'F']
+    nationality_code_list: accepts a list of nationality codes to return.
+    team_id_list: enter a list of sofascore's team IDs to return data only from those teams.
+    min_appearances: min appearances of a player in the given league/year
+    age: age of the player in the given league/year
+    age_bound: options: 'LT', 'EQ', 'GT'
+    home_or_away_only: 'home', 'away'
+    preferred_foot: 'left', 'right'. Returns only players with the given preferred foot.
+
+    return: a dataframe containing all records of stat data of given filters, for players in a given season and league
+
+### fotmob_scraper.py:
+
+A FotmobScraper object comes with the following methods:
+
+ `get_player_data_df(self, year: int, league_name: str) -> pd.DataFrame`
+
+This method scrapes statistical player data, for every player in the league for a given season
+
+    year: starting year of the league (ex. 2025)
+    league_name: league name in the package's standardised format: (ex. 'ENG-Premier League')
+    return: a pd.Dataframe instance containing the player data.
+
+
+
+### transfermarkt_scraper.py:
+
+A TransfermarktScraper object comes with the following methods:
+
+`transfermarkt_scraper.get_player_transfer_history(self, player_id: str) -> pd.DataFrame:`
+
+This method returns a dataframe containing a player's transfer history, and data such as fees and contract durations for each transfer.
+
+    player_id: transfermarkt's player id (as seen in the url of a player's profile)
+    return: A pandas dataframe containing a row for every recorded transfer a given player has had according to transfermarkt. If there is no history, an empty dataframe is returned. Rows include;
+
+
+`transfermarkt_scraper.get_player_overview_df(self, year: int, league: str)`
+
+This method scrapes general player data such as transfermarkt ids, nationality and club for every player in the given league
+
+    year: starting year of the league (ex. 2025)
+    league: league name according to the package's naming conventions (ex. 'ENG-Premier League', 'FRA-Ligue 1')
+    return: a dataframe containing: player_name, player_id, club_name, nation. If data can't be found, an empty dataframe is returned.
+

getfootballdata-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+GetFootballData/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+GetFootballData/fotmob_scraper.py,sha256=J6WCSooIHvjpM4WXrlYY6wx0yJxYZIXYXfbqY0vxfLA,14925
+GetFootballData/sofascore_scraper.py,sha256=HGc6-k-crftIXVfkreoORsABdMtMsGVKiOXsC8DOJg0,20556
+GetFootballData/transfermarkt_scraper.py,sha256=VRJtkGSWj9qQ3vQm_ehPVSUvbhv7Qkc1XCDO5Vq4yXk,12067
+GetFootballData/scraper_utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+GetFootballData/scraper_utilities/league_to_fotmob_map.py,sha256=HK6vs_raRvRGYAxgBvFlnWbnKVyqGvigR93VWC66APc,783
+GetFootballData/scraper_utilities/league_to_sofascore_map.py,sha256=_t2EHCprmMfCJ9hbf1swm9OQSDY4_LNfdgklgZIZ5Y4,32945
+GetFootballData/scraper_utilities/league_to_tm_map.py,sha256=HKSChPmTmbc3NTloI3SQW_x7tcxOGBKlWOCoD9fiFKk,320
+GetFootballData/scraper_utilities/year_maps.py,sha256=Ia33YqPyokMlRq51vhijFKq7ab7eZiCxGY1YdcqSxUM,1768
+getfootballdata-0.1.0.dist-info/licenses/LICENSE.md,sha256=sxfZgBWf-IozZGoJkRPevqw7flHraKK9PtjvoMiEs0w,37188
+getfootballdata-0.1.0.dist-info/METADATA,sha256=EZS9J-ApYwJv2gwse_buqmHZxwxjrvU_iPRYPJvbK5w,5260
+getfootballdata-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+getfootballdata-0.1.0.dist-info/top_level.txt,sha256=vH3LQZS30zltBF8geUokep7WtnAV-ZDZDPPhQQHy0Wg,16
+getfootballdata-0.1.0.dist-info/RECORD,,

getfootballdata-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+