datafc 0.1.0__tar.gz → 1.0.0__tar.gz

{datafc-0.1.0 → datafc-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datafc
-Version: 0.1.0
+Version: 1.0.0
 Summary: A scalable Python library for fetching, processing, and exporting structured football match data.
 Home-page: https://github.com/urazakgul/datafc
 Author: Uraz Akgül
@@ -14,7 +14,7 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 
-# datafc v0.1.0
+# datafc v1.0.0
 
 ## Overview
 
@@ -53,7 +53,7 @@ pip install git+https://github.com/urazakgul/datafc.git
 To install a specific version of `datafc`, use:
 
 ```bash
-pip install datafc==0.1.0
+pip install datafc==1.0.0
 ```
 
 ## Why Selenium?

{datafc-0.1.0 → datafc-1.0.0}/README.md

@@ -1,4 +1,4 @@
-# datafc v0.1.0
+# datafc v1.0.0
 
 ## Overview
 
@@ -37,7 +37,7 @@ pip install git+https://github.com/urazakgul/datafc.git
 To install a specific version of `datafc`, use:
 
 ```bash
-pip install datafc==0.1.0
+pip install datafc==1.0.0
 ```
 
 ## Why Selenium?

datafc-1.0.0/datafc/__init__.py

@@ -0,0 +1,3 @@
+from .sofascore import *
+
+__all__ = ["sofascore"]

datafc-1.0.0/datafc/sofascore/__init__.py

@@ -0,0 +1,23 @@
+from .fetch_match_data import match_data
+from .fetch_match_stats_data import match_stats_data
+from .fetch_standings_data import standings_data
+from .fetch_shots_data import shots_data
+from .fetch_goal_networks_data import goal_networks_data
+from .fetch_lineups_data import lineups_data
+from .fetch_coordinates_data import coordinates_data
+from .fetch_substitutions_data import substitutions_data
+from .fetch_match_odds_data import match_odds_data
+from .fetch_momentum_data import momentum_data
+
+__all__ = [
+    "match_data",
+    "match_stats_data",
+    "standings_data",
+    "shots_data",
+    "goal_networks_data",
+    "lineups_data",
+    "coordinates_data",
+    "substitutions_data",
+    "match_odds_data",
+    "momentum_data"
+]

datafc-1.0.0/datafc/sofascore/fetch_coordinates_data.py

@@ -0,0 +1,116 @@
+import json
+import pandas as pd
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import TimeoutException, WebDriverException
+from datafc.utils._setup_webdriver import setup_webdriver
+from datafc.utils._save_files import save_json, save_excel
+from datafc.utils._config import ALLOWED_SOURCES, API_BASE_URLS
+
+def coordinates_data(
+    lineups_df: pd.DataFrame,
+    data_source: str = "sofascore",
+    element_load_timeout: int = 10,
+    enable_json_export: bool = False,
+    enable_excel_export: bool = False
+) -> pd.DataFrame:
+    """
+    Fetches coordinate data for each player in the provided lineup dataset.
+
+    Args:
+        lineups_df (pd.DataFrame): A DataFrame containing player and match metadata,
+            which should be generated by the `lineups_data` function.
+        data_source (str): The data source ('sofavpn' or 'sofascore'). Defaults to 'sofascore'.
+        element_load_timeout (int): The maximum time (in seconds) to wait for the API response. Defaults to 10.
+        enable_json_export (bool): If `True`, exports the fetched coordinate data as a JSON file. Defaults to `False`.
+        enable_excel_export (bool): If `True`, exports the fetched coordinate data as an Excel file. Defaults to `False`.
+    """
+    if data_source not in ALLOWED_SOURCES:
+        raise ValueError(f"Invalid data source: {data_source}. Must be one of {ALLOWED_SOURCES}")
+
+    if lineups_df is None or lineups_df.empty:
+        raise ValueError("Match dataframe must be provided and cannot be empty.")
+
+    try:
+        webdriver_instance = setup_webdriver()
+        heatmap_data = []
+
+        unique_players = lineups_df[[
+            "country", "tournament", "season", "week", "game_id", "team", "player_id", "player_name"
+        ]].drop_duplicates()
+
+        if unique_players.empty:
+            raise ValueError("No unique players found in lineup data.")
+
+        def process_heatmap_points(data, row):
+            if not data:
+                return []
+            return [
+                {
+                    "country": row["country"],
+                    "tournament": row["tournament"],
+                    "season": row["season"],
+                    "week": row["week"],
+                    "game_id": row["game_id"],
+                    "team": row["team"],
+                    "player_id": row["player_id"],
+                    "player_name": row["player_name"],
+                    "x": point.get("x"),
+                    "y": point.get("y")
+                }
+                for point in data if isinstance(point, dict) and "x" in point and "y" in point
+            ]
+
+        for _, row in unique_players.iterrows():
+            try:
+                url = f"{API_BASE_URLS[data_source]}/api/v1/event/{row['game_id']}/player/{row['player_id']}/heatmap"
+                webdriver_instance.get(url)
+
+                pre_tag = WebDriverWait(webdriver_instance, element_load_timeout).until(
+                    EC.visibility_of_element_located((By.TAG_NAME, "pre"))
+                )
+
+                heatmap_json = json.loads(pre_tag.text).get("heatmap", [])
+                player_heatmap_data = process_heatmap_points(heatmap_json, row)
+                heatmap_data.extend(player_heatmap_data)
+
+            except (TimeoutException, json.JSONDecodeError, WebDriverException) as e:
+                raise RuntimeError(f"Error fetching heatmap data for player {row['player_name']} (ID: {row['player_id']}): {str(e)}")
+
+        heatmap_df = pd.DataFrame(heatmap_data)
+
+        if heatmap_df.empty:
+            raise ValueError("No heatmap data extracted.")
+
+        if enable_json_export or enable_excel_export:
+            first_row = heatmap_df.iloc[0]
+
+            if enable_json_export:
+                save_json(
+                    data=heatmap_df,
+                    country=first_row["country"],
+                    tournament=first_row["tournament"],
+                    season=first_row["season"],
+                    week_number=first_row["week"]
+                )
+
+            if enable_excel_export:
+                save_excel(
+                    data=heatmap_df,
+                    country=first_row["country"],
+                    tournament=first_row["tournament"],
+                    season=first_row["season"],
+                    week_number=first_row["week"]
+                )
+
+        return heatmap_df
+
+    except WebDriverException as e:
+        raise RuntimeError(f"Selenium WebDriver error: {str(e)}")
+    except Exception as e:
+        raise RuntimeError(f"Unexpected error while fetching heatmap data: {e.__class__.__name__} - {e}")
+
+    finally:
+        if webdriver_instance:
+            webdriver_instance.quit()

datafc-1.0.0/datafc/sofascore/fetch_goal_networks_data.py

@@ -0,0 +1,151 @@
+import json
+import pandas as pd
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import TimeoutException, WebDriverException
+from datafc.utils._setup_webdriver import setup_webdriver
+from datafc.utils._save_files import save_json, save_excel
+from datafc.utils._config import ALLOWED_SOURCES, API_BASE_URLS
+
+def goal_networks_data(
+    match_df: pd.DataFrame,
+    data_source: str = "sofascore",
+    element_load_timeout: int = 10,
+    enable_json_export: bool = False,
+    enable_excel_export: bool = False
+) -> pd.DataFrame:
+    """
+    Fetches goal network data for each match in the provided match dataset.
+
+    Args:
+        match_df (pd.DataFrame): A DataFrame containing match metadata,
+            which should be generated by the `match_data` function.
+        data_source (str): The data source ('sofavpn' or 'sofascore'). Defaults to 'sofascore'.
+        element_load_timeout (int): The maximum time (in seconds) to wait for the API response. Defaults to 10.
+        enable_json_export (bool): If `True`, exports the fetched goal network data as a JSON file. Defaults to `False`.
+        enable_excel_export (bool): If `True`, exports the fetched goal network data as an Excel file. Defaults to `False`.
+    """
+    if data_source not in ALLOWED_SOURCES:
+        raise ValueError(f"Invalid data source: {data_source}. Must be one of {ALLOWED_SOURCES}")
+
+    if match_df is None or match_df.empty:
+        raise ValueError("Match dataframe must be provided and cannot be empty.")
+
+    webdriver_instance = None
+    try:
+        webdriver_instance = setup_webdriver()
+        goal_networks_list = []
+
+        for _, row in match_df.iterrows():
+            country, tournament, season, week, game_id = row[
+                ["country", "tournament", "season", "week", "game_id"]
+            ]
+
+            api_request_url = f"{API_BASE_URLS[data_source]}/api/v1/event/{game_id}/incidents"
+            webdriver_instance.get(api_request_url)
+
+            try:
+                response_element = WebDriverWait(webdriver_instance, element_load_timeout).until(
+                    EC.visibility_of_element_located((By.TAG_NAME, "pre"))
+                )
+                json_data = json.loads(response_element.text).get("incidents", [])
+
+                if isinstance(json_data, list):
+                    goal_networks_df = pd.DataFrame(json_data)
+                    goal_networks_df["country"] = country
+                    goal_networks_df["tournament"] = tournament
+                    goal_networks_df["season"] = season
+                    goal_networks_df["week"] = week
+                    goal_networks_df["game_id"] = game_id
+                    goal_networks_list.append(goal_networks_df)
+
+            except TimeoutException:
+                raise RuntimeError(f"Timeout while fetching goal network data for game {game_id}.")
+            except json.JSONDecodeError:
+                raise RuntimeError(f"Failed to decode goal network data for game {game_id}.")
+
+        goal_networks_df = pd.concat(goal_networks_list, ignore_index=True)
+
+        if goal_networks_df.empty:
+            raise ValueError("No goal network data found for the specified parameters.")
+
+        actions_list = []
+        for _, row in goal_networks_df[["id", "footballPassingNetworkAction", "country", "tournament", "season", "week", "game_id"]].iterrows():
+            if isinstance(row["footballPassingNetworkAction"], list):
+                for event in row["footballPassingNetworkAction"]:
+                    event["id"] = row["id"]
+                    event.update({
+                        "country": row["country"],
+                        "tournament": row["tournament"],
+                        "season": row["season"],
+                        "week": row["week"],
+                        "game_id": row["game_id"]
+                    })
+                    actions_list.append(event)
+
+        actions_list_df = pd.DataFrame(actions_list)
+
+        if actions_list_df.empty:
+            raise ValueError("No passing network data found.")
+
+        def extract_coordinates(coords, coord_type):
+            return coords.get(coord_type) if isinstance(coords, dict) else None
+
+        actions_list_df = actions_list_df.assign(
+            player_name=actions_list_df["player"].apply(lambda p: p.get("name") if isinstance(p, dict) else None),
+            player_id=actions_list_df["player"].apply(lambda p: p.get("id") if isinstance(p, dict) else None),
+            event_type=actions_list_df["eventType"],
+            player_x=actions_list_df["playerCoordinates"].apply(lambda coord: extract_coordinates(coord, "x")),
+            player_y=actions_list_df["playerCoordinates"].apply(lambda coord: extract_coordinates(coord, "y")),
+            pass_end_x=actions_list_df["passEndCoordinates"].apply(lambda coord: extract_coordinates(coord, "x")),
+            pass_end_y=actions_list_df["passEndCoordinates"].apply(lambda coord: extract_coordinates(coord, "y")),
+            id=actions_list_df["id"],
+            is_assist=actions_list_df["isAssist"],
+            goalkeeper_x=actions_list_df["gkCoordinates"].apply(lambda coord: extract_coordinates(coord, "x")),
+            goalkeeper_y=actions_list_df["gkCoordinates"].apply(lambda coord: extract_coordinates(coord, "y")),
+            goal_shot_x=actions_list_df["goalShotCoordinates"].apply(lambda coord: extract_coordinates(coord, "x")),
+            goal_shot_y=actions_list_df["goalShotCoordinates"].apply(lambda coord: extract_coordinates(coord, "y")),
+            goal_mouth_x=actions_list_df["goalMouthCoordinates"].apply(lambda coord: extract_coordinates(coord, "x")),
+            goal_mouth_y=actions_list_df["goalMouthCoordinates"].apply(lambda coord: extract_coordinates(coord, "y")),
+            goalkeeper_name=actions_list_df["goalkeeper"].apply(lambda gk: gk.get("name") if isinstance(gk, dict) else None),
+            goalkeeper_id=actions_list_df["goalkeeper"].apply(lambda gk: gk.get("id") if isinstance(gk, dict) else None)
+        )[
+            [
+                "country", "tournament", "season", "week", "game_id", "player_name", "player_id", "event_type",
+                "player_x", "player_y", "pass_end_x", "pass_end_y", "is_assist", "id", "goalkeeper_x", "goalkeeper_y",
+                "goal_shot_x", "goal_shot_y", "goal_mouth_x", "goal_mouth_y", "goalkeeper_name", "goalkeeper_id"
+            ]
+        ]
+
+        if enable_json_export or enable_excel_export:
+            first_row = actions_list_df.iloc[0]
+
+            if enable_json_export:
+                save_json(
+                    data=actions_list_df,
+                    country=first_row["country"],
+                    tournament=first_row["tournament"],
+                    season=first_row["season"],
+                    week_number=first_row["week"]
+                )
+
+            if enable_excel_export:
+                save_excel(
+                    data=actions_list_df,
+                    country=first_row["country"],
+                    tournament=first_row["tournament"],
+                    season=first_row["season"],
+                    week_number=first_row["week"]
+                )
+
+        return actions_list_df
+
+    except WebDriverException as e:
+        raise RuntimeError(f"Selenium WebDriver error: {str(e)}")
+    except Exception as e:
+        raise RuntimeError(f"Unexpected error while fetching goal network data: {e.__class__.__name__} - {e}")
+
+    finally:
+        if webdriver_instance:
+            webdriver_instance.quit()

datafc-1.0.0/datafc/sofascore/fetch_lineups_data.py

@@ -0,0 +1,146 @@
+import json
+import pandas as pd
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import TimeoutException, WebDriverException
+from datafc.utils._setup_webdriver import setup_webdriver
+from datafc.utils._save_files import save_json, save_excel
+from datafc.utils._config import ALLOWED_SOURCES, API_BASE_URLS
+
+def lineups_data(
+    match_df: pd.DataFrame,
+    data_source: str = "sofascore",
+    element_load_timeout: int = 10,
+    enable_json_export: bool = False,
+    enable_excel_export: bool = False
+) -> pd.DataFrame:
+    """
+    Fetches lineup data for each match in the provided match dataset.
+
+    Args:
+        match_df (pd.DataFrame): A DataFrame containing match metadata,
+            which should be generated by the `match_data` function.
+        data_source (str): The data source ('sofavpn' or 'sofascore'). Defaults to 'sofascore'.
+        element_load_timeout (int): The maximum time (in seconds) to wait for the API response. Defaults to 10.
+        enable_json_export (bool): If `True`, saves the fetched data as a JSON file. Defaults to `False`.
+        enable_excel_export (bool): If `True`, saves the fetched data as an Excel file. Defaults to `False`.
+    """
+    if data_source not in ALLOWED_SOURCES:
+        raise ValueError(f"Invalid data source: {data_source}. Must be one of {ALLOWED_SOURCES}")
+
+    if match_df is None or match_df.empty:
+        raise ValueError("Match dataframe must be provided and cannot be empty.")
+
+    webdriver_instance = None
+    try:
+        webdriver_instance = setup_webdriver()
+        lineups_data_df = pd.DataFrame()
+
+        def process_player_statistics(lineups_data, team_key):
+            extracted_data = []
+            for _, row in lineups_data.iterrows():
+                for player in row[team_key]:
+                    for stat_name, stat_value in player.get("statistics", {}).items():
+                        extracted_data.append({
+                            "country": row["country"],
+                            "tournament": row["tournament"],
+                            "season": row["season"],
+                            "week": row["week"],
+                            "game_id": row["game_id"],
+                            "team": team_key,
+                            "player_name": player["player"]["name"],
+                            "player_id": player["player"]["id"],
+                            "stat_name": stat_name,
+                            "stat_value": stat_value
+                        })
+            return extracted_data
+
+        for _, row in match_df.iterrows():
+            country = row["country"]
+            tournament = row["tournament"]
+            game_id = row["game_id"]
+            season = row["season"]
+            week = row["week"]
+
+            url = f"{API_BASE_URLS[data_source]}/api/v1/event/{game_id}/lineups"
+            webdriver_instance.get(url)
+
+            try:
+                pre_tag = WebDriverWait(webdriver_instance, element_load_timeout).until(
+                    EC.visibility_of_element_located((By.TAG_NAME, "pre"))
+                )
+
+                lineups_json = json.loads(pre_tag.text)
+                lineups_data = pd.DataFrame.from_dict(lineups_json)
+                lineups_data["country"] = country
+                lineups_data["tournament"] = tournament
+                lineups_data["season"] = season
+                lineups_data["week"] = week
+                lineups_data["game_id"] = game_id
+
+                lineups_data = lineups_data[lineups_data.index == "players"]
+                lineups_data_df = pd.concat([lineups_data_df, lineups_data], ignore_index=True)
+
+            except TimeoutException:
+                raise RuntimeError(f"Timeout while fetching lineup data for game_id {game_id}.")
+            except json.JSONDecodeError:
+                raise RuntimeError(f"Failed to decode lineup data for game_id {game_id}.")
+            except WebDriverException as e:
+                raise RuntimeError(f"Selenium WebDriver error while fetching lineup data for game_id {game_id}: {str(e)}")
+            except Exception as e:
+                raise RuntimeError(f"Unexpected error while fetching lineup data for game_id {game_id}: {e.__class__.__name__} - {e}")
+
+        if lineups_data_df.empty:
+            raise ValueError("No lineup data found for the specified parameters.")
+
+        extracted_lineups_data = []
+        for team_key in ["home", "away"]:
+            extracted_lineups_data.extend(process_player_statistics(lineups_data_df, team_key))
+
+        extracted_lineups_data_df = pd.DataFrame(extracted_lineups_data)
+
+        def extract_original_rating(rating_versions):
+            if isinstance(rating_versions, dict) and "original" in rating_versions:
+                return rating_versions["original"]
+            return rating_versions
+
+        extracted_lineups_data_df["stat_value"] = extracted_lineups_data_df.apply(
+            lambda row: extract_original_rating(row["stat_value"]) if row["stat_name"] == "ratingVersions" else row["stat_value"],
+            axis=1
+        )
+
+        if extracted_lineups_data_df.empty:
+            raise ValueError("No extracted lineup data available.")
+
+        if enable_json_export or enable_excel_export:
+            first_row = extracted_lineups_data_df.iloc[0]
+
+            if enable_json_export:
+                save_json(
+                    data=extracted_lineups_data_df,
+                    country=first_row["country"],
+                    tournament=first_row["tournament"],
+                    season=first_row["season"],
+                    week_number=first_row["week"]
+                )
+
+            if enable_excel_export:
+                save_excel(
+                    data=extracted_lineups_data_df,
+                    country=first_row["country"],
+                    tournament=first_row["tournament"],
+                    season=first_row["season"],
+                    week_number=first_row["week"]
+                )
+
+        return extracted_lineups_data_df
+
+    except WebDriverException as e:
+        raise RuntimeError(f"Selenium WebDriver error: {str(e)}")
+    except Exception as e:
+        raise RuntimeError(f"Unexpected error while fetching lineup data: {e.__class__.__name__} - {e}")
+
+    finally:
+        if webdriver_instance:
+            webdriver_instance.quit()

datafc-1.0.0/datafc/sofascore/fetch_match_data.py

@@ -0,0 +1,102 @@
+import json
+import pandas as pd
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import TimeoutException, WebDriverException
+from datafc.utils._setup_webdriver import setup_webdriver
+from datafc.utils._save_files import save_json, save_excel
+from datafc.utils._config import ALLOWED_SOURCES, API_BASE_URLS
+
+def match_data(
+    tournament_id: int,
+    season_id: int,
+    week_number: int,
+    data_source: str = "sofascore",
+    element_load_timeout: int = 10,
+    enable_json_export: bool = False,
+    enable_excel_export: bool = False
+) -> pd.DataFrame:
+    """
+    Fetches match data for a specified tournament, season, and matchweek.
+
+    Args:
+        tournament_id (int): The unique identifier for the tournament.
+        season_id (int): The unique identifier for the season.
+        week_number (int): The matchweek number within the season.
+        data_source (str): The data source ('sofavpn' or 'sofascore'). Defaults to 'sofascore'.
+        element_load_timeout (int): The maximum time (in seconds) to wait for the API response. Defaults to 10.
+        enable_json_export (bool): If `True`, exports the fetched data as a JSON file. Defaults to `False`.
+        enable_excel_export (bool): If `True`, exports the fetched data as an Excel file. Defaults to `False`.
+    """
+    if data_source not in ALLOWED_SOURCES:
+        raise ValueError(f"Invalid data source: {data_source}. Must be one of {ALLOWED_SOURCES}")
+
+    api_request_url = f"{API_BASE_URLS[data_source]}/api/v1/unique-tournament/{tournament_id}/season/{season_id}/events/round/{week_number}"
+
+    try:
+        webdriver_instance = setup_webdriver()
+        webdriver_instance.get(api_request_url)
+
+        response_element = WebDriverWait(webdriver_instance, element_load_timeout).until(
+            EC.visibility_of_element_located((By.TAG_NAME, "pre"))
+        )
+        response_text = response_element.text.strip()
+        if not response_text:
+            raise RuntimeError("API response is empty.")
+
+        api_response_data = json.loads(response_text)
+        if "events" not in api_response_data or not isinstance(api_response_data["events"], list):
+            raise ValueError("Invalid API response format: 'events' key is missing or not a list.")
+
+        events_df = pd.DataFrame(api_response_data.get("events", []))
+        if events_df.empty:
+            raise ValueError("No match data found for the specified parameters.")
+
+        match_data_df = pd.DataFrame({
+            "country": events_df["tournament"].apply(lambda x: x.get("category", {}).get("name", "")),
+            "tournament": events_df["tournament"].apply(lambda x: x.get("name", "")),
+            "season": events_df["season"].apply(lambda x: x.get("year", "")),
+            "week": events_df["roundInfo"].apply(lambda x: x.get("round", "")),
+            "game_id": events_df["id"],
+            "home_team": events_df["homeTeam"].apply(lambda x: x.get("name", "")),
+            "away_team": events_df["awayTeam"].apply(lambda x: x.get("name", "")),
+            "start_timestamp": events_df["startTimestamp"],
+            "status": events_df["status"].apply(lambda x: x.get("description", ""))
+        })
+
+        if enable_json_export or enable_excel_export:
+            first_row = match_data_df.iloc[0]
+
+            if enable_json_export:
+                save_json(
+                    data=match_data_df,
+                    country=first_row["country"],
+                    tournament=first_row["tournament"],
+                    season=first_row["season"],
+                    week_number=first_row["week"]
+                )
+
+            if enable_excel_export:
+                save_excel(
+                    data=match_data_df,
+                    country=first_row["country"],
+                    tournament=first_row["tournament"],
+                    season=first_row["season"],
+                    week_number=first_row["week"]
+                )
+
+        return match_data_df
+
+    except TimeoutException:
+        raise RuntimeError("Timeout occurred while waiting for the page or API response.")
+    except WebDriverException as e:
+        raise RuntimeError(f"Selenium WebDriver error: {str(e)}")
+    except json.JSONDecodeError:
+        raise RuntimeError("Failed to decode API response as JSON.")
+    except Exception as e:
+        raise RuntimeError(f"Unexpected error while fetching match data: {e.__class__.__name__} - {e}")
+
+    finally:
+        if webdriver_instance:
+            webdriver_instance.quit()