pyfootix 0.1.0__py3-none-any.whl

footix/data_io/base_scrapper.py

@@ -0,0 +1,58 @@
+import pathlib
+
+import pandas as pd
+import requests
+
+from footix.data_io.utils_scrapper import MAPPING_COMPETITIONS
+
+
+class Scraper:
+    base_url: str = ""
+    scraper_name: str | None = None
+
+    def __init__(self, path: str, mapping_teams: dict[str, str] | None) -> None:
+        self.mapping_teams = mapping_teams
+        self.headers = {
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/124.0.0.0 Safari/537.36"
+            )
+        }
+        self.path = self.manage_path(path=path)
+
+    def _check_competitions(self, competition_name: str):
+        list_comp = self.competitions()
+        if competition_name not in list_comp:
+            raise ValueError(f"{competition_name} not available for scraper {self.scraper_name}")
+
+    @classmethod
+    def competitions(cls) -> list[str]:
+        if cls.scraper_name is None:
+            raise AttributeError("Scraper name undefined")
+        list_compet = []
+        for compet, info in MAPPING_COMPETITIONS.items():
+            if cls.scraper_name in info.keys():
+                list_compet.append(compet)
+        return list_compet
+
+    def replace_name_team(self, df: pd.DataFrame, columns: list[str]) -> pd.DataFrame:
+        tmp_df = df.copy()
+        if self.mapping_teams is not None:
+            for column in columns:
+                tmp_df[column] = tmp_df[column].replace(self.mapping_teams)
+        return tmp_df
+
+    def get(self, url: str) -> str:
+        return requests.get(url, headers=self.headers).content.decode("utf-8-sig")
+
+    @staticmethod
+    def manage_path(path: str) -> pathlib.Path:
+        tmp_pth = pathlib.Path(path)
+        if tmp_pth.is_file():
+            raise ValueError("Path should be a directory")
+        if tmp_pth.exists():
+            return tmp_pth
+        else:
+            tmp_pth.mkdir(parents=True, exist_ok=True)
+        return tmp_pth

footix/data_io/data_reader.py

@@ -0,0 +1,85 @@
+import dataclasses
+from typing import Iterator, Protocol
+
+import pandas as pd
+
+
+class DataProtocol(Protocol):
+    def __len__(self) -> int:
+        ...
+
+
+@dataclasses.dataclass
+class MatchupResult:
+    """A dataclass representing the result of a football match.
+
+    Attributes:
+        home_team (str): The name of the home team.
+        away_team (str): The name of the away team.
+        result (str): The final result of the match:
+            ('H' for Home Win, 'A' for Away Win, 'D' for Draw).
+        away_goals (float): The number of goals scored by the away team.
+        home_goals (float): The number of goals scored by the home team.
+
+    """
+
+    home_team: str
+    away_team: str
+    result: str
+    away_goals: float
+    home_goals: float
+
+    @staticmethod
+    def from_dict(dict_row: dict) -> "MatchupResult":
+        """Factory method to create a MatchupResult object from a dictionary row.
+
+        Parameters:
+            dict_row (dict): A dictionary containing the match results with keys:
+            - 'HomeTeam': The name of the home team.
+            - 'AwayTeam': The name of the away team.
+            - 'FTR': The final result
+                ('H' for Home Win, 'A' for Away Win, 'D' for Draw).
+            - 'FTAG': The number of goals scored by the away team.
+            - 'FTHG': The number of goals scored by the home team.
+        Returns:
+            MatchupResult: An instance of the MatchupResult class populated with data
+            from the dictionary row.
+
+        """
+        return MatchupResult(
+            home_team=dict_row["HomeTeam"],
+            away_team=dict_row["AwayTeam"],
+            result=dict_row["FTR"],
+            away_goals=dict_row["FTAG"],
+            home_goals=dict_row["FTHG"],
+        )
+
+
+class EloDataReader(DataProtocol):
+    def __init__(self, df_data: pd.DataFrame):
+        self.df_data = self._process_df(df_data)
+        # Better performances for iteration over rows
+        self.data = self.df_data.to_dict(orient="index")
+
+    # TODO: Add a sanity check to verify if columns are presents in the df
+    def _process_df(self, df_data: pd.DataFrame) -> pd.DataFrame:
+        df = df_data.copy().reset_index(drop=True)
+        df = df[["Date", "HomeTeam", "AwayTeam", "FTHG", "FTAG", "FTR"]]
+        df["Date"] = pd.to_datetime(df["Date"], dayfirst=True)
+        df = df.sort_values(by="Date", ascending=True)
+        return df
+
+    def __len__(self) -> int:
+        return len(self.df_data)
+
+    def unique_teams(self) -> list[str]:
+        list_unique_team = list(
+            set(self.df_data["HomeTeam"].unique()).intersection(self.df_data["AwayTeam"].unique())
+        )
+        return sorted(list_unique_team)
+
+    def __iter__(self) -> Iterator[MatchupResult]:
+        return iter(self.__getitem__(idx) for idx in range(len(self)))
+
+    def __getitem__(self, idx: int) -> MatchupResult:
+        return MatchupResult.from_dict(self.data[idx])

footix/data_io/footballdata.py

@@ -0,0 +1,126 @@
+import io
+
+import pandas as pd
+
+import footix.data_io.utils_scrapper as utils_scrapper
+from footix.data_io.base_scrapper import Scraper
+
+
+class ScrapFootballData(Scraper):
+    """Scraper for downloading and processing football match data from football-data.co.uk.
+
+    This class handles the retrieval, local storage, and preprocessing of football match data
+    for a given competition and season. It supports automatic downloading, file management,
+    column sanitization, and team name mapping.
+
+    Args:
+        competition (str): The competition code (e.g., 'E0' for Premier League).
+        season (str): The season string (e.g., '2020/2021', '2020-2021', or '2021').
+        path (str): Directory path to store the downloaded CSV files.
+        force_reload (bool, optional): If True, forces re-download of data even if file exists.
+        mapping_teams (dict[str, str] | None, optional): Optional mapping for team name
+        normalization.
+
+    Attributes:
+        base_url (str): Base URL for football-data.co.uk.
+        scraper_name (str): Name identifier for the scraper.
+        competition (str): Competition code.
+        season (str): Processed season string.
+        path (Path): Path object for data storage.
+        force_reload (bool): Whether to force data reload.
+        infered_url (str): Constructed URL for the CSV file.
+        df (pd.DataFrame): Loaded and processed match data.
+
+    Methods:
+        download(): Downloads and saves the competition data as a CSV file.
+        load() -> pd.DataFrame: Loads the data from file or downloads if not present.
+        sanitize_columns(): Converts DataFrame columns to snake_case.
+        get_fixtures() -> pd.DataFrame: Returns the processed match data.
+
+    """
+
+    base_url: str = "https://www.football-data.co.uk/mmz4281/"
+    scraper_name = "footballdata"
+
+    def __init__(
+        self,
+        competition: str,
+        season: str,
+        path: str,
+        force_reload: bool = False,
+        mapping_teams: dict[str, str] | None = None,
+    ) -> None:
+        super().__init__(path=path, mapping_teams=mapping_teams)
+        self._check_competitions(competition_name=competition)
+        self.competition = competition
+        slug = utils_scrapper.MAPPING_COMPETITIONS[self.competition]["footballdata"]["slug"]
+        self.season = _process_season(season)
+        self.path = self.manage_path(path)
+        self.force_reload = force_reload
+        self.infered_url = self.base_url + self.season + "/" + slug + ".csv"
+        self.df = self.load()
+        self.sanitize_columns()
+        self.df = utils_scrapper.add_mathc_id(self.df)
+
+    def download(self):
+        response = self.get(self.infered_url)
+        df = (
+            pd.read_csv(io.StringIO(response), encoding="utf-8")
+            .sort_index()
+            .pipe(self.replace_name_team, columns=["home_team", "away_team"])
+        )
+        df.to_csv(
+            self.path / (self.competition + "_" + self.season + ".csv"),
+            index=False,
+            encoding="utf-8",
+        )
+
+    def load(self) -> pd.DataFrame:
+        if self._check_if_file_exist() and not self.force_reload:
+            df = pd.read_csv(self.path / (self.competition + "_" + self.season + ".csv"))
+        else:
+            self.download()
+            df = pd.read_csv(self.path / (self.competition + "_" + self.season + ".csv"))
+        return df
+
+    def sanitize_columns(self):
+        self.df.columns = [utils_scrapper.to_snake_case(x) for x in self.df.columns]
+
+    def get_fixtures(self) -> pd.DataFrame:
+        return self.df
+
+    def _check_if_file_exist(self) -> bool:
+        name = self.competition + "_" + self.season + ".csv"
+        expected_file = self.path / name
+        if not expected_file.is_file():
+            return False
+        return True
+
+
+def _process_season(season: str) -> str:
+    """Process a season string to extract a standardized format.
+
+    Args:
+        season (str): A string representing a football season in the format
+            'YYYY/YYYY' or 'YYYY-YYYY'. For example: '2020/2021' or '2020-2021'
+
+    Raises:
+        ValueError: if the season string cannot be split into exactly two years.
+
+    Returns:
+        str: the formatted season string.
+
+    """
+    if len(season) == 4 and season.isdigit():
+        if int(season[-2:]) - int(season[:2]) != 1:
+            raise ValueError("Years must be consecutive")
+        return season
+    # Remove any whitespace and split on common separators
+    clean_season = season.replace(" ", "-").replace("/", "-").split("-")
+    # Extract last 2 digits from each year
+    year1 = clean_season[0][-2:]
+    year2 = clean_season[-1][-2:]
+    # Check if the years are consecutive
+    if int(year2) - int(year1) != 1:
+        raise ValueError("Years must be consecutive")
+    return year1 + year2

footix/data_io/understat.py

@@ -0,0 +1,190 @@
+import json
+import re
+from datetime import datetime
+from functools import lru_cache
+from typing import Any
+from urllib.parse import urljoin
+
+import numpy as np
+import pandas as pd
+from lxml import html
+
+import footix.data_io.utils_scrapper as utils_scrapper
+from footix.data_io.base_scrapper import Scraper
+
+
+class ShotDataNotFound(RuntimeError):
+    """Raised when the expected shotsData <script> block is not present."""
+
+
+class FixtureDataNotFound(RuntimeError):
+    """Raised when the fixture data are not present."""
+
+
+class ScrapUnderstat(Scraper):
+    """
+    Scraper for downloading and processing football match data from understat.com.
+    This class function is heavily inspired/copied from its counterpart from penalty blog:
+    https://github.com/martineastwood/penaltyblog
+
+    This class retrieves, parses, and processes football match data for a given competition
+    and season from Understat. It extracts fixture details, expected goals (xG), forecasts,
+    and normalizes team names. The data is returned as a processed pandas DataFrame.
+
+    Args:
+        competition (str): The competition code (e.g., 'EPL' for Premier League).
+        season (str): The season string (e.g., '2020/2021', '2020-2021', or '2021').
+        path (str): Directory path for any required file operations.
+        force_reload (bool, optional): If True, forces re-download or reprocessing of data.
+        mapping_teams (dict[str, str] | None, optional): Optional mapping for team name
+        normalization.
+
+    Attributes:
+        base_url (str): Base URL for understat.com.
+        scraper_name (str): Name identifier for the scraper.
+        season (str): Processed season string.
+        force_reload (bool): Whether to force data reload.
+        slug (str): Slug for the competition used in URL construction.
+
+    Methods:
+        sanitize_columns(df): Converts DataFrame columns to snake_case.
+        get_fixtures() -> pd.DataFrame: Downloads, parses, and returns processed match data.
+        _process_season(season: str) -> str: Processes the season string for URL usage.
+    """
+
+    base_url: str = "https://understat.com/"
+    scraper_name = "understat"
+
+    def __init__(
+        self,
+        competition: str,
+        season: str,
+        path: str,
+        force_reload: bool = False,
+        mapping_teams: dict[str, str] | None = None,
+    ):
+        self._check_competitions(competition_name=competition)
+        super().__init__(path=path, mapping_teams=mapping_teams)
+        self.season = self._process_season(season)
+        self.force_reload = force_reload
+        self.slug = utils_scrapper.MAPPING_COMPETITIONS[competition]["understat"]["slug"]
+        self.competition = competition
+
+    @staticmethod
+    def sanitize_columns(df: pd.DataFrame):
+        df.columns = [utils_scrapper.to_snake_case(x) for x in df.columns]
+
+    @lru_cache(maxsize=256)
+    def get_fixtures(self):
+        implied_url = urljoin(self.base_url, f"league/{self.slug}/{self.season}")
+        content = self.get(implied_url)
+        tree = html.fromstring(content)
+        events = None
+        for s in tree.cssselect("script"):
+            if "datesData" in s.text:
+                script = s.text
+                script = " ".join(script.split())
+                script = str(script.encode(), "unicode-escape")
+                script = re.match(r"var datesData = JSON\.parse\('(?P<json>.*?)'\)", script)
+                if script is not None:
+                    script = script.group("json")
+                events = json.loads(script)
+                break
+
+        if events is None:
+            raise FixtureDataNotFound
+
+        fixtures = list()
+        for e in events:
+            if not e["isResult"]:
+                continue
+
+            tmp: dict[str, Any] = dict()
+            tmp["understat_id"] = str(e["id"])
+            tmp["datetime"] = e["datetime"]
+            tmp["home_team"] = e["h"]["title"]
+            tmp["away_team"] = e["a"]["title"]
+            tmp["fthg"] = int(e["goals"]["h"])
+            tmp["ftag"] = int(e["goals"]["a"])
+            tmp["fthxg"] = float(e["xG"]["h"])
+            tmp["ftaxg"] = float(e["xG"]["a"])
+            tmp["forecast_w"] = float(e["forecast"]["w"])
+            tmp["forecast_d"] = float(e["forecast"]["d"])
+            tmp["forecast_l"] = float(e["forecast"]["l"])
+            fixtures.append(tmp)
+
+        df = (
+            pd.DataFrame(fixtures)
+            .pipe(self.replace_name_team, columns=["home_team", "away_team"])
+            .sort_index()
+        )
+
+        def _get_date(date: str) -> str:
+            dt = datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
+            return dt.strftime("%d/%m/%Y")
+
+        def get_ftr(row) -> str:
+            fthg = row["fthg"]
+            ftag = row["ftag"]
+            if fthg > ftag:
+                return "H"
+            if fthg == ftag:
+                return "D"
+            return "A"
+
+        self.sanitize_columns(df)
+        df["ftr"] = df.apply(get_ftr, axis=1)
+        df["date"] = df["datetime"].apply(_get_date)
+        df = utils_scrapper.add_mathc_id(df)
+        return df
+
+    def _process_season(self, season: str) -> str:
+        clean_season = season.replace(" ", "-").replace("/", "-").split("-")
+        return clean_season[0]
+
+    @lru_cache(maxsize=256)
+    def get_shots(self, understat_id: str) -> pd.DataFrame:
+        url = urljoin(self.base_url, f"match/{understat_id}")
+        content = self.get(url)
+        tree = html.fromstring(content)
+        events = None
+
+        for s in tree.cssselect("script"):
+            if "shotsData" in s.text:
+                script = s.text
+                script = " ".join(script.split())
+                script = str(script.encode(), "unicode-escape")
+                script = re.match(r"var shotsData = JSON\.parse\('(?P<json>.*?)'\)", script)
+                if script is not None:
+                    script = script.group("json")
+                events = json.loads(script)
+                break
+
+        if events is None:
+            raise ShotDataNotFound
+
+        shots = list()
+        shots.extend(events["h"])
+        shots.extend(events["a"])
+
+        col_renames = {
+            "h_team": "home_team",
+            "a_team": "away_team",
+            "h_goals": "goals_home",
+            "a_goals": "goals_away",
+            "match_id": "understat_id",
+        }
+
+        df = (
+            pd.DataFrame(shots)
+            .rename(columns=col_renames)
+            .assign(season=self.season)
+            .assign(competition=self.competition)
+            .assign(date=lambda x: pd.to_datetime(x["date"]).dt.strftime("%d-%m-%Y"))
+            .pipe(self.replace_name_team, columns=["home_team", "away_team"])
+            .sort_index()
+        )
+        df["h_a"] = np.where(df["h_a"] == "h", df["home_team"], df["away_team"])
+        df = utils_scrapper.add_mathc_id(df)
+        self.sanitize_columns(df)
+        return df

footix/data_io/utils_scrapper.py

@@ -0,0 +1,65 @@
+# Mapping of the different competitions to their respective slugs
+import re
+from typing import Any
+
+import pandas as pd
+
+from footix.utils.decorators import verify_required_column
+
+MAPPING_COMPETITIONS: dict[str, dict[str, Any]] = {
+    "FRA Ligue 1": {"footballdata": {"slug": "F1"}, "understat": {"slug": "Ligue_1"}},
+    "FRA Ligue 2": {"footballdata": {"slug": "F2"}},
+    "ENG Premier League": {"footballdata": {"slug": "E0"}, "understat": {"slug": "EPL"}},
+    "ENG Championship": {"footballdata": {"slug": "E1"}},
+    "DEU Bundesliga 1": {"footballdata": {"slug": "D1"}, "understat": {"slug": "Bundesliga"}},
+    "DEU Bundesliga 2": {"footballdata": {"slug": "D2"}},
+    "ITA Serie A": {"footballdata": {"slug": "I1"}, "understat": {"slug": "Serie_A"}},
+    "ITA Serie B": {"footballdata": {"slug": "I2"}},
+    "SPA La Liga": {"footballdata": {"slug": "SP1"}, "understat": {"slug": "La_Liga"}},
+    "SPA La Liga 2": {"footballdata": {"slug": "SP2"}},
+}
+
+
+def check_competition_exists(competition: str) -> bool:
+    """Check if the competition exists in the MAPPING_COMPETITIONS dictionary.
+
+    Args:
+        competition (str): The name of the competition to check.
+
+    Returns:
+        bool: True if the competition exists, False otherwise.
+
+    """
+    return competition in MAPPING_COMPETITIONS
+
+
+def process_string(input_string):
+    lower_string = input_string.lower()
+    no_space_string = lower_string.replace(" ", "")
+    return no_space_string
+
+
+def to_snake_case(name: str) -> str:
+    """
+    Convert the string name into a snake case string.
+    Shamelessly copied from:
+    https://stackoverflow.com/questions/1175208/
+    elegant-python-function-to-convert-camelcase-to-snake-case
+
+    Args:
+        name (str): the name to convert
+
+    Returns:
+        str: the name in snake case
+    """
+    name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name)
+    name = re.sub("__([A-Z])", r"_\1", name)
+    name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", name)
+    return name.lower()
+
+
+@verify_required_column(["home_team", "away_team", "date"])
+def add_mathc_id(df: pd.DataFrame) -> pd.DataFrame:
+    tmp_df = df.copy()
+    tmp_df["match_id"] = tmp_df["home_team"] + " - " + tmp_df["away_team"] + " - " + tmp_df["date"]
+    return tmp_df

footix/implied_odds/__init__.py

@@ -0,0 +1,3 @@
+from .implied import multiplicative, power, shin
+
+__all__ = ["multiplicative", "power", "shin"]

footix/implied_odds/implied.py

@@ -0,0 +1,116 @@
+import operator
+from typing import cast
+
+import numpy as np
+from scipy import optimize
+
+from footix.utils.typing import ArrayLikeF
+
+# Most of those functions are inspired by the awesome package penaltyblog
+# https://github.com/martineastwood/penaltyblog/tree/master
+
+
+def _assert_odds(odds: ArrayLikeF, axis: None | int = None) -> None:
+    if (not isinstance(odds, list)) and (not isinstance(odds, np.ndarray)):
+        raise TypeError("Odds must be a list or an numpy array.")
+    if isinstance(odds, list):
+        odds = np.array(odds)
+    if axis is not None:
+        if odds.shape[axis] != 3:
+            raise ValueError("It is a football package ! You must provide 3 odds.")
+    else:
+        if odds.shape[0] != 3:
+            raise ValueError("It is a football package ! You must provide 3 odds.")
+    if (odds < 1.0).any():
+        raise ValueError("All odds must be greater then 1.")
+
+
+def multiplicative(odds: ArrayLikeF, axis: int = -1) -> tuple[np.ndarray, float | np.ndarray]:
+    """Multiplicative way to normalize the odds. Work for multidimensionnal array.
+
+    Args:
+        odds (list or np.array): list of odds
+        axis (int) : axis where compute the probabilities
+
+    """
+    _assert_odds(odds, axis=axis)
+    if isinstance(odds, list):
+        odds = np.array(odds)
+    if len(odds.shape) > 1:
+        normalization = np.sum(1.0 / odds, axis=axis, keepdims=True)
+    else:
+        normalization = np.sum(1.0 / odds, axis=axis)
+    margin = normalization - 1.0
+    return 1.0 / (normalization * odds), margin
+
+
+def power(odds: ArrayLikeF) -> tuple[np.ndarray, float]:
+    """From penaltyblog package. The power method computes the implied probabilities by solving
+    for the power coefficient that normalizes the inverse of the odds to sum to 1.0.
+
+    Args:
+        odds : (list or np.array): list of odds
+
+    """
+    _assert_odds(odds)
+    if isinstance(odds, list):
+        odds = np.array(odds)
+    inv_odds = 1.0 / odds
+    margin = cast(float, np.sum(inv_odds) - 1.0)
+
+    def _fit(k: float, inv_odds: np.ndarray) -> float:
+        implied = operator.pow(inv_odds, k)
+        return 1 - np.sum(implied)
+
+    res = optimize.ridder(_fit, 0, 100, args=(inv_odds,))
+    normalized = operator.pow(inv_odds, res)
+    return normalized, margin
+
+
+def shin(odds: ArrayLikeF) -> tuple[np.ndarray, float]:
+    """Computes the implied probabilities via Shin's method (1992, 1993).
+
+    Args:
+        odds (list or np.ndarray): An array of size 3 containing the odds for Home victory, draw,
+                                   or Away victory, respectively.
+
+    Returns:
+        tuple:
+            - np.ndarray: The implied probabilities for each outcome.
+            - float: The margin.
+
+    """
+    _assert_odds(odds)
+
+    if isinstance(odds, list):
+        odds = np.array(odds)
+
+    inv_odds = 1.0 / odds
+    margin = cast(float, np.sum(inv_odds) - 1.0)
+
+    def _fit(z_param: float, inv_odds: np.ndarray) -> float:
+        implied = _shin(z_param, inv_odds)
+        return 1.0 - np.sum(implied)
+
+    res = optimize.ridder(_fit, 0, 100, args=(inv_odds,))
+    normalized = _shin(res, inv_odds)
+    return normalized, margin
+
+
+def _shin(z_param: float, inv_odds: np.ndarray) -> np.ndarray:
+    """Computes the implied probabilities using Shin's method.
+
+    Args:
+        z_param (float): The Shin adjustment parameter.
+        inv_odds (np.ndarray): An array of size 3 containing the inverse odds for Home victory,
+                               draw, or Away victory, respectively.
+
+    Returns:
+        np.ndarray: The implied probabilities for each outcome.
+
+    """
+    normalized = np.sum(inv_odds)
+    implied = (
+        np.sqrt(z_param**2 + 4 * (1 - z_param) * inv_odds**2 / normalized) - z_param
+    ) / (2 - 2 * z_param)
+    return implied

footix/metrics/__init__.py

@@ -0,0 +1,3 @@
+from .metrics_function import incertity, rps, zscore
+
+__all__ = ["incertity", "rps", "zscore"]