tinyshift 0.0.2__py3-none-any.whl

tinyshift/base/model.py

@@ -0,0 +1,249 @@
+from ..plot import plot
+import numpy as np
+from scipy.stats import norm
+from typing import Callable, Union, Tuple, Dict
+import pandas as pd
+
+
+class BaseModel:
+    def __init__(
+        self,
+        reference: pd.DataFrame,
+        confidence_level: float,
+        statistic: Callable,
+        n_resamples: int,
+        random_state: int,
+        drift_limit: Union[str, Tuple[float, float]],
+    ):
+        """
+        Initializes the BaseModel class with reference distribution, statistics, and drift limits.
+
+        Parameters:
+        ----------
+        reference : pd.DataFrame
+            Data containing the reference distribution with a "metric" column.
+        confidence_level : float
+            Desired confidence level for statistical calculations (e.g., 0.95).
+        statistic : Callable
+            Function to compute summary statistics (e.g., np.mean).
+        n_resamples : int
+            Number of bootstrap resamples.
+        random_state : int
+            Seed for reproducibility.
+        drift_limit : Union[str, Tuple[float, float]]
+            Method ("deviation" or "mad") or custom limits for drift thresholding.
+        """
+
+        self.statistics = self._statistic_generate(
+            reference,
+            confidence_level,
+            statistic,
+            n_resamples,
+            random_state,
+        )
+        self.plot = plot.Plot(self.statistics, reference)
+        self._drift_limit_generate(self.statistics, reference, drift_limit)
+
+    def _jackknife_acceleration(self, data: np.ndarray, statistic: Callable) -> float:
+        """Calculate the acceleration parameter using jackknife resampling."""
+        n = len(data)
+        jackknife = np.array([statistic(np.delete(data, i)) for i in range(n)])
+        jackknife_mean = jackknife.mean()
+        diffs = jackknife - jackknife_mean
+        acceleration = np.sum(diffs**3) / (6.0 * (np.sum(diffs**2) ** 1.5))
+        return acceleration
+
+    def _bootstrap_statistics(
+        self, data: np.ndarray, statistic: Callable, n_resamples: int
+    ) -> np.ndarray:
+        """Perform bootstrap resampling and calculate statistics."""
+        return np.array(
+            [
+                statistic(np.random.choice(data, size=len(data), replace=True))
+                for _ in range(n_resamples)
+            ]
+        )
+
+    def _bootstrapping_bca(
+        self,
+        data: pd.Series,
+        confidence_level: float,
+        statistic: Callable,
+        n_resamples: int,
+        random_state: int,
+    ) -> Union[float, float]:
+        """
+        Calculates the bias-corrected and accelerated (BCa) bootstrap confidence interval for the given data.
+
+        Parameters:
+        - data (list or numpy array): Sample data.
+        - confidence_level (float): Desired confidence level (e.g., 0.95 for 95%).
+        - statistic (function): Statistical function to apply to the data. Default is np.mean.
+        - n_resamples (int): Number of bootstrap resamples to perform. Default is 1000.
+        - random_state (int): Random seed for reproducibility.
+
+        Returns:
+        - tuple: A tuple containing the lower and upper bounds of the BCa confidence interval.
+        """
+        np.random.seed(random_state)
+        data = np.asarray(data)
+
+        # Bootstrap resampling
+        sample_statistics = self._bootstrap_statistics(data, statistic, n_resamples)
+
+        # Jackknife resampling for acceleration
+        acceleration = self._jackknife_acceleration(data, statistic)
+
+        # Bias correction
+        observed_stat = statistic(data)
+        bias = np.mean(sample_statistics < observed_stat)
+        z0 = norm.ppf(bias)
+
+        # Adjusting percentiles
+        alpha = 1 - confidence_level
+        z_alpha = norm.ppf(1 - alpha / 2)
+
+        z_lower_bound = (z0 - z_alpha) / (1 - acceleration * (z0 - z_alpha)) + z0
+        z_upper_bound = (z0 + z_alpha) / (1 - acceleration * (z0 + z_alpha)) + z0
+
+        alpha_lower = norm.cdf(z_lower_bound)
+        alpha_upper = norm.cdf(z_upper_bound)
+
+        # Calculate lower and upper bounds from the percentiles
+        lower_bound = np.quantile(sample_statistics, alpha_lower)
+        upper_bound = np.quantile(sample_statistics, alpha_upper)
+
+        return lower_bound, upper_bound
+
+    def _statistic_generate(
+        self,
+        df: pd.DataFrame,
+        confidence_level: float,
+        statistic: Callable,
+        n_resamples: int,
+        random_state: int,
+    ):
+        """
+        Calculate statistics for the reference distances, including confidence intervals and thresholds.
+        """
+        ci_lower, ci_upper = self._bootstrapping_bca(
+            df["metric"],
+            confidence_level,
+            statistic,
+            n_resamples,
+            random_state,
+        )
+        estimated_mean = np.mean(df["metric"])
+
+        return {
+            "ci_lower": ci_lower,
+            "ci_upper": ci_upper,
+            "mean": estimated_mean,
+        }
+
+    def _custom_threshold(
+        self, data: pd.Series, custom_func: Callable
+    ) -> Tuple[float, float]:
+        """Calculate thresholds using a custom function."""
+        return custom_func(data)
+
+    def _calculate_threshold(
+        self, data: pd.Series, center: Callable, spread: Callable, factor: float = 3
+    ) -> Tuple[float, float]:
+        """Calculate thresholds using a central tendency and spread function."""
+        center_value = center(data)
+        spread_value = spread(data)
+        lower_limit = center_value - factor * spread_value
+        upper_limit = center_value + factor * spread_value
+        return lower_limit, upper_limit
+
+    def _iqr_threshold(self, df: pd.DataFrame) -> Tuple[float, float]:
+        """
+        Calculates thresholds using IQR and median with a default factor of 1.5.
+        """
+
+        def iqr(x):
+            q75, q25 = np.percentile(x, [75, 25])
+            return q75 - q25
+
+        return self._calculate_threshold(df["metric"], np.median, iqr, factor=1.5)
+
+    def _deviation_threshold(self, df: pd.DataFrame) -> Tuple[float, float]:
+        """Calculates thresholds using mean and standard deviation."""
+        return self._calculate_threshold(df["metric"], np.mean, np.std)
+
+    def _mad_threshold(self, df: pd.DataFrame) -> Tuple[float, float]:
+        """Calculates thresholds using mean and Median Absolute Deviation (MAD)."""
+        mad = lambda x: np.median(np.abs(x - np.median(x)))
+        return self._calculate_threshold(df["metric"], np.mean, mad)
+
+    def _drift_limit_generate(
+        self,
+        statistics: Dict,
+        distribution: pd.DataFrame,
+        drift_limit: Union[str, Callable, Tuple[float]],
+    ):
+        """
+        Determines the drift limits based on the specified method.
+        """
+        if isinstance(drift_limit, str):
+            if drift_limit == "deviation":
+                lower_limit, upper_limit = self._deviation_threshold(distribution)
+            elif drift_limit == "mad":
+                lower_limit, upper_limit = self._mad_threshold(distribution)
+            elif drift_limit == "iqr":
+                lower_limit, upper_limit = self._iqr_threshold(distribution)
+            else:
+                raise ValueError(f"Unsupported drift limit method: {drift_limit}")
+        elif callable(drift_limit):
+            lower_limit, upper_limit = self._custom_threshold(
+                distribution["metric"], drift_limit
+            )
+        elif isinstance(drift_limit, tuple) and len(drift_limit) == 2:
+            lower_limit, upper_limit = drift_limit
+        else:
+            raise ValueError("Invalid drift limit specification.")
+
+        # Update the statistics dictionary with the new thresholds
+        statistics["lower_limit"] = lower_limit
+        statistics["upper_limit"] = upper_limit
+
+    def _validate_columns(
+        self,
+        df: pd.DataFrame,
+        target_col: str,
+        datetime_col: str,
+    ):
+        if target_col not in df.columns:
+            raise KeyError(f"Column {target_col} is not in the DataFrame.")
+        if datetime_col not in df.columns:
+            raise KeyError(f"Datetime column {datetime_col} is not in the DataFrame.")
+        if not pd.api.types.is_datetime64_any_dtype(df[datetime_col]):
+            raise TypeError(f"Column {datetime_col} must be of datetime type.")
+
+    def _validate_params(
+        self,
+        confidence_level: float,
+        n_resamples: int,
+        period: str,
+    ):
+        if not 0 < confidence_level <= 1:
+            raise ValueError("confidence_level must be between 0 and 1.")
+        if n_resamples <= 0:
+            raise ValueError("n_resamples must be a positive integer.")
+        if not isinstance(period, str):
+            raise TypeError("period must be a string (e.g., 'W', 'M').")
+
+    def _is_drifted(self, df: pd.DataFrame) -> pd.Series:
+        """
+        Checks if metrics in the DataFrame are outside specified limits
+        and returns the drift status.
+        """
+        is_drifted = pd.Series([False] * len(df))
+
+        if self.statistics["lower_limit"] is not None:
+            is_drifted |= df["metric"] <= self.statistics["lower_limit"]
+        if self.statistics["upper_limit"] is not None:
+            is_drifted |= df["metric"] >= self.statistics["upper_limit"]
+
+        return is_drifted

tinyshift/detector/__init__.py

@@ -0,0 +1,2 @@
+from .continuous import ContinuousDriftDetector
+from .categorical import CategoricalDriftDetector

tinyshift/detector/categorical.py

@@ -0,0 +1,226 @@
+import numpy as np
+import pandas as pd
+from scipy.spatial.distance import jensenshannon
+from ..base.model import BaseModel
+from typing import Callable, Tuple, Union
+
+
+def l_infinity(a, b):
+    """
+    Compute the L-infinity distance between two distributions.
+    """
+    return np.max(np.abs(a - b))
+
+
+class CategoricalDriftDetector(BaseModel):
+    def __init__(
+        self,
+        reference: pd.DataFrame,
+        target_col: str,
+        datetime_col: str,
+        period: str,
+        func: str = "l_infinity",
+        statistic: Callable = np.mean,
+        confidence_level: float = 0.997,
+        n_resamples: int = 1000,
+        random_state: int = 42,
+        drift_limit: Union[str, Tuple[float, float]] = "deviation",
+    ):
+        """
+        A detector for identifying drift in categorical data over time. The detector uses
+        a reference dataset to compute a baseline distribution and compare subsequent data
+        for deviations based on a distance metric and drift limits.
+
+        Parameters:
+        ----------
+        reference : DataFrame
+            The reference dataset used to compute the baseline distribution.
+        target_col : str
+            The name of the column containing the categorical variable to analyze.
+        datetime_col : str
+            The name of the column containing datetime values for temporal grouping.
+        period : str
+            The frequency for grouping data (e.g., 'D' for daily, 'M' for monthly).
+        func : str, optional
+            The distance function to use ('l_infinity' or 'jensenshannon').
+            Default is 'l_infinity'.
+        statistic : callable, optional
+            The statistic function used to summarize the reference distances.
+            Default is `np.mean`.
+        confidence_level : float, optional
+            The confidence level for calculating statistical thresholds.
+            Default is 0.997.
+        n_resamples : int, optional
+            Number of resamples for bootstrapping when calculating statistics.
+            Default is 1000.
+        random_state : int, optional
+            Seed for reproducibility of random resampling.
+            Default is 42.
+        drift_limit : tuple, optional
+            User-defined thresholds for drift detection.
+            Default is the 'deviation method'.
+
+        Attributes:
+        ----------
+        period : str
+            The grouping frequency used for analysis.
+        reference_frequency : DataFrame
+            The frequency distribution of the reference dataset.
+        reference_distance : DataFrame
+            The distance metric values for the reference dataset.
+        statistics : dict
+            Statistical thresholds and summary statistics for drift detection.
+        plot : Plot
+            A plotting utility for visualizing drift results.
+        """
+
+        self._validate_columns(reference, target_col, datetime_col)
+        self._validate_params(confidence_level, n_resamples, period)
+
+        self.period = period
+        self.func = self._selection_function(func)
+
+        self.reference_frequency = self._calculate_frequency(
+            reference,
+            target_col,
+            datetime_col,
+            period,
+        )
+
+        self.reference_distance = self._generate_distance(
+            self.reference_frequency,
+        )
+
+        super().__init__(
+            self.reference_distance,
+            confidence_level,
+            statistic,
+            n_resamples,
+            random_state,
+            drift_limit,
+        )
+
+    def _calculate_frequency(
+        self,
+        df: pd.DataFrame,
+        target_col: str,
+        datetime_col: str,
+        period: str,
+    ) -> pd.DataFrame:
+        """
+        Calculate the frequency distribution of the target column grouped by a time period.
+
+        Parameters:
+        ----------
+        df : DataFrame
+            The dataset to analyze.
+        target_col : str
+            The name of the categorical column.
+        datetime_col : str
+            The name of the datetime column for temporal grouping.
+        period : str
+            The frequency for grouping (e.g., 'D', 'M').
+
+        Returns:
+        -------
+        DataFrame
+            A pivot table of frequencies with time periods as rows and categorical
+            values as columns.
+        """
+        freq = (
+            df.groupby([pd.Grouper(key=datetime_col, freq=period), target_col])
+            .size()
+            .unstack(fill_value=0)
+        )
+
+        return freq
+
+    def _selection_function(self, func_name: str) -> Callable:
+        """Returns a specific function based on the given function name."""
+
+        if func_name == "l_infinity":
+            selected_func = l_infinity
+        elif func_name == "jensenshannon":
+            selected_func = jensenshannon
+        else:
+            raise ValueError(f"Unsupported distance function: {func_name}")
+        return selected_func
+
+    def _generate_distance(
+        self,
+        p: pd.DataFrame,
+    ) -> pd.DataFrame:
+        """
+        Compute a distance metric between consecutive periods in the frequency distribution.
+
+        Parameters:
+        ----------
+        p : DataFrame
+            The frequency distribution with time periods as rows and categorical values as columns.
+        func : str
+            The distance function to use ('l_infinity' or 'jensenshannon').
+
+        Returns:
+        -------
+        DataFrame
+            A DataFrame containing datetime values and the calculated distances.
+        """
+        n = p.shape[0]
+        distances = np.zeros(n)
+        past_value = np.zeros(p.shape[1], dtype=np.int32)
+        index = p.index[1:]
+        p = np.asarray(p)
+
+        for i in range(1, n):
+            past_value = past_value + p[i - 1]
+            past_value = past_value / np.sum(past_value)
+            current_value = p[i] / np.sum(p[i])
+            dist = self.func(past_value, current_value)
+            distances[i] = dist
+
+        return pd.DataFrame({"datetime": index, "metric": distances[1:]})
+
+    def score(
+        self,
+        analysis: pd.DataFrame,
+        target_col: str,
+        datetime_col: str,
+    ) -> pd.DataFrame:
+        """
+        Assess drift in the provided dataset by comparing its distribution to the reference.
+
+        Parameters:
+        ----------
+        analysis : DataFrame
+            The dataset to analyze for drift.
+        target_col : str
+            The name of the categorical column in the analysis dataset.
+        datetime_col : str
+            The name of the datetime column in the analysis dataset.
+
+        Returns:
+        -------
+        DataFrame
+            A DataFrame containing metrics and drift detection results for each time period.
+        """
+        self._validate_columns(analysis, target_col, datetime_col)
+
+        # Calculate frequency and percentage distribution
+        freq = self._calculate_frequency(
+            analysis, target_col, datetime_col, self.period
+        )
+        percent = freq.div(freq.sum(axis=1), axis=0)
+
+        # Calculate percentage distribution
+        ref_freq = self.reference_frequency.sum(axis=0)
+        ref_dist = ref_freq / np.sum(ref_freq)
+
+        # Calculate drift metrics for each time period
+        metrics = (
+            percent.apply(lambda row: self.func(row, ref_dist), axis=1)
+            .rename("metric")
+            .reset_index()
+        )
+        metrics["is_drifted"] = self._is_drifted(metrics)
+
+        return metrics

tinyshift/detector/continuous.py

@@ -0,0 +1,222 @@
+import numpy as np
+import pandas as pd
+from scipy.stats import ks_2samp, wasserstein_distance
+from ..base.model import BaseModel
+from typing import Callable, Tuple, Union
+
+
+class ContinuousDriftDetector(BaseModel):
+    def __init__(
+        self,
+        reference: pd.DataFrame,
+        target_col: str,
+        datetime_col: str,
+        period: str,
+        func: str = "ws",
+        statistic: Callable = np.mean,
+        confidence_level: float = 0.997,
+        n_resamples: int = 1000,
+        random_state: int = 42,
+        drift_limit: Union[str, Tuple[float, float]] = "deviation",
+    ):
+        """
+        A detector for identifying drift in continuous data over time. The detector uses
+        a reference dataset to compute a baseline distribution and compare subsequent data
+        for deviations using the Kolmogorov-Smirnov test and statistical thresholds.
+
+        Parameters:
+        ----------
+        reference : DataFrame
+            The reference dataset used to compute the baseline distribution.
+        target_col : str
+            The name of the column containing the continuous variable to analyze.
+        datetime_col : str
+            The name of the column containing datetime values for temporal grouping.
+        period : str
+            The frequency for grouping data (e.g., '1D' for daily, '1H' for hourly).
+        statistic : callable, optional
+            The statistic function used to summarize the reference KS metrics.
+            Default is `np.mean`.
+        confidence_level : float, optional
+            The confidence level for calculating statistical thresholds.
+            Default is 0.997.
+        n_resamples : int, optional
+            Number of resamples for bootstrapping when calculating statistics.
+            Default is 1000.
+        random_state : int, optional
+            Seed for reproducibility of random resampling.
+            Default is 42.
+        thresholds : tuple, optional
+            User-defined thresholds for drift detection.
+            Default is an empty tuple.
+
+        Attributes:
+        ----------
+        period : str
+            The grouping frequency used for analysis.
+        reference_distribution : Series
+            The distribution of the reference dataset grouped by the specified period.
+        reference_ks : DataFrame
+            The Kolmogorov-Smirnov test results for the reference dataset.
+        statistics : dict
+            Statistical thresholds and summary statistics for drift detection.
+        plot : Plot
+            A plotting utility for visualizing drift results.
+        """
+
+        self._validate_columns(reference, target_col, datetime_col)
+        self._validate_params(confidence_level, n_resamples, period)
+
+        self.period = period
+        self.func = func
+
+        # Initialize frequency and statistics
+        self.reference_distribution = self._calculate_distribution(
+            reference,
+            target_col,
+            datetime_col,
+            period,
+        )
+
+        self.reference_distance = self._generate_distance(
+            self.reference_distribution, func
+        )
+
+        super().__init__(
+            self.reference_distance,
+            confidence_level,
+            statistic,
+            n_resamples,
+            random_state,
+            drift_limit,
+        )
+
+    def _calculate_distribution(
+        self,
+        df: pd.DataFrame,
+        column_name: str,
+        timestamp: str,
+        period: str,
+    ) -> pd.Series:
+        """
+        Calculate the continuous distribution of a target column grouped by a given period.
+
+        Parameters:
+        ----------
+        df : pd.DataFrame
+            The dataset to analyze.
+        column_name : str
+            The name of the column containing the continuous variable.
+        timestamp : str
+            The name of the datetime column for temporal grouping.
+        period : str
+            The frequency for grouping (e.g., '1D', '1H').
+
+        Returns:
+        -------
+        pd.Series
+            A Pandas Series where each index corresponds to a time period, and each value is
+            a list of continuous values for that period.
+        """
+        return (
+            df[[timestamp, column_name]]
+            .copy()
+            .groupby(pd.Grouper(key=timestamp, freq=period))[column_name]
+            .agg(list)
+        )
+
+    def _ks(self, a, b):
+        """Calculate the Kolmogorov-Smirnov test and return the p_value."""
+        _, p_value = ks_2samp(a, b)
+        return p_value
+
+    def _wasserstein(self, a, b):
+        """Calculate the Wasserstein Distance."""
+        return wasserstein_distance(a, b)
+
+    def _selection_function(self, func_name: str) -> Callable:
+        """Returns a specific function based on the given function name."""
+
+        if func_name == "ws":
+            selected_func = self._wasserstein
+        elif func_name == "ks":
+            selected_func = self._ks
+        else:
+            raise ValueError(f"Unsupported function: {func_name}")
+        return selected_func
+
+    def _generate_distance(
+        self,
+        p: pd.Series,
+        func_name: Callable,
+    ) -> pd.DataFrame:
+        """
+        Calculate the Kolmogorov-Smirnov test metric over a rolling cumulative window.
+
+        Parameters:
+        ----------
+        p : Series
+            A Pandas Series where each element is a list representing the distribution
+            of values for a specific period.
+
+        Returns:
+        -------
+        DataFrame
+            A DataFrame containing datetime indices and the calculated KS test metric
+            for each period.
+        """
+        func = self._selection_function(func_name)
+
+        n = p.shape[0]
+        values = np.zeros(n)
+        past_values = np.array([], dtype=float)
+
+        for i in range(1, n):
+            past_values = np.concatenate([past_values, p[i - 1]])
+            value = func(past_values, p[i])
+            values[i] = value
+
+        return pd.DataFrame({"datetime": p.index[1:], "metric": values[1:]})
+
+    def score(
+        self,
+        analysis: pd.DataFrame,
+        target_col: str,
+        datetime_col: str,
+    ) -> pd.DataFrame:
+        """
+        Assess drift in the provided dataset by comparing its distribution to the reference.
+
+        Parameters:
+        ----------
+        analysis : DataFrame
+            The dataset to analyze for drift.
+        target_col : str
+            The name of the continuous column in the analysis dataset.
+        datetime_col : str
+            The name of the datetime column in the analysis dataset.
+
+        Returns:
+        -------
+        DataFrame
+            A DataFrame containing datetime values, drift metrics, and a boolean
+            indicating whether drift was detected for each time period.
+        """
+
+        self._validate_columns(analysis, target_col, datetime_col)
+
+        reference = np.concatenate(self.reference_distribution)
+        dist = self._calculate_distribution(
+            analysis, target_col, datetime_col, self.period
+        )
+
+        func = self._selection_function(self.func)
+        metrics = np.array([func(reference, row) for row in dist])
+        metrics = pd.DataFrame(
+            {
+                "datetime": dist.index,
+                "metric": metrics,
+            },
+        )
+        metrics["is_drifted"] = self._is_drifted(metrics)
+        return metrics

tinyshift/plot/plot.py

@@ -0,0 +1,213 @@
+import plotly.graph_objects as go
+import numpy as np
+import plotly.express as px
+import scipy.stats
+import pandas as pd
+
+
+class Plot:
+    def __init__(self, statistics, distribution):
+        self.statistics = statistics
+        self.distribution = distribution
+
+    def _update_layout(
+        self,
+        title: str,
+        xaxis_title: str,
+        yaxis_title: str,
+        width: int,
+        height: int,
+    ):
+        """
+        Helper function to update layout settings for the plots.
+        """
+        return dict(
+            title=title,
+            xaxis_title=xaxis_title,
+            yaxis_title=yaxis_title,
+            width=width,
+            height=height,
+            showlegend=True,
+            bargap=0,
+            bargroupgap=0,
+        )
+
+    def _add_limits(self, fig):
+        """
+        Helper function to add the lower and upper limits and the mean line to the plot.
+        """
+        lower_limit, upper_limit = self.statistics.get(
+            "lower_limit"
+        ), self.statistics.get("upper_limit")
+        if lower_limit:
+            fig.add_hline(
+                y=lower_limit,
+                line_dash="dash",
+                line_color="firebrick",
+                name="Lower Limit",
+                opacity=0.5,
+            )
+        if upper_limit:
+            fig.add_hline(
+                y=upper_limit,
+                line_dash="dash",
+                line_color="firebrick",
+                name="Upper Limit",
+                opacity=0.5,
+            )
+
+        fig.add_hline(
+            y=self.statistics["mean"],
+            line_dash="dash",
+            line_color="darkslateblue",
+            opacity=0.3,
+            name="Mean",
+        )
+
+    def kde(
+        self,
+        width: int = 600,
+        height: int = 400,
+        fig_type: str = None,
+    ):
+        """
+        Generate a Kernel Density Estimate (KDE) plot for the distribution's metric.
+        """
+        x_vals = np.linspace(
+            self.distribution["metric"].min(), self.distribution["metric"].max(), 1000
+        )
+        kde = scipy.stats.gaussian_kde(self.distribution["metric"])
+
+        # Create KDE plot using plotly.express
+        fig = px.line(x=x_vals, y=kde(x_vals))
+        fig.update_layout(
+            self._update_layout(
+                "Distribution of metric with Kernel Density Estimate (KDE)",
+                "Metric",
+                "Density",
+                width,
+                height,
+            )
+        )
+
+        return fig.show(fig_type)
+
+    def diverging_bar_over_time(
+        self,
+        analysis: pd.DataFrame,
+        width: int = 800,
+        height: int = 400,
+        fig_type: str = None,
+    ):
+        """
+        Generate a diverging bar plot showing metric over time relative to a reference line.
+        """
+        reference_line = self.statistics["mean"]
+        positive_bars = np.maximum(analysis["metric"] - reference_line, 0)
+        negative_bars = np.maximum(reference_line - analysis["metric"], 0)
+
+        fig = go.Figure()
+
+        fig.add_trace(
+            go.Bar(
+                x=analysis["datetime"],
+                y=positive_bars,
+                base=[reference_line] * len(positive_bars),
+                name="Above Reference",
+                marker_color="lightslategrey",
+                customdata=analysis.loc[analysis["metric"] >= reference_line, "metric"],
+                hovertemplate="(%{x},%{y:.3f})",
+                opacity=0.7,
+            )
+        )
+
+        fig.add_trace(
+            go.Bar(
+                x=analysis["datetime"],
+                y=negative_bars,
+                base=reference_line - negative_bars,
+                name="Below Reference",
+                marker_color="crimson",
+                customdata=analysis.loc[analysis["metric"] < reference_line, "metric"],
+                hovertemplate="(%{x},%{base:.3f})",
+                opacity=0.7,
+            )
+        )
+
+        fig.add_hrect(
+            y0=self.statistics["ci_lower"],
+            y1=self.statistics["ci_upper"],
+            line_width=0,
+            fillcolor="lightblue",
+            opacity=0.5,
+            name="Fixed Confidence Interval",
+        )
+
+        self._add_limits(fig)
+
+        fig.update_layout(
+            self._update_layout(
+                "Metric Over Time with Fixed Confidence Interval",
+                "Time",
+                "Metric",
+                width,
+                height,
+            )
+        )
+
+        return fig.show(fig_type)
+
+    def scatterplot_over_time(
+        self,
+        analysis: pd.DataFrame,
+        width: int = 800,
+        height: int = 400,
+        fig_type: str = None,
+    ):
+        """
+        Generate a time-series plot showing the metric performance with confidence interval and thresholds.
+        """
+        fig = go.Figure()
+
+        fig.add_trace(
+            go.Scatter(
+                x=analysis["datetime"],
+                y=analysis["metric"],
+                mode="lines+markers",
+                name="Metric",
+            )
+        )
+
+        # Confidence interval shading
+        fig.add_trace(
+            go.Scatter(
+                x=analysis["datetime"],
+                y=[self.statistics["ci_lower"], self.statistics["ci_upper"]],
+                fill="toself",
+                fillcolor="rgba(0, 100, 255, 0.2)",
+                line=dict(color="rgba(255,255,255,0)"),
+                name="Fixed Confidence Interval",
+            )
+        )
+
+        fig.add_hrect(
+            y0=self.statistics["ci_lower"],
+            y1=self.statistics["ci_upper"],
+            line_width=0,
+            fillcolor="lightblue",
+            opacity=0.5,
+        )
+
+        self._add_limits(fig)
+
+        fig.update_layout(
+            self._update_layout(
+                "Metric Over Time with Fixed Confidence Interval",
+                "Time",
+                "Metric",
+                width,
+                height,
+            )
+        )
+
+        return fig.show(fig_type)

tinyshift/tracker/__init__.py

@@ -0,0 +1 @@
+from .performance import PerformanceTracker

tinyshift/tracker/performance.py

@@ -0,0 +1,193 @@
+import numpy as np
+from sklearn.metrics import f1_score
+import pandas as pd
+from ..base.model import BaseModel
+from typing import Callable, Tuple, Union
+
+
+class PerformanceTracker(BaseModel):
+    def __init__(
+        self,
+        reference: pd.DataFrame,
+        target_col: str,
+        prediction_col: str,
+        datetime_col: str,
+        period: str,
+        metric_score: Callable = f1_score,
+        statistic: Callable = np.mean,
+        confidence_level: float = 0.997,
+        n_resamples: int = 1000,
+        random_state: int = 42,
+        drift_limit: Union[str, Tuple[float, float]] = "deviation",
+    ):
+        """
+        A tracker for monitoring model performance over time using a specified evaluation metric.
+        The tracker compares the performance metric across time periods to a reference distribution
+        and identifies potential performance degradation.
+
+        Parameters:
+        ----------
+        reference : DataFrame
+            The reference dataset used to compute the baseline metric distribution.
+        target_col : str
+            The name of the column containing the actual target values.
+        prediction_col : str
+            The name of the column containing the predicted values.
+        datetime_col : str
+            The name of the column containing datetime values for temporal grouping.
+        period : str
+            The frequency for grouping data (e.g., 'W' for weekly, 'M' for monthly).
+        metric_score : callable, optional
+            The function to compute the evaluation metric (e.g., `f1_score`).
+            Default is `f1_score`.
+        statistic : callable, optional
+            The statistic function used to summarize the reference metric distribution.
+            Default is `np.mean`.
+        confidence_level : float, optional
+            The confidence level for calculating statistical thresholds.
+            Default is 0.997.
+        n_resamples : int, optional
+            Number of resamples for bootstrapping when calculating statistics.
+            Default is 1000.
+        random_state : int, optional
+            Seed for reproducibility of random resampling.
+            Default is 42.
+        thresholds : tuple, optional
+            User-defined thresholds for drift detection.
+            Default is an empty tuple.
+
+        Attributes:
+        ----------
+        period : str
+            The grouping frequency used for analysis.
+        metric_score : callable
+            The evaluation metric function used for tracking performance.
+        reference_distribution : DataFrame
+            The performance metric distribution of the reference dataset.
+        statistics : dict
+            Statistical thresholds and summary statistics for performance monitoring.
+        plot : Plot
+            A plotting utility for visualizing performance over time.
+        """
+
+        self._validate_params(
+            confidence_level,
+            n_resamples,
+            period,
+        )
+        self._validate_columns(
+            reference,
+            target_col,
+            datetime_col,
+        )
+
+        if not callable(metric_score):
+            raise TypeError("metric_score must be a callable function.")
+
+        self.period = period
+        self.metric_score = metric_score
+
+        # Initialize distributions and statistics
+        self.reference_distribution = self._calculate_metric(
+            reference,
+            target_col,
+            prediction_col,
+            datetime_col,
+        )
+        super().__init__(
+            self.reference_distribution,
+            confidence_level,
+            statistic,
+            n_resamples,
+            random_state,
+            drift_limit,
+        )
+
+    def _calculate_metric(
+        self,
+        df: pd.DataFrame,
+        target_col: str,
+        prediction_col: str,
+        datetime_col: str,
+    ):
+        """
+        Calculate the performance metric for each time period in the dataset.
+
+        Parameters:
+        ----------
+        df : DataFrame
+            The dataset containing the data to analyze.
+        target_col : str
+            The name of the column containing the actual target values.
+        prediction_col : str
+            The name of the column containing the predicted values.
+        datetime_col : str
+            The name of the datetime column for temporal grouping.
+
+        Returns:
+        -------
+        DataFrame
+            A DataFrame with the calculated metric for each time period.
+        """
+        if target_col not in df.columns or prediction_col not in df.columns:
+            raise KeyError(
+                f"Columns {target_col} and/or {prediction_col} are not in the DataFrame."
+            )
+        if datetime_col not in df.columns:
+            raise KeyError(f"Datetime column {datetime_col} is not in the DataFrame.")
+        if not pd.api.types.is_datetime64_any_dtype(df[datetime_col]):
+            raise TypeError(f"Column {datetime_col} must be of datetime type.")
+
+        grouped = df.groupby(pd.Grouper(key=datetime_col, freq=self.period)).apply(
+            lambda x: self.metric_score(x[target_col], x[prediction_col])
+        )
+        return grouped.reset_index(name="metric")
+
+    def score(
+        self,
+        analysis: pd.DataFrame,
+        target_col: str,
+        prediction_col: str,
+        datetime_col: str,
+    ):
+        """
+        Assess model performance over time by calculating the evaluation metric
+        for each time period and comparing it to the reference distribution.
+
+        Parameters:
+        ----------
+        analysis : DataFrame
+            The dataset to analyze for performance drift.
+        target_col : str
+            The name of the column containing the actual target values.
+        prediction_col : str
+            The name of the column containing the predicted values.
+        datetime_col : str
+            The name of the datetime column for temporal grouping.
+
+        Returns:
+        -------
+        DataFrame
+            A DataFrame containing datetime values, calculated metrics, and a boolean
+            indicating whether performance drift was detected for each time period.
+        """
+
+        self._validate_columns(analysis, target_col, datetime_col)
+
+        if target_col not in analysis.columns or prediction_col not in analysis.columns:
+            raise KeyError(
+                f"Columns {target_col} and/or {prediction_col} are not in the DataFrame."
+            )
+        if datetime_col not in analysis.columns:
+            raise KeyError(f"Datetime column {datetime_col} is not in the DataFrame.")
+        if not pd.api.types.is_datetime64_any_dtype(analysis[datetime_col]):
+            raise TypeError(f"Column {datetime_col} must be of datetime type.")
+
+        if analysis.empty:
+            raise ValueError("Input DataFrame is empty.")
+
+        metrics = self._calculate_metric(
+            analysis, target_col, prediction_col, datetime_col
+        )
+        metrics["is_drifted"] = self._is_drifted(metrics)
+        return metrics

tinyshift-0.0.2.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Lucas Leão
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tinyshift-0.0.2.dist-info/METADATA

@@ -0,0 +1,117 @@
+Metadata-Version: 2.1
+Name: tinyshift
+Version: 0.0.2
+Summary: A small toolbox for MLOps
+Author-email: Lucas Leão <heylucasleao@gmail.com>
+License: MIT License
+Keywords: mlops,toolbox,machine-learning
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas
+Requires-Dist: scipy
+Requires-Dist: plotly
+Requires-Dist: scikit-learn
+Requires-Dist: numpy
+
+# TinyShift
+
+**TinyShift** is a small experimental Python library designed to detect **data drifts** and **performance drops** in machine learning models over time. The main goal of the project is to provide quick and tiny monitoring tools to help identify when data or model performance unexpectedly change.
+For more robust solutions, I highly recommend [Nannyml.](https://github.com/NannyML/nannyml)
+
+## Technologies Used
+
+- **Python 3.x**
+- **Scikit-learn**
+- **Pandas**
+- **NumPy**
+- **Plotly**
+- **Scipy**
+
+## Installation
+
+To install **TinyShift** in your development environment, use **pip**:
+
+
+```bash
+pip install tinyshift
+```
+If you prefer to clone the repository and install manually:
+```bash
+git clone https://github.com/HeyLucasLeao/tinyshift.git
+cd tinyshift    
+pip install .
+```
+
+## Usage
+Below are basic examples of how to use TinyShift's features.
+### 1. Data Drift Detection
+To detect data drift, simply score in a new dataset to compare with the reference data. The DataDriftDetector will calculate metrics to identify significant differences.
+
+```python
+from tinyshift.detector import CategoricalDriftDetector
+
+df = pd.DataFrame("examples.csv")
+df_reference = df[(df["datetime"] < '2024-07-01')].copy()
+df_analysis = df[(df["datetime"] >= '2024-07-01')].copy()
+
+detector = CategoricalDriftDetector(df_reference, 'discrete_1', "datetime", "W", drift_limit='mad')
+
+analysis_score = detector.score(df_analysis, "discrete_1", "datetime")
+
+print(analysis_score)
+```
+
+### 2. Performance Tracker
+To track model performance over time, use the PerformanceMonitor, which will compare model accuracy on both old and new data.
+```python
+from tinyshift.tracker import PerformanceTracker
+
+df_reference = pd.read_csv('refence.csv')
+df_analysis = pd.read_csv('analysis.csv')
+model = load_model('model.pkl') 
+df_analysis['prediction'] = model.predict(df_analysis["feature_0"])
+
+tracker = PerformanceTracker(df_reference, 'target', 'prediction', 'datetime', "W")
+
+analysis_score = tracker.score(df_analysis, 'target', 'prediction', 'datetime')
+
+print(analysis_score)
+```
+
+### 3. Visualization
+TinyShift also provides graphs to visualize the magnitude of drift and performance changes over time.
+```python
+tracker.plot.scatterplot_over_time(analysis_score, fig_type="png")
+
+tracker.plot.diverging_bar_over_time(analysis_score, fig_type="png")
+```
+
+## Project Structure
+The basic structure of the project is as follows:
+```
+tinyshift
+├── LICENSE
+├── README.md
+├── example.ipynb
+├── pyproject.toml
+└── tinyshift
+    ├── base
+    │   ├── __init__.py
+    │   └── model.py
+    ├── detector
+    │   ├── __init__.py
+    │   ├── categorical.py
+    │   └── continuous.py
+    ├── plot
+    │   ├── __init__.py
+    │   └── plot.py
+    └── tracker
+        ├── __init__.py
+        └── performance.py          
+```
+
+### License
+This project is licensed under the MIT License - see the LICENSE file for more details.

tinyshift-0.0.2.dist-info/RECORD

@@ -0,0 +1,14 @@
+tinyshift/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tinyshift/base/model.py,sha256=riQfiFrQieK6X0zsj1djoWrM3S_bTOuKdO6VlzpCo7o,9270
+tinyshift/detector/__init__.py,sha256=hjP0xqHERdl1DHD3Fs6GrcbCou1EqZmIHwtbH-VhNG0,98
+tinyshift/detector/categorical.py,sha256=oFj0dIuA34_pxqOVCxadSA4CqEgT5xpSBiFpHyOWFow,7491
+tinyshift/detector/continuous.py,sha256=zRu5wYgQlYj9KpPnFaBg0JzFCxiq0Z9e--LyuCG7rl8,7394
+tinyshift/plot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tinyshift/plot/plot.py,sha256=ON9E653BXm1kTxRd2Vxx-32u6D9ubqgnu698e6LAiyg,5985
+tinyshift/tracker/__init__.py,sha256=ElIARMomp32pFeJ3zwt4ISPv26BUtDc0Bx0JOORQICs,44
+tinyshift/tracker/performance.py,sha256=5-eDcF1BEX79Z6oFHvCFneOqA7pEUlIApgnWlJY-trA,7049
+tinyshift-0.0.2.dist-info/LICENSE,sha256=oGMyi5YKRfPvxPthLZi3eiJ9zVC3oHJ94YoesVfp1Hg,1068
+tinyshift-0.0.2.dist-info/METADATA,sha256=d0YKrhDG9OkhvH8Hj9AYUSOAgaIFCUmAw3En3WzLBms,3563
+tinyshift-0.0.2.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
+tinyshift-0.0.2.dist-info/top_level.txt,sha256=yzWi5DWDzkaUzuJMahKQm_igjQxnWX04bPz09c9sQ4k,10
+tinyshift-0.0.2.dist-info/RECORD,,

tinyshift-0.0.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.7.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tinyshift-0.0.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+tinyshift