anomaly-pipeline 0.1.27__py3-none-any.whl → 0.1.61__py3-none-any.whl

anomaly_pipeline/helpers/ewma.py

@@ -1,14 +1,15 @@
 import pandas as pd
 import numpy as np
 import statistics
+from .Preprocessing import classify
 
 # # EWMA functions
 
-def ewma_forecast(train, alpha):
-    """Return last EWMA forecast value based on training data."""
+"""def ewma_forecast(train, alpha):
+    Return last EWMA forecast value based on training data.
     ewma = train.ewm(alpha=alpha, adjust=False).mean()
     return ewma.iloc[-1]
-
+"""
 
 """
 def ew_std(series, alpha):
@@ -42,13 +43,13 @@ def ew_std(series, alpha):
     # Std = sqrt(var)
     return np.sqrt(ewma_var.iloc[-1]) """
 
-
+"""
 def ewma_with_anomalies_rolling_group(group, group_columns, variable, date_column, alpha, sigma, eval_period):
     
-    """
+
     Rolling (expanding window) EWMA anomaly detection for a SINGLE GROUP ONLY.
     Expects `group` to already be filtered to one group.
-    """
+
 
     group = group.sort_values(date_column).reset_index(drop=True)
     n = len(group)
@@ -115,5 +116,102 @@ def ewma_with_anomalies_rolling_group(group, group_columns, variable, date_colum
     final_output = pd.concat(results, ignore_index=True)
     # Type Safety Check: Ensure the date column is always datetime before returning
     final_output[date_column] = pd.to_datetime(final_output[date_column])
-    return final_output
+    return final_output"""
+
+
+
+def ewma_with_anomalies_rolling_group(group, group_columns, variable, date_column, alpha, sigma, eval_period):
+    """
+    Rolling (expanding window) EWMA anomaly detection for a SINGLE GROUP ONLY.
+    Expects `group` to already be filtered to one group.
+
+    # 📉 EWMA Rolling Anomaly Detection
+    ---
+
+    The `ewma_with_anomalies_rolling_group` function implements a **statistically weighted** approach to identifying outliers.
+    It uses an **Expanding Window** (Walk-Forward) strategy to adapt to recent trends while maintaining a memory of historical data.
+
+    ## 📋 Functional Overview
+    This function calculates the **Exponentially Weighted Moving Average (EWMA)**, which assigns higher importance to recent observations.
+    By combining this forecast with a dynamic standard deviation "envelope," the function identifies points that deviate significantly from the expected trend.
+
 
+
+    ## 🧠 Core Logic Components
+
+    ### 1. Forecast Engine (`ewma_forecast`)
+    * **Weighting Mechanism:** Uses an `alpha` parameter (between 0 and 1) to determine the "decay" of information. A **higher alpha** makes the model more sensitive to recent changes.
+    * **Calculation:** Employs the formula:
+      $$EWMA_t = \\alpha \\cdot Y_t + (1 - \\alpha) \\cdot EWMA_{t-1}$$
+
+    ### 2. The Rolling Anomaly Loop
+    The function partitions data into **TRAIN** and **TEST** sets and iterates through the evaluation period:
+    * **Expanding Training Set:** For every evaluation point, the function uses all preceding data to re-calculate the baseline.
+    * **Dynamic Thresholding:** * **Upper Limit:** `Forecast + (Sigma * Standard Deviation)`
+        * **Lower Limit:** `max(Forecast - (Sigma * Standard Deviation), 0)`
+    * **Iterative Evaluation:** It forecasts exactly **one point ahead**, checks for an anomaly, and then moves that point into the training set for the next iteration.
+
+    ## 📤 Key Output Columns
+    The function returns a concatenated DataFrame containing:
+    * **`EWMA_forecast`**: The predicted value for that timestamp.
+    * **`STD`**: The standard deviation used to calculate the threshold.
+    * **`EWMA_high` / `EWMA_low`**: The dynamic boundaries (the "envelope") for the test period.
+    * **`set`**: Labels data as either **"TRAIN"** (historical baseline) or **"TEST"** (anomaly detection window).
+    * **`is_EWMA_anomaly`**: A boolean flag indicating if the actual value fell outside the limits.
+
+    ## 💡 Usage Context
+    EWMA is ideal for **streaming-style data** or metrics that exhibit **level shifts**.
+    Because it weights recent data more heavily than a simple moving average, it is faster to adapt to new "normals" while still filtering out minor noise.
+
+    ---
+    ### ⚙️ Parameter Tuning
+    * **`alpha`**: Adjust this to control how quickly the model "forgets" old data (Typical range: `0.1 - 0.3`).
+    * **`sigma`**: Adjust this to control sensitivity. A **lower sigma** results in more anomalies, while a **higher sigma** (e.g., `3.0`) only flags extreme outliers.    
+    """
+
+    # 1. Prepare Data
+    group = group.sort_values(date_column).reset_index(drop=True)
+    vals = group[variable].astype(float)
+    
+    # 2. Calculate Statistics (Vectorized)
+    # Shift(1) ensures we use history only (no data leakage)
+    ewma_forecast = vals.ewm(alpha=alpha, adjust=False).mean().shift(1)
+    std_expanding = vals.expanding().std().shift(1)
+    
+    # 3. Construct Output DataFrame
+    results = group[group_columns + [date_column]].copy()
+    results[variable] = vals
+    results["alpha"] = alpha
+    results["sigma"] = sigma
+    
+    # 4. Handle Nulls for the first two rows (The "Backfill" logic)
+    # Backfilling allows us to have a baseline even for the very first point
+    results["EWMA_forecast"] = ewma_forecast.bfill()
+    results["STD"] = std_expanding.bfill().fillna(0) # fillna(0) in case there's only 1 row total
+    
+    # 5. Define Bounds (Now that nulls are handled)
+    results["EWMA_high"] = results["EWMA_forecast"] + (sigma * results["STD"])
+    results["EWMA_low"] = (results["EWMA_forecast"] - (sigma * results["STD"])).clip(lower=0)
+    
+    # 6. USE THE CLASSIFY FUNCTION
+    # Note: Ensure 'classify' function is defined in your script!
+    results["EWMA_anomaly"] = results.apply(
+        lambda row: classify(row[variable], row["EWMA_low"], row["EWMA_high"]), 
+        axis=1
+    )
+    
+    # If the first row was backfilled, we should force it to 'none' 
+    # to be safe since it's not a "real" statistical forecast.
+    results.loc[0, "EWMA_anomaly"] = 'none'
+    
+    # 7. Final Flags and Labels
+    results["is_EWMA_anomaly"] = results["EWMA_anomaly"] != 'none'
+    results["EWMA_residual"] = vals - results["EWMA_forecast"]
+    
+    results["set"] = "TRAIN"
+    if eval_period > 0 and len(results) >= eval_period:
+        results.iloc[-eval_period:, results.columns.get_loc("set")] = "TEST"
+        
+    results[date_column] = pd.to_datetime(results[date_column])
+    
+    return results

anomaly_pipeline/helpers/fb_prophet.py

@@ -5,6 +5,7 @@ from prophet import Prophet
 import warnings
 import os
 import sys
+from .Preprocessing import classify
 from contextlib import contextmanager
 
 warnings.filterwarnings("ignore")
@@ -21,13 +22,15 @@ def suppress_stdout_stderr():
         finally:
             sys.stdout, sys.stderr = old_stdout, old_stderr
 
+"""
 def detect_time_series_anomalies_fb_walkforward(
     group,
     variable,
     date_column,
     eval_period,
-    interval_width
+    prophet_CI
 ):
+    
     # 1. Silence the cmdstanpy logger completely
     logger = logging.getLogger('cmdstanpy')
     logger.addHandler(logging.NullHandler())
@@ -54,7 +57,8 @@ def detect_time_series_anomalies_fb_walkforward(
                 weekly_seasonality=True,
                 yearly_seasonality=True,
                 daily_seasonality=False,
-                interval_width=interval_width
+                interval_width=prophet_CI,
+                # prophet_CI=prophet_CI
             )
             
             # --- WRAP THE FIT IN THE MUTER ---
@@ -91,4 +95,148 @@ def detect_time_series_anomalies_fb_walkforward(
     group.loc[train_mask, "FB_residual"] = np.nan
     group.loc[train_mask, "is_FB_anomaly"] = np.nan
 
+    return group
+"""
+
+def detect_time_series_anomalies_fb_walkforward(
+    group,
+    variable,
+    date_column,
+    eval_period,
+    prophet_CI
+):
+    """
+    # 🚀 Facebook Prophet Walk-Forward Model
+    ---
+
+    The `detect_time_series_anomalies_fb_walkforward` function is a sophisticated forecasting tool designed for **iterative anomaly detection**. It utilizes the Facebook Prophet library to perform a **walk-forward validation**, forecasting one data point at a time and expanding the training set as it progresses.
+
+    ## 📋 Functional Overview
+    Unlike standard batch forecasting, this function operates by simulating a real-world scenario where the model is updated as soon as new data arrives. It establishes a **cutoff date** based on the specified `eval_period`, then iteratively predicts the next point, compares it to the observed value, and incorporates that value back into the training history.
+
+    ## 🧠 Core Logic Stages
+
+    ### 1. Data Preparation and Cutoff
+    * **Standardization:** The input data is sorted by date and converted to **datetime objects** to ensure proper time-series alignment.
+    * **Partitioning:** The dataset is split into an **Initial Training Set** (all data before the cutoff) and an **Evaluation Set** (the rolling forecast window).
+
+    ### 2. Walk-Forward Loop (Sequential Testing)
+    * **Model Fitting:** For every point in the evaluation set, a new **Prophet model** is initialized with weekly and yearly seasonality enabled.
+    * **One-Step Forecast:** The model generates a prediction (`yhat`) and an uncertainty interval (`yhat_lower`, `yhat_upper`) specifically for the **next single point**.
+    * **Dynamic Training Expansion:** After each prediction, the actual observed value is appended to the training data. This ensures the model learns from the most recent information before making the next prediction.
+    * **Robust Error Handling:** If the Prophet fit fails, the function falls back to a **baseline persistence model** (last observed value) to prevent pipeline failure.
+
+    ### 3. Anomaly Classification
+    * **Uncertainty Bounds:** Anomalies are defined by the `prophet_CI` parameter. Any observation falling outside the predicted upper or lower bounds is flagged.
+    * **Residual Calculation:** The function computes the **FB_residual** (Actual - Forecast) to quantify the magnitude of deviations.
+
+    ## 📤 Key Output Columns
+    The function appends the following columns to the returned DataFrame:
+    * **`FB_forecast`**: The point estimate predicted by Prophet for that date.
+    * **`FB_low` / `FB_high`**: The dynamic boundaries based on the specified uncertainty interval.
+    * **`FB_residual`**: The difference between the actual observed metric and the forecast.
+    * **`FB_anomaly`**: A categorical label designating the deviation as **"high"** or **"low"**.
+    * **`is_FB_anomaly`**: A boolean flag identifying outliers in the evaluation region.
+
+
+    ## 💡 Usage Context
+    This approach is highly effective for metrics with **strong seasonality and complex trends**. Because it uses a walk-forward loop, it is significantly more accurate than a static forecast for long evaluation periods, as it corrects itself based on the most recent trends. It is ideal for detecting "sudden" shifts that standard statistical models (like Z-Score) might miss.
+
+    ---
+    ### 📊 Evaluation Strategy
+    This function strictly ignores the training region for anomaly reporting, ensuring that all reported anomalies are based on "out-of-sample" performance where the model had no prior knowledge of the specific data point being tested.
+    
+    """
+    
+    # 1. Silence the cmdstanpy logger
+    logger = logging.getLogger('cmdstanpy')
+    logger.addHandler(logging.NullHandler())
+    logger.propagate = False
+    logger.setLevel(logging.CRITICAL) 
+    
+    group = group.sort_values(date_column).copy()
+    group[date_column] = pd.to_datetime(group[date_column])
+
+    # Calculate cutoff for the walk-forward
+    cutoff_date = group[date_column].max() - pd.Timedelta(weeks=eval_period)
+
+    group["FB_forecast"] = np.nan
+    group["FB_low"] = np.nan
+    group["FB_high"] = np.nan
+
+    train = group[group[date_column] <= cutoff_date].copy()
+    test = group[group[date_column] > cutoff_date].copy()
+
+    # --- INITIAL FIT FOR TRAIN DATA ---
+    prophet_train_initial = train.rename(columns={date_column: "ds", variable: "y"})
+    try:
+        model_initial = Prophet(
+            weekly_seasonality=True,
+            yearly_seasonality=True,
+            daily_seasonality=False,
+            interval_width=prophet_CI # Fixed: Prophet uses interval_width
+        )
+        with suppress_stdout_stderr():
+            model_initial.fit(prophet_train_initial)
+        
+        # Predict on the training dates to get historical bounds
+        train_forecast = model_initial.predict(prophet_train_initial)
+        
+        # Map back to group (Train indices)
+        train_indices = group[group[date_column] <= cutoff_date].index
+        group.loc[train_indices, "FB_forecast"] = train_forecast["yhat"].values
+        group.loc[train_indices, "FB_low"] = train_forecast["yhat_lower"].clip(lower=0).values
+        group.loc[train_indices, "FB_high"] = train_forecast["yhat_upper"].values
+
+    except Exception as e:
+        print(f"Initial Prophet fit failed: {e}")
+
+    # --- WALK-FORWARD FOR TEST DATA ---
+    for i, row in test.iterrows():
+        prophet_train = train.rename(columns={date_column: "ds", variable: "y"})
+        try:
+            model = Prophet(
+                weekly_seasonality=True,
+                yearly_seasonality=True,
+                daily_seasonality=False,
+                interval_width=prophet_CI
+            )
+            
+            with suppress_stdout_stderr():
+                model.fit(prophet_train)
+
+            future = pd.DataFrame({"ds": [row[date_column]]})
+            fc = model.predict(future).iloc[0]
+
+            group.loc[i, "FB_forecast"] = fc["yhat"]
+            group.loc[i, "FB_low"] = max(fc["yhat_lower"], 0)
+            group.loc[i, "FB_high"] = fc["yhat_upper"]
+
+        except Exception as e:
+            print(f"Prophet failed for KEY={group.get('key', ['NA'])[0]} on date={row[date_column]}: {e}")
+            # Fallback to naive logic
+            last_val = train[variable].iloc[-1]
+            group.loc[i, "FB_forecast"] = last_val
+            group.loc[i, "FB_low"] = last_val
+            group.loc[i, "FB_high"] = last_val
+
+        # Update train for next iteration
+        new_train_row = row.to_frame().T
+        train = pd.concat([train, new_train_row], ignore_index=True)
+
+    # --- UNIFIED ANOMALY DETECTION (Train + Test) ---
+    group["FB_residual"] = group[variable] - group["FB_forecast"]
+
+    # Applying your custom classify function row by row
+    group["FB_anomaly"] = group.apply(
+        lambda row: classify(row[variable], row["FB_low"], row["FB_high"]), 
+        axis=1
+    )
+
+    group["is_FB_anomaly"] = group["FB_anomaly"] != 'none'
+    
+    # Label set
+    group["set"] = "TRAIN"
+    group.loc[group[date_column] > cutoff_date, "set"] = "TEST"
+
     return group