priorityx 0.1.1__py3-none-any.whl

priorityx/__init__.py

@@ -0,0 +1,7 @@
+"""priorityx: Entity prioritization and escalation detection using GLMM statistical models."""
+
+__version__ = "0.1.0"
+
+# API will be imported
+
+__all__ = ["__version__"]

priorityx/core/filters.py

@@ -0,0 +1,224 @@
+"""Data quality filters for entity analysis."""
+
+from datetime import timedelta
+
+import pandas as pd
+import polars as pl
+
+
+def filter_sparse_entities(
+    df: pl.DataFrame,
+    entity_col: str,
+    timestamp_col: str,
+    min_total_count: int = 20,
+    min_observations: int = 3,
+    temporal_granularity: str = "yearly",
+) -> pl.DataFrame:
+    """
+    Filter entities with insufficient data.
+
+    Removes entities that don't meet minimum thresholds for:
+    - Total count across all time periods
+    - Number of time period observations
+
+    Args:
+        df: Input polars DataFrame
+        entity_col: Column name for entity identifier
+        timestamp_col: Column name for timestamp
+        min_total_count: Minimum total count (default: 20)
+        min_observations: Minimum time periods (default: 3)
+                         Auto-adjusts for temporal_granularity
+        temporal_granularity: 'yearly', 'quarterly', or 'semiannual'
+
+    Returns:
+        Filtered DataFrame with only entities meeting thresholds
+
+    Examples:
+        >>> df_filtered = filter_sparse_entities(
+        ...     df, entity_col="service", timestamp_col="date",
+        ...     min_total_count=50, min_observations=8
+        ... )
+    """
+    n_before = df.select(entity_col).n_unique()
+
+    # filter by total count
+    if min_total_count > 0:
+        total_counts = df.group_by(entity_col).agg(pl.len().alias("total_count"))
+        valid_entities = total_counts.filter(
+            pl.col("total_count") >= min_total_count
+        )[entity_col]
+        df = df.filter(pl.col(entity_col).is_in(valid_entities))
+
+    # filter by observations (time periods)
+    if min_observations > 0:
+        # auto-adjust for granularity
+        min_obs = min_observations
+        if min_obs == 3 and temporal_granularity == "quarterly":
+            min_obs = 8  # 2 years
+        elif min_obs == 3 and temporal_granularity == "semiannual":
+            min_obs = 4  # 2 years
+
+        # count unique periods per entity
+        if temporal_granularity == "quarterly":
+            df_temp = df.with_columns(
+                [
+                    pl.col(timestamp_col).dt.year().alias("year"),
+                    pl.col(timestamp_col).dt.quarter().alias("quarter"),
+                ]
+            )
+            period_counts = df_temp.group_by(entity_col).agg(
+                pl.struct(["year", "quarter"]).n_unique().alias("n_periods")
+            )
+        elif temporal_granularity == "semiannual":
+            df_temp = df.with_columns(
+                [
+                    pl.col(timestamp_col).dt.year().alias("year"),
+                    pl.col(timestamp_col).dt.quarter().alias("quarter"),
+                ]
+            ).with_columns(
+                pl.when(pl.col("quarter") <= 2)
+                .then(pl.lit(1))
+                .otherwise(pl.lit(2))
+                .alias("semester")
+            )
+            period_counts = df_temp.group_by(entity_col).agg(
+                pl.struct(["year", "semester"]).n_unique().alias("n_periods")
+            )
+        else:  # yearly
+            df_temp = df.with_columns(pl.col(timestamp_col).dt.year().alias("year"))
+            period_counts = df_temp.group_by(entity_col).agg(
+                pl.col("year").n_unique().alias("n_periods")
+            )
+
+        valid_entities = period_counts.filter(
+            pl.col("n_periods") >= min_obs
+        )[entity_col]
+        df = df.filter(pl.col(entity_col).is_in(valid_entities))
+
+    n_after = df.select(entity_col).n_unique()
+    n_filtered = n_before - n_after
+
+    if n_filtered > 0:
+        print(f"Filtered {n_filtered} sparse entities")
+
+    return df
+
+
+def filter_stale_entities(
+    df: pl.DataFrame,
+    entity_col: str,
+    timestamp_col: str,
+    decline_window_quarters: int = 6,
+) -> pl.DataFrame:
+    """
+    Filter entities inactive for more than N quarters.
+
+    Removes entities whose last observation is too far in the past
+    relative to the dataset's most recent date. Prevents long-inactive
+    entities from contaminating the model baseline.
+
+    Args:
+        df: Input polars DataFrame
+        entity_col: Column name for entity identifier
+        timestamp_col: Column name for timestamp (Date type)
+        decline_window_quarters: Maximum quarters since last observation (default: 6)
+
+    Returns:
+        Filtered DataFrame excluding stale entities
+
+    Examples:
+        >>> df_active = filter_stale_entities(
+        ...     df, entity_col="service", timestamp_col="date",
+        ...     decline_window_quarters=6
+        ... )
+    """
+    if decline_window_quarters <= 0:
+        return df
+
+    # find last observation per entity
+    last_observation = df.group_by(entity_col).agg(
+        pl.col(timestamp_col).max().alias("last_date")
+    )
+
+    # get dataset max date
+    dataset_max_datetime = df.select(pl.col(timestamp_col).max()).item()
+    dataset_max_date = (
+        dataset_max_datetime.date()
+        if hasattr(dataset_max_datetime, "date")
+        else dataset_max_datetime
+    )
+
+    # calculate cutoff date
+    decline_cutoff = dataset_max_date - timedelta(
+        days=decline_window_quarters * 91  # ~91 days per quarter
+    )
+
+    n_before = df.select(entity_col).n_unique()
+
+    # identify stale entities
+    stale_entities = last_observation.filter(
+        pl.col("last_date").cast(pl.Date) < pl.lit(decline_cutoff).cast(pl.Date)
+    )[entity_col]
+
+    # filter them out
+    df = df.filter(~pl.col(entity_col).is_in(stale_entities))
+
+    n_after = df.select(entity_col).n_unique()
+    n_filtered = n_before - n_after
+
+    if n_filtered > 0:
+        print(
+            f"  Filtered {n_filtered} stale entities (inactive >{decline_window_quarters}Q)"
+        )
+
+    return df
+
+
+def filter_sparse_quarters(
+    movement_df: pd.DataFrame,
+    quarter_col: str = "quarter",
+    entity_col: str = "entity",
+    min_entities_per_quarter: int = 3,
+) -> pd.DataFrame:
+    """
+    Remove quarters with too few entity observations.
+
+    Prevents unstable GLMM estimates from sparse time periods.
+    Useful for cumulative movement tracking where early quarters
+    may have insufficient entities.
+
+    Args:
+        movement_df: Pandas DataFrame with movement tracking data
+        quarter_col: Column name for quarter identifier (default: "quarter")
+        entity_col: Column name for entity identifier (default: "entity")
+        min_entities_per_quarter: Minimum entities required (default: 3)
+
+    Returns:
+        Filtered DataFrame with only quarters meeting threshold
+
+    Examples:
+        >>> movement_clean = filter_sparse_quarters(
+        ...     movement_df, min_entities_per_quarter=5
+        ... )
+    """
+    # count unique entities per quarter
+    quarter_counts = movement_df.groupby(quarter_col)[entity_col].nunique()
+
+    # identify valid quarters
+    valid_quarters = quarter_counts[
+        quarter_counts >= min_entities_per_quarter
+    ].index
+
+    # filter
+    filtered_df = movement_df[movement_df[quarter_col].isin(valid_quarters)]
+
+    n_dropped = len(movement_df) - len(filtered_df)
+    n_quarters_dropped = len(quarter_counts) - len(valid_quarters)
+
+    if n_dropped > 0:
+        print(
+            f"  Filtered {n_quarters_dropped} sparse quarters "
+            f"({n_dropped} observations, <{min_entities_per_quarter} entities/quarter)"
+        )
+
+    return filtered_df

priorityx/core/glmm.py

@@ -0,0 +1,377 @@
+"""GLMM estimation for entity prioritization using Poisson mixed models."""
+
+from typing import Dict, Literal, Optional, Tuple
+
+import pandas as pd
+import polars as pl
+from statsmodels.genmod.bayes_mixed_glm import PoissonBayesMixedGLM
+
+# default priors (validated on regulatory data)
+DEFAULT_VCP_P = 3.5  # prior scale for random effects (higher = less shrinkage)
+DEFAULT_FE_P = 3.0  # prior scale for fixed effects
+
+
+def _extract_random_effects(
+    glmm_model, glmm_result
+) -> tuple[dict[str, float], dict[str, float]]:
+    """Extract random intercepts and slopes from statsmodels result."""
+    intercepts: dict[str, float] = {}
+    slopes: dict[str, float] = {}
+    for name, value in zip(glmm_model.vc_names, glmm_result.vc_mean):
+        entity = name.split("[", 1)[1].split("]", 1)[0]
+        val = float(value)
+        if ":time" in name:
+            slopes[entity] = val
+        else:
+            intercepts[entity] = val
+    return intercepts, slopes
+
+
+def fit_priority_matrix(
+    df: pl.LazyFrame | pl.DataFrame,
+    entity_col: str,
+    timestamp_col: str,
+    count_col: Optional[str] = None,
+    date_filter: Optional[str] = None,
+    min_observations: int = 3,
+    min_total_count: int = 0,
+    decline_window_quarters: int = 6,
+    temporal_granularity: Literal["yearly", "quarterly", "semiannual"] = "yearly",
+    vcp_p: float = DEFAULT_VCP_P,
+    fe_p: float = DEFAULT_FE_P,
+) -> Tuple[pd.DataFrame, Dict]:
+    """
+    Fit Poisson GLMM to classify entities into priority quadrants.
+
+    Args:
+        df: Input DataFrame
+        entity_col: Entity identifier column
+        timestamp_col: Date column
+        count_col: Count metric column (defaults to row count)
+        date_filter: Date filter (e.g., "< 2025-01-01")
+        min_observations: Minimum time periods required
+        min_total_count: Minimum total count threshold
+        decline_window_quarters: Filter entities inactive >N quarters
+        temporal_granularity: Time aggregation level
+        vcp_p: Random effects prior scale
+        fe_p: Fixed effects prior scale
+
+    Returns:
+        Tuple of (results DataFrame, statistics dict)
+    """
+    # ensure dataframe
+    if isinstance(df, pl.LazyFrame):
+        df = df.collect()
+
+    # ensure timestamp column is Date type
+    df = df.with_columns(pl.col(timestamp_col).cast(pl.Date))
+
+    # apply date filter if specified
+    if date_filter:
+        date_filter_clean = date_filter.strip()
+
+        # parse filter operators
+        if date_filter_clean.startswith("<="):
+            date_value = date_filter_clean[2:].strip()
+            df = df.filter(
+                pl.col(timestamp_col) <= pl.lit(date_value).cast(pl.Date)
+            )
+        elif date_filter_clean.startswith(">="):
+            date_value = date_filter_clean[2:].strip()
+            df = df.filter(
+                pl.col(timestamp_col) >= pl.lit(date_value).cast(pl.Date)
+            )
+        elif date_filter_clean.startswith("<"):
+            date_value = date_filter_clean[1:].strip()
+            df = df.filter(
+                pl.col(timestamp_col) < pl.lit(date_value).cast(pl.Date)
+            )
+        elif date_filter_clean.startswith(">"):
+            date_value = date_filter_clean[1:].strip()
+            df = df.filter(
+                pl.col(timestamp_col) > pl.lit(date_value).cast(pl.Date)
+            )
+        else:
+            # assume date only (use < for backward compatibility)
+            df = df.filter(
+                pl.col(timestamp_col) < pl.lit(date_filter).cast(pl.Date)
+            )
+
+    # filter by minimum total count if specified
+    n_before_volume_filter = df.select(entity_col).n_unique()
+    if min_total_count > 0:
+        total_counts = df.group_by(entity_col).agg(pl.len().alias("total_count"))
+        valid_entities = total_counts.filter(
+            pl.col("total_count") >= min_total_count
+        )[entity_col]
+        df = df.filter(pl.col(entity_col).is_in(valid_entities))
+        n_after_volume_filter = df.select(entity_col).n_unique()
+        n_filtered_volume = n_before_volume_filter - n_after_volume_filter
+        if n_filtered_volume > 0:
+            print(
+                f"  Filtered {n_filtered_volume} entities (<{min_total_count} total count)"
+            )
+
+    # filter stale entities (decline window)
+    if decline_window_quarters > 0 and temporal_granularity == "quarterly":
+        last_observation = df.group_by(entity_col).agg(
+            pl.col(timestamp_col).max().alias("last_date")
+        )
+
+        from datetime import timedelta
+
+        # use max date in dataset for historical analysis
+        dataset_max_datetime = df.select(pl.col(timestamp_col).max()).item()
+        dataset_max_date = (
+            dataset_max_datetime.date()
+            if hasattr(dataset_max_datetime, "date")
+            else dataset_max_datetime
+        )
+        decline_cutoff = dataset_max_date - timedelta(
+            days=decline_window_quarters * 91  # ~91 days per quarter
+        )
+
+        n_before_decline_filter = df.select(entity_col).n_unique()
+        stale_entities = last_observation.filter(
+            pl.col("last_date").cast(pl.Date) < pl.lit(decline_cutoff).cast(pl.Date)
+        )[entity_col]
+
+        df = df.filter(~pl.col(entity_col).is_in(stale_entities))
+        n_after_decline_filter = df.select(entity_col).n_unique()
+        n_filtered_stale = n_before_decline_filter - n_after_decline_filter
+
+        if n_filtered_stale > 0:
+            print(
+                f"  Filtered {n_filtered_stale} entities (inactive >{decline_window_quarters}Q)"
+            )
+
+    # auto-adjust min_observations for temporal granularity
+    if min_observations == 3 and temporal_granularity == "quarterly":
+        min_observations = 8  # 2 years quarterly
+    elif min_observations == 3 and temporal_granularity == "semiannual":
+        min_observations = 4  # 2 years semiannual
+
+    # prepare aggregation based on temporal granularity
+    if temporal_granularity == "quarterly":
+        df = df.with_columns(
+            [
+                pl.col(timestamp_col).dt.year().alias("year"),
+                pl.col(timestamp_col).dt.quarter().alias("quarter"),
+            ]
+        )
+
+        if count_col:
+            df_prepared = (
+                df.group_by(["year", "quarter", entity_col])
+                .agg(pl.col(count_col).sum().alias("count"))
+                .sort(["year", "quarter", entity_col])
+            )
+        else:
+            df_prepared = (
+                df.group_by(["year", "quarter", entity_col])
+                .agg(pl.len().alias("count"))
+                .sort(["year", "quarter", entity_col])
+            )
+
+    elif temporal_granularity == "semiannual":
+        df = df.with_columns(
+            [
+                pl.col(timestamp_col).dt.year().alias("year"),
+                pl.col(timestamp_col).dt.quarter().alias("quarter"),
+            ]
+        ).with_columns(
+            # semester: Q1-Q2 = 1, Q3-Q4 = 2
+            pl.when(pl.col("quarter") <= 2)
+            .then(pl.lit(1))
+            .otherwise(pl.lit(2))
+            .alias("semester")
+        )
+
+        if count_col:
+            df_prepared = (
+                df.group_by(["year", "semester", entity_col])
+                .agg(pl.col(count_col).sum().alias("count"))
+                .sort(["year", "semester", entity_col])
+            )
+        else:
+            df_prepared = (
+                df.group_by(["year", "semester", entity_col])
+                .agg(pl.len().alias("count"))
+                .sort(["year", "semester", entity_col])
+            )
+
+    else:  # yearly
+        df = df.with_columns(pl.col(timestamp_col).dt.year().alias("year"))
+
+        if count_col:
+            df_prepared = (
+                df.group_by(["year", entity_col])
+                .agg(pl.col(count_col).sum().alias("count"))
+                .sort(["year", entity_col])
+            )
+        else:
+            df_prepared = (
+                df.group_by(["year", entity_col])
+                .agg(pl.len().alias("count"))
+                .sort(["year", entity_col])
+            )
+
+    # filter entities with sufficient observations
+    if min_observations > 0:
+        entity_counts = df_prepared.group_by(entity_col).agg(
+            pl.len().alias("n_periods")
+        )
+        valid_entities = entity_counts.filter(
+            pl.col("n_periods") >= min_observations
+        )[entity_col]
+        df_prepared = df_prepared.filter(pl.col(entity_col).is_in(valid_entities))
+
+    # ensure count is integer
+    df_prepared = df_prepared.with_columns(pl.col("count").cast(pl.Int64))
+
+    # create time variable based on temporal granularity
+    if temporal_granularity == "quarterly":
+        # continuous quarterly time: year + (quarter-1)/4
+        df_prepared = df_prepared.with_columns(
+            (pl.col("year") + (pl.col("quarter") - 1) / 4).alias("time_continuous")
+        )
+
+        # center for numerical stability
+        mean_time = df_prepared["time_continuous"].mean()
+        df_prepared = df_prepared.with_columns(
+            (pl.col("time_continuous") - mean_time).alias("time")
+        )
+
+    elif temporal_granularity == "semiannual":
+        # continuous semiannual time: year + (semester-1)/2
+        df_prepared = df_prepared.with_columns(
+            (pl.col("year") + (pl.col("semester") - 1) / 2).alias("time_continuous")
+        )
+
+        # center for numerical stability
+        mean_time = df_prepared["time_continuous"].mean()
+        df_prepared = df_prepared.with_columns(
+            (pl.col("time_continuous") - mean_time).alias("time")
+        )
+
+    else:  # yearly
+        # center year for numerical stability
+        mean_year = df_prepared["year"].mean()
+        df_prepared = df_prepared.with_columns(
+            (pl.col("year") - mean_year).alias("time")
+        )
+
+    # ensure categorical type for entity
+    df_prepared = df_prepared.with_columns(pl.col(entity_col).cast(pl.Categorical))
+
+    # make period categorical for seasonal effects
+    if temporal_granularity == "quarterly":
+        df_prepared = df_prepared.with_columns(pl.col("quarter").cast(pl.Categorical))
+    elif temporal_granularity == "semiannual":
+        df_prepared = df_prepared.with_columns(pl.col("semester").cast(pl.Categorical))
+
+    # ensure positive counts for poisson
+    df_prepared = df_prepared.filter(pl.col("count") > 0)
+
+    # convert to pandas for statsmodels
+    df_pandas = df_prepared.to_pandas()
+    df_pandas["time"] = df_pandas["time"].astype(float)
+    df_pandas[entity_col] = df_pandas[entity_col].astype("category")
+
+    # build fixed-effect formula with seasonal dummies
+    formula = "count ~ time"
+    if temporal_granularity == "quarterly":
+        df_pandas["quarter"] = df_pandas["quarter"].astype("category")
+        formula += " + C(quarter)"
+    elif temporal_granularity == "semiannual":
+        df_pandas["semester"] = df_pandas["semester"].astype("category")
+        formula += " + C(semester)"
+
+    # random effects: intercept + slope per entity
+    random_formulas = {
+        "re_int": f"0 + C({entity_col})",
+        "re_slope": f"0 + C({entity_col}):time",
+    }
+
+    # fit poisson bayesian mixed model
+    glmm_model = PoissonBayesMixedGLM.from_formula(
+        formula,
+        random_formulas,
+        df_pandas,
+        vcp_p=vcp_p,
+        fe_p=fe_p,
+    )
+
+    # use variational bayes (returns posterior mean)
+    # avoids boundary convergence issues vs map
+    glmm_result = glmm_model.fit_vb()
+
+    # extract random effects
+    intercepts_dict, slopes_dict = _extract_random_effects(glmm_model, glmm_result)
+
+    # convert to lists for dataframe
+    entities = list(intercepts_dict.keys())
+    intercepts = [intercepts_dict[ent] for ent in entities]
+    slopes = [slopes_dict[ent] for ent in entities]
+
+    # create results dataframe
+    df_random_effects = pl.DataFrame(
+        {"entity": entities, "Random_Intercept": intercepts, "Random_Slope": slopes}
+    )
+
+    # calculate totals from original filtered data
+    df_totals = (
+        df.group_by(entity_col)
+        .agg(pl.len().alias("count"))
+        .sort(entity_col)
+    )
+
+    # convert to pandas for merging
+    df_random_effects_pd = df_random_effects.to_pandas()
+    df_totals_pd = df_totals.to_pandas()
+
+    # merge
+    results_df = df_random_effects_pd.merge(
+        df_totals_pd, left_on="entity", right_on=entity_col, how="left"
+    )
+
+    # import quadrant classifier
+    from .quadrants import classify_quadrant
+
+    # add quadrant classification
+    results_df["quadrant"] = results_df.apply(
+        lambda row: classify_quadrant(
+            row["Random_Intercept"],
+            row["Random_Slope"],
+            count=row.get("count"),
+            min_q1_count=50,  # crisis threshold
+        ),
+        axis=1,
+    )
+
+    # model statistics
+    model_stats = {
+        "n_entities": len(results_df),
+        "n_observations": len(df_prepared),
+        "method": "VB",
+        "vcp_p": vcp_p,
+        "fe_p": fe_p,
+        "temporal_granularity": temporal_granularity,
+    }
+
+    # add fixed effects if available
+    try:
+        if hasattr(glmm_result, "params"):
+            params = glmm_result.params
+            model_stats["fixed_intercept"] = float(
+                params.get("Intercept", params.get("(Intercept)", 0.0))
+            )
+            model_stats["fixed_slope"] = float(params.get("time", 0.0))
+        else:
+            model_stats["fixed_intercept"] = None
+            model_stats["fixed_slope"] = None
+    except Exception:
+        model_stats["fixed_intercept"] = None
+        model_stats["fixed_slope"] = None
+
+    return results_df, model_stats

priorityx/core/quadrants.py

@@ -0,0 +1,60 @@
+"""Quadrant classification for priority matrices."""
+
+
+def classify_quadrant(
+    intercept: float,
+    slope: float,
+    count: float = None,
+    min_q1_count: int = 50
+) -> str:
+    """
+    Classify entity into priority quadrant.
+
+    Args:
+        intercept: Random intercept (volume indicator)
+        slope: Random slope (growth indicator)
+        count: Absolute count for Q1 threshold
+        min_q1_count: Minimum count for Crisis classification
+
+    Returns:
+        Quadrant code: Q1 (Crisis), Q2 (Investigate), Q3 (Monitor), Q4 (Low)
+    """
+    if intercept > 0 and slope > 0:
+        # would be Q1, but check absolute count threshold
+        if count is not None and count < min_q1_count:
+            return "Q2"  # high growth but insufficient count → investigate
+        return "Q1"  # crisis
+    elif intercept <= 0 and slope > 0:
+        return "Q2"  # investigate
+    elif intercept <= 0 and slope <= 0:
+        return "Q3"  # monitor
+    else:
+        return "Q4"  # low priority
+
+
+def get_quadrant_label(quadrant_code: str, x_label: str = "Volume", y_label: str = "Growth") -> str:
+    """Get human-readable label for quadrant code.
+
+    Args:
+        quadrant_code: Q1-Q4
+        x_label: Label for X-axis metric (e.g., "Volume", "Market Share", "Bugs")
+        y_label: Label for Y-axis metric (e.g., "Growth", "Incident Rate")
+    """
+    labels = {
+        "Q1": f"Q1 (High {x_label}, High {y_label})",
+        "Q2": f"Q2 (Low {x_label}, High {y_label})",
+        "Q3": f"Q3 (Low {x_label}, Low {y_label})",
+        "Q4": f"Q4 (High {x_label}, Low {y_label})",
+    }
+    return labels.get(quadrant_code, quadrant_code)
+
+
+def get_risk_level(quadrant_code: str) -> str:
+    """Map quadrant to risk level."""
+    risk_map = {
+        "Q1": "Critical",
+        "Q2": "Investigate",
+        "Q3": "Monitor",
+        "Q4": "Low",
+    }
+    return risk_map.get(quadrant_code, "Unknown")