batch-analytics 0.3.5__tar.gz → 0.3.7__tar.gz

{batch_analytics-0.3.5 → batch_analytics-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.5
+Version: 0.3.7
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
@@ -8,6 +8,7 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 Requires-Dist: pyspark<3.6,>=3.4
 Requires-Dist: numpy>=1.19.0
+Requires-Dist: scipy>=1.5.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Provides-Extra: ttest
@@ -65,7 +66,7 @@ analytics/
 pip install -e .
 # or install every runtime dependency used anywhere in the package, then editable:
 pip install -r requirements.txt && pip install -e .
-# PyPI wheel pulls numpy automatically (pyspark.ml); extras: ttest, s3, clickhouse, output, full
+# PyPI install includes numpy and scipy (t-test); extras: s3, clickhouse, output, full
 pip install "batch-analytics[full]"
 ```
 

{batch_analytics-0.3.5 → batch_analytics-0.3.7}/README.md

@@ -36,7 +36,7 @@ analytics/
 pip install -e .
 # or install every runtime dependency used anywhere in the package, then editable:
 pip install -r requirements.txt && pip install -e .
-# PyPI wheel pulls numpy automatically (pyspark.ml); extras: ttest, s3, clickhouse, output, full
+# PyPI install includes numpy and scipy (t-test); extras: s3, clickhouse, output, full
 pip install "batch-analytics[full]"
 ```
 

{batch_analytics-0.3.5 → batch_analytics-0.3.7}/pyproject.toml

@@ -4,20 +4,22 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.5"
+version = "0.3.7"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
     "pyspark>=3.4,<3.6",
     "numpy>=1.19.0",
+    # Welch t-test (t_test.py); keep on core deps so `pip install batch-analytics` works in minimal driver images
+    "scipy>=1.5.0",
 ]
 authors = [{ name = "Litewave Analytics Team" }]
 license = { text = "MIT" }
 
 [project.optional-dependencies]
 dev = ["pytest>=7.0"]
-# Welch t-test module (batch_analytics.analytics.t_test)
+# Legacy: scipy is a core dependency; kept so `pip install "batch-analytics[ttest]"` still resolves.
 ttest = ["scipy>=1.5.0"]
 s3 = ["boto3>=1.28"]
 # 0.9+ uses list[...] etc. and breaks on Python 3.8; 3.9+ can take current clickhouse-connect.

{batch_analytics-0.3.5 → batch_analytics-0.3.7}/src/batch_analytics/analytics/t_test.py

@@ -1,9 +1,9 @@
 """
-Module 4: T-test to compare means of two sets of data.
+Module 4: T-test and one-way ANOVA for comparing means across groups.
 """
 
 import logging
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.functions import col, avg, stddev, count
@@ -20,18 +20,17 @@ def run_t_test(
     config: BatchAnalyticsConfig,
 ) -> Dict[str, Any]:
     """
-    Perform an independent samples t-test to compare the means of two groups.
+    Compare means across groups or two numeric columns.
 
-    Supports two modes:
-    1. Value + group: one numeric column, one categorical column with 2 levels
-    2. Two columns: two numeric columns, compare their means
-
-    Uses Welch's t-test (does not assume equal variances).
+    Supports:
+    1. Value + group: one numeric column, one categorical column.
+       - **2 groups:** Welch's t-test (unequal variances).
+       - **3+ groups:** one-way ANOVA (F-test on equal means).
+    2. Two columns: two numeric columns, Welch t-test on their column means.
 
     Returns:
-        - group_a, group_b: names/summary of each group
-        - mean_a, mean_b, std_a, std_b, n_a, n_b
-        - t_statistic, p_value, difference (mean_a - mean_b)
+        For Welch: ``group_a``, ``group_b``, ``t_statistic``, ``p_value``, ``test`` = ``\"Welch\"``.
+        For ANOVA: ``groups``, ``f_statistic``, ``p_value``, ``test`` = ``\"one_way_anova\"``, SS/df.
     """
     value_col = (config.analytics.ttest_value_column or "").strip()
     group_col = (config.analytics.ttest_group_column or "").strip()
@@ -46,7 +45,7 @@ def run_t_test(
     if col_a and col_b and col_a in df.columns and col_b in df.columns:
         return _run_t_test_two_columns(df, col_a, col_b)
 
-    # Fallback: find first numeric + first string-like column with 2 distinct values
+    # Fallback: first numeric + string-like column with at least 2 distinct groups
     numeric_cols = [
         f.name for f in df.schema.fields
         if "double" in str(f.dataType).lower()
@@ -60,8 +59,10 @@ def run_t_test(
     for nc in numeric_cols:
         for sc in string_cols:
             distinct = df.select(sc).distinct().count()
-            if distinct == 2:
-                logger.info("Auto-selected t-test: value=%s, group=%s", nc, sc)
+            if distinct >= 2:
+                logger.info(
+                    "Auto-selected value=%s, group=%s (%d groups)", nc, sc, distinct
+                )
                 return _run_t_test_by_group(df, nc, sc)
 
     raise ValueError(
@@ -76,7 +77,7 @@ def _run_t_test_by_group(
     value_col: str,
     group_col: str,
 ) -> Dict[str, Any]:
-    """T-test: compare mean of value_col across two levels of group_col."""
+    """Compare mean of value_col across groups: Welch (2 groups) or one-way ANOVA (3+)."""
     df_num = df.select(
         col(value_col).cast(DoubleType()).alias("_val"),
         col(group_col).cast("string").alias("_grp"),
@@ -92,11 +93,15 @@ def _run_t_test_by_group(
         .collect()
     )
 
-    if len(stats) != 2:
+    k = len(stats)
+    if k < 2:
         raise ValueError(
-            f"T-test requires exactly 2 groups in {group_col}. Found: {[r['_grp'] for r in stats]}"
+            f"Need at least 2 groups in {group_col} for comparison. Found: {[r['_grp'] for r in stats]}"
         )
 
+    if k > 2:
+        return _run_one_way_anova(stats, value_col, group_col)
+
     r0, r1 = stats[0], stats[1]
     return _compute_t_test_result(
         group_a=r0["_grp"],
@@ -110,6 +115,98 @@ def _run_t_test_by_group(
     )
 
 
+def _run_one_way_anova(
+    stats_rows: List[Any],
+    value_col: str,
+    group_col: str,
+) -> Dict[str, Any]:
+    """
+    One-way ANOVA from per-group mean, sample stddev, and n (Spark ``stddev`` uses ddof=1).
+
+    SS_within = sum_i (n_i - 1) * s_i^2
+    SS_between = sum_i n_i * (mean_i - grand_mean)^2
+    """
+    try:
+        from scipy import stats as scipy_stats
+    except ImportError:
+        raise ImportError("ANOVA requires scipy. Install with: pip install scipy")
+
+    groups: List[Dict[str, Any]] = []
+    for r in stats_rows:
+        name = r["_grp"]
+        mean = float(r["mean"])
+        std_raw = r["std"]
+        std = 0.0 if std_raw is None else float(std_raw)
+        n = int(r["n"])
+        groups.append(
+            {"name": name, "mean": mean, "std": std, "n": n}
+        )
+
+    k = len(groups)
+    N = sum(g["n"] for g in groups)
+    if N <= k:
+        raise ValueError(
+            f"ANOVA needs more observations than groups (N={N}, k={k})"
+        )
+
+    grand_mean = sum(g["n"] * g["mean"] for g in groups) / N
+    ss_between = sum(g["n"] * (g["mean"] - grand_mean) ** 2 for g in groups)
+    ss_within = sum(
+        (g["n"] - 1) * (g["std"] ** 2) for g in groups if g["n"] > 1
+    )
+
+    df_between = k - 1
+    df_within = N - k
+    if df_within <= 0:
+        raise ValueError(
+            f"ANOVA: df_within must be positive (N={N}, k={k})"
+        )
+
+    ms_between = ss_between / df_between
+    ms_within = ss_within / df_within if df_within > 0 else 0.0
+
+    if ms_within <= 0.0:
+        if ms_between <= 0.0:
+            f_stat = 0.0
+            p_value = 1.0
+        else:
+            f_stat = float("inf")
+            p_value = 0.0
+    else:
+        f_stat = ms_between / ms_within
+        p_value = float(scipy_stats.f.sf(f_stat, df_between, df_within))
+
+    out: Dict[str, Any] = {
+        "test": "one_way_anova",
+        "value_column": value_col,
+        "group_column": group_col,
+        "k_groups": k,
+        "n_total": N,
+        "grand_mean": grand_mean,
+        "f_statistic": f_stat,
+        "p_value": p_value,
+        "df_between": df_between,
+        "df_within": df_within,
+        "ss_between": ss_between,
+        "ss_within": ss_within,
+        "ms_between": ms_between,
+        "ms_within": ms_within,
+        "groups": [
+            {
+                "name": g["name"],
+                "mean": g["mean"],
+                "std": g["std"],
+                "n": g["n"],
+            }
+            for g in sorted(groups, key=lambda x: str(x["name"]))
+        ],
+    }
+    if f_stat == float("inf"):
+        out["f_statistic"] = None
+        out["f_statistic_note"] = "infinite (MS_within == 0, reject equal means if SS_between > 0)"
+    return out
+
+
 def _run_t_test_two_columns(df: DataFrame, col_a: str, col_b: str) -> Dict[str, Any]:
     """T-test: compare means of two numeric columns."""
     df_num = df.select(

{batch_analytics-0.3.5 → batch_analytics-0.3.7}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.5
+Version: 0.3.7
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
@@ -8,6 +8,7 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 Requires-Dist: pyspark<3.6,>=3.4
 Requires-Dist: numpy>=1.19.0
+Requires-Dist: scipy>=1.5.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Provides-Extra: ttest
@@ -65,7 +66,7 @@ analytics/
 pip install -e .
 # or install every runtime dependency used anywhere in the package, then editable:
 pip install -r requirements.txt && pip install -e .
-# PyPI wheel pulls numpy automatically (pyspark.ml); extras: ttest, s3, clickhouse, output, full
+# PyPI install includes numpy and scipy (t-test); extras: s3, clickhouse, output, full
 pip install "batch-analytics[full]"
 ```
 

{batch_analytics-0.3.5 → batch_analytics-0.3.7}/src/batch_analytics.egg-info/requires.txt

@@ -1,5 +1,6 @@
 pyspark<3.6,>=3.4
 numpy>=1.19.0
+scipy>=1.5.0
 
 [clickhouse]