batch-analytics 0.3.24__tar.gz → 0.3.27__tar.gz

{batch_analytics-0.3.24 → batch_analytics-0.3.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.24
+Version: 0.3.27
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.24 → batch_analytics-0.3.27}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.24"
+version = "0.3.27"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.3.24 → batch_analytics-0.3.27}/src/batch_analytics/analytics/t_test.py

@@ -146,7 +146,10 @@ def _run_one_way_anova(
     N = sum(g["n"] for g in groups)
     if N <= k:
         raise ValueError(
-            f"ANOVA needs more observations than groups (N={N}, k={k})"
+            f"ANOVA needs total observations N > number of groups k (got N={N}, k={k}). "
+            "Common cause: exactly one row per group (e.g. one raw-material charge per batch per material), "
+            "so there is no within-group residual. Use data with replicates per group, a different group column, "
+            "or compare batches using a table with many rows per batch (e.g. operations or equipment_usage)."
         )
 
     grand_mean = sum(g["n"] * g["mean"] for g in groups) / N

{batch_analytics-0.3.24 → batch_analytics-0.3.27}/src/batch_analytics/config.py

@@ -92,6 +92,10 @@ class TransformConfig:
     add_dimension_column: str = os.environ.get("BATCH_ADD_DIMENSION_COLUMN", "add_dimension")
     # Legacy: no longer used; output column names match JSON keys (e.g. anchor_id). Kept for env compatibility.
     anchor_id_column: str = os.environ.get("BATCH_ANCHOR_ID_COLUMN", "anchor_id")
+    # JSON object: {"new_col": "Spark SQL expression"} applied after KV expansion, before dedupe.
+    expr_columns_json: str = os.environ.get("BATCH_TRANSFORM_EXPR_COLUMNS", "").strip()
+    # JSON pivot spec (group_by, pivot_column, value_column, agg, optional pivot_values, column_name_prefix).
+    pivot_json: str = os.environ.get("BATCH_TRANSFORM_PIVOT_JSON", "").strip()
 
 
 @dataclass

{batch_analytics-0.3.24 → batch_analytics-0.3.27}/src/batch_analytics/transform.py

@@ -10,7 +10,7 @@ import re
 from typing import Any, Dict, List, Optional, Sequence, Set
 
 from pyspark.sql import DataFrame, SparkSession
-from pyspark.sql.functions import col, explode, map_keys, udf
+from pyspark.sql.functions import col, explode, expr as spark_expr, map_keys, udf
 from pyspark.sql.types import MapType, StringType
 
 from .config import BatchAnalyticsConfig
@@ -165,16 +165,141 @@ def remove_duplicates(
     return df_cleaned
 
 
+def apply_spark_expr_columns(
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    """
+    Add or replace columns from Spark SQL expressions (``BATCH_TRANSFORM_EXPR_COLUMNS``).
+
+    Value must be a JSON object mapping **output column name** → **expression** (same dialect as
+    ``selectExpr``), e.g. ``{"y": "cast(duration_minutes as double)", "x": "cast(regexp_extract(operation_id, '([0-9]+)$', 1) as double)"}`` (Java regex; prefer trailing digits to avoid hyphen/range issues in patterns like ``ETC-1-OP-…``).
+    """
+    raw = (config.transform.expr_columns_json or "").strip()
+    if not raw:
+        return df
+    try:
+        mapping = json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise ValueError(
+            f"BATCH_TRANSFORM_EXPR_COLUMNS must be valid JSON object: {e}"
+        ) from e
+    if not isinstance(mapping, dict):
+        raise ValueError("BATCH_TRANSFORM_EXPR_COLUMNS must be a JSON object of column -> sql_expr")
+    out = df
+    for name, sql in mapping.items():
+        col_name = str(name).strip()
+        expr_sql = str(sql).strip()
+        if not col_name or not expr_sql:
+            continue
+        out = out.withColumn(col_name, spark_expr(expr_sql))
+    return out
+
+
+def _pivot_output_column_name(prefix: str, pivot_value: str) -> str:
+    """Stable identifier for pivoted columns (e.g. imp_rm_001 from RM-001)."""
+    p = (prefix or "v").rstrip("_")
+    body = str(pivot_value).replace("-", "_").replace(" ", "_").lower()
+    return f"{p}_{body}"
+
+
+def apply_pivot(
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    """
+    GroupBy + pivot + aggregate (``BATCH_TRANSFORM_PIVOT_JSON``).
+
+    Example::
+
+        {
+          "group_by": "batch_id",
+          "pivot_column": "material_id",
+          "value_column": "total_impurities_pct",
+          "agg": "max",
+          "pivot_values": ["RM-001", "RM-002"],
+          "column_name_prefix": "imp",
+          "rename_pivot_columns": true
+        }
+
+    If ``pivot_values`` is omitted, distinct values are collected (sorted by string order).
+    """
+    raw = (config.transform.pivot_json or "").strip()
+    if not raw:
+        return df
+    try:
+        spec = json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"BATCH_TRANSFORM_PIVOT_JSON must be valid JSON: {e}") from e
+    if not isinstance(spec, dict):
+        raise ValueError("BATCH_TRANSFORM_PIVOT_JSON must be a JSON object")
+
+    group_by = spec.get("group_by") or spec.get("groupBy")
+    pivot_col = (spec.get("pivot_column") or "").strip()
+    value_col = (spec.get("value_column") or "").strip()
+    if not group_by or not pivot_col or not value_col:
+        raise ValueError("pivot spec requires group_by, pivot_column, value_column")
+    agg_name = (spec.get("agg") or "max").strip().lower()
+    prefix = (spec.get("column_name_prefix") or "v").strip() or "v"
+    rename_pivot = spec.get("rename_pivot_columns", True)
+
+    from pyspark.sql import functions as F
+
+    gcols = [c.strip() for c in str(group_by).split(",") if c.strip()]
+    for c in gcols + [pivot_col, value_col]:
+        if c not in df.columns:
+            raise ValueError(f"pivot: column {c!r} not in dataframe; have {df.columns}")
+
+    vc = F.col(value_col)
+    if agg_name == "max":
+        agg_expr = F.max(vc)
+    elif agg_name == "min":
+        agg_expr = F.min(vc)
+    elif agg_name in ("sum",):
+        agg_expr = F.sum(vc)
+    elif agg_name in ("avg", "mean"):
+        agg_expr = F.avg(vc)
+    elif agg_name == "first":
+        agg_expr = F.first(vc)
+    else:
+        raise ValueError(
+            f"pivot agg must be one of max,min,sum,avg,mean,first; got {agg_name!r}"
+        )
+
+    pivot_values = spec.get("pivot_values")
+    if pivot_values is not None:
+        pv: List[str] = [str(v) for v in pivot_values]
+    else:
+        pv = [r[0] for r in df.select(pivot_col).distinct().sort(pivot_col).collect()]
+        logger.info("pivot: inferred %d distinct %s values", len(pv), pivot_col)
+
+    out = df.groupBy(*gcols).pivot(pivot_col, pv).agg(agg_expr)
+
+    if rename_pivot:
+        for v in pv:
+            old = v
+            if old not in out.columns:
+                continue
+            new_name = _pivot_output_column_name(prefix, old)
+            if new_name != old:
+                out = out.withColumnRenamed(old, new_name)
+    return out
+
+
 def transform(
     df: DataFrame,
     config: BatchAnalyticsConfig,
 ) -> DataFrame:
     """
     Apply transformation only: (1) expand JSON/KV blob column into one column per top-level key,
-    (2) deduplicate by BATCH_DEDUP_COLUMNS if set, else by full row.
+    (2) optional Spark SQL expression columns (``BATCH_TRANSFORM_EXPR_COLUMNS``),
+    (3) optional groupBy+pivot (``BATCH_TRANSFORM_PIVOT_JSON``),
+    (4) deduplicate by BATCH_DEDUP_COLUMNS if set, else by full row.
     Does not write anywhere. Use stage_to_clickhouse() separately to persist.
     """
     transformed = expand_kv_blob_column(df, config)
+    transformed = apply_spark_expr_columns(transformed, config)
+    transformed = apply_pivot(transformed, config)
     dedup_cols = (
         [c.strip() for c in config.transform.dedup_columns.split(",") if c.strip()]
         if config.transform.dedup_columns

{batch_analytics-0.3.24 → batch_analytics-0.3.27}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.24
+Version: 0.3.27
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT