batch-analytics 0.3.24__tar.gz → 0.3.26__tar.gz

{batch_analytics-0.3.24 → batch_analytics-0.3.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.24
+Version: 0.3.26
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.24 → batch_analytics-0.3.26}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.24"
+version = "0.3.26"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.3.24 → batch_analytics-0.3.26}/src/batch_analytics/analytics/t_test.py

@@ -146,7 +146,10 @@ def _run_one_way_anova(
     N = sum(g["n"] for g in groups)
     if N <= k:
         raise ValueError(
-            f"ANOVA needs more observations than groups (N={N}, k={k})"
+            f"ANOVA needs total observations N > number of groups k (got N={N}, k={k}). "
+            "Common cause: exactly one row per group (e.g. one raw-material charge per batch per material), "
+            "so there is no within-group residual. Use data with replicates per group, a different group column, "
+            "or compare batches using a table with many rows per batch (e.g. operations or equipment_usage)."
         )
 
     grand_mean = sum(g["n"] * g["mean"] for g in groups) / N

{batch_analytics-0.3.24 → batch_analytics-0.3.26}/src/batch_analytics/config.py

@@ -92,6 +92,8 @@ class TransformConfig:
     add_dimension_column: str = os.environ.get("BATCH_ADD_DIMENSION_COLUMN", "add_dimension")
     # Legacy: no longer used; output column names match JSON keys (e.g. anchor_id). Kept for env compatibility.
     anchor_id_column: str = os.environ.get("BATCH_ANCHOR_ID_COLUMN", "anchor_id")
+    # JSON object: {"new_col": "Spark SQL expression"} applied after KV expansion, before dedupe.
+    expr_columns_json: str = os.environ.get("BATCH_TRANSFORM_EXPR_COLUMNS", "").strip()
 
 
 @dataclass

{batch_analytics-0.3.24 → batch_analytics-0.3.26}/src/batch_analytics/transform.py

@@ -10,7 +10,7 @@ import re
 from typing import Any, Dict, List, Optional, Sequence, Set
 
 from pyspark.sql import DataFrame, SparkSession
-from pyspark.sql.functions import col, explode, map_keys, udf
+from pyspark.sql.functions import col, explode, expr as spark_expr, map_keys, udf
 from pyspark.sql.types import MapType, StringType
 
 from .config import BatchAnalyticsConfig
@@ -165,16 +165,49 @@ def remove_duplicates(
     return df_cleaned
 
 
+def apply_spark_expr_columns(
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    """
+    Add or replace columns from Spark SQL expressions (``BATCH_TRANSFORM_EXPR_COLUMNS``).
+
+    Value must be a JSON object mapping **output column name** → **expression** (same dialect as
+    ``selectExpr``), e.g. ``{"y": "cast(duration_minutes as double)", "x": "cast(regexp_extract(operation_id, '([0-9]+)$', 1) as double)"}`` (Java regex; prefer trailing digits to avoid hyphen/range issues in patterns like ``ETC-1-OP-…``).
+    """
+    raw = (config.transform.expr_columns_json or "").strip()
+    if not raw:
+        return df
+    try:
+        mapping = json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise ValueError(
+            f"BATCH_TRANSFORM_EXPR_COLUMNS must be valid JSON object: {e}"
+        ) from e
+    if not isinstance(mapping, dict):
+        raise ValueError("BATCH_TRANSFORM_EXPR_COLUMNS must be a JSON object of column -> sql_expr")
+    out = df
+    for name, sql in mapping.items():
+        col_name = str(name).strip()
+        expr_sql = str(sql).strip()
+        if not col_name or not expr_sql:
+            continue
+        out = out.withColumn(col_name, spark_expr(expr_sql))
+    return out
+
+
 def transform(
     df: DataFrame,
     config: BatchAnalyticsConfig,
 ) -> DataFrame:
     """
     Apply transformation only: (1) expand JSON/KV blob column into one column per top-level key,
-    (2) deduplicate by BATCH_DEDUP_COLUMNS if set, else by full row.
+    (2) optional Spark SQL expression columns (``BATCH_TRANSFORM_EXPR_COLUMNS``),
+    (3) deduplicate by BATCH_DEDUP_COLUMNS if set, else by full row.
     Does not write anywhere. Use stage_to_clickhouse() separately to persist.
     """
     transformed = expand_kv_blob_column(df, config)
+    transformed = apply_spark_expr_columns(transformed, config)
     dedup_cols = (
         [c.strip() for c in config.transform.dedup_columns.split(",") if c.strip()]
         if config.transform.dedup_columns

{batch_analytics-0.3.24 → batch_analytics-0.3.26}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.24
+Version: 0.3.26
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT