batch-analytics 0.3.26__tar.gz → 0.3.27__tar.gz

{batch_analytics-0.3.26 → batch_analytics-0.3.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.26
+Version: 0.3.27
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.26 → batch_analytics-0.3.27}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.26"
+version = "0.3.27"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.3.26 → batch_analytics-0.3.27}/src/batch_analytics/config.py

@@ -94,6 +94,8 @@ class TransformConfig:
     anchor_id_column: str = os.environ.get("BATCH_ANCHOR_ID_COLUMN", "anchor_id")
     # JSON object: {"new_col": "Spark SQL expression"} applied after KV expansion, before dedupe.
     expr_columns_json: str = os.environ.get("BATCH_TRANSFORM_EXPR_COLUMNS", "").strip()
+    # JSON pivot spec (group_by, pivot_column, value_column, agg, optional pivot_values, column_name_prefix).
+    pivot_json: str = os.environ.get("BATCH_TRANSFORM_PIVOT_JSON", "").strip()
 
 
 @dataclass

{batch_analytics-0.3.26 → batch_analytics-0.3.27}/src/batch_analytics/transform.py

@@ -196,6 +196,96 @@ def apply_spark_expr_columns(
     return out
 
 
+def _pivot_output_column_name(prefix: str, pivot_value: str) -> str:
+    """Stable identifier for pivoted columns (e.g. imp_rm_001 from RM-001)."""
+    p = (prefix or "v").rstrip("_")
+    body = str(pivot_value).replace("-", "_").replace(" ", "_").lower()
+    return f"{p}_{body}"
+
+
+def apply_pivot(
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    """
+    GroupBy + pivot + aggregate (``BATCH_TRANSFORM_PIVOT_JSON``).
+
+    Example::
+
+        {
+          "group_by": "batch_id",
+          "pivot_column": "material_id",
+          "value_column": "total_impurities_pct",
+          "agg": "max",
+          "pivot_values": ["RM-001", "RM-002"],
+          "column_name_prefix": "imp",
+          "rename_pivot_columns": true
+        }
+
+    If ``pivot_values`` is omitted, distinct values are collected (sorted by string order).
+    """
+    raw = (config.transform.pivot_json or "").strip()
+    if not raw:
+        return df
+    try:
+        spec = json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"BATCH_TRANSFORM_PIVOT_JSON must be valid JSON: {e}") from e
+    if not isinstance(spec, dict):
+        raise ValueError("BATCH_TRANSFORM_PIVOT_JSON must be a JSON object")
+
+    group_by = spec.get("group_by") or spec.get("groupBy")
+    pivot_col = (spec.get("pivot_column") or "").strip()
+    value_col = (spec.get("value_column") or "").strip()
+    if not group_by or not pivot_col or not value_col:
+        raise ValueError("pivot spec requires group_by, pivot_column, value_column")
+    agg_name = (spec.get("agg") or "max").strip().lower()
+    prefix = (spec.get("column_name_prefix") or "v").strip() or "v"
+    rename_pivot = spec.get("rename_pivot_columns", True)
+
+    from pyspark.sql import functions as F
+
+    gcols = [c.strip() for c in str(group_by).split(",") if c.strip()]
+    for c in gcols + [pivot_col, value_col]:
+        if c not in df.columns:
+            raise ValueError(f"pivot: column {c!r} not in dataframe; have {df.columns}")
+
+    vc = F.col(value_col)
+    if agg_name == "max":
+        agg_expr = F.max(vc)
+    elif agg_name == "min":
+        agg_expr = F.min(vc)
+    elif agg_name in ("sum",):
+        agg_expr = F.sum(vc)
+    elif agg_name in ("avg", "mean"):
+        agg_expr = F.avg(vc)
+    elif agg_name == "first":
+        agg_expr = F.first(vc)
+    else:
+        raise ValueError(
+            f"pivot agg must be one of max,min,sum,avg,mean,first; got {agg_name!r}"
+        )
+
+    pivot_values = spec.get("pivot_values")
+    if pivot_values is not None:
+        pv: List[str] = [str(v) for v in pivot_values]
+    else:
+        pv = [r[0] for r in df.select(pivot_col).distinct().sort(pivot_col).collect()]
+        logger.info("pivot: inferred %d distinct %s values", len(pv), pivot_col)
+
+    out = df.groupBy(*gcols).pivot(pivot_col, pv).agg(agg_expr)
+
+    if rename_pivot:
+        for v in pv:
+            old = v
+            if old not in out.columns:
+                continue
+            new_name = _pivot_output_column_name(prefix, old)
+            if new_name != old:
+                out = out.withColumnRenamed(old, new_name)
+    return out
+
+
 def transform(
     df: DataFrame,
     config: BatchAnalyticsConfig,
@@ -203,11 +293,13 @@ def transform(
     """
     Apply transformation only: (1) expand JSON/KV blob column into one column per top-level key,
     (2) optional Spark SQL expression columns (``BATCH_TRANSFORM_EXPR_COLUMNS``),
-    (3) deduplicate by BATCH_DEDUP_COLUMNS if set, else by full row.
+    (3) optional groupBy+pivot (``BATCH_TRANSFORM_PIVOT_JSON``),
+    (4) deduplicate by BATCH_DEDUP_COLUMNS if set, else by full row.
     Does not write anywhere. Use stage_to_clickhouse() separately to persist.
     """
     transformed = expand_kv_blob_column(df, config)
     transformed = apply_spark_expr_columns(transformed, config)
+    transformed = apply_pivot(transformed, config)
     dedup_cols = (
         [c.strip() for c in config.transform.dedup_columns.split(",") if c.strip()]
         if config.transform.dedup_columns

{batch_analytics-0.3.26 → batch_analytics-0.3.27}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.26
+Version: 0.3.27
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT