batch-analytics 0.3.26__tar.gz → 0.3.28__tar.gz

{batch_analytics-0.3.26 → batch_analytics-0.3.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.26
+Version: 0.3.28
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.26 → batch_analytics-0.3.28}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.26"
+version = "0.3.28"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.3.26 → batch_analytics-0.3.28}/src/batch_analytics/analytics/__init__.py

@@ -10,10 +10,12 @@ from .linear_regression import run_linear_regression
 from .correlation import run_correlation
 from .pca_clustering import run_pca_clustering
 from .t_test import run_t_test
+from .equipment_oee import run_equipment_oee
 
 __all__ = [
     "run_linear_regression",
     "run_correlation",
     "run_pca_clustering",
     "run_t_test",
+    "run_equipment_oee",
 ]

batch_analytics-0.3.28/src/batch_analytics/analytics/equipment_oee.py

@@ -0,0 +1,201 @@
+"""
+Equipment-level OEE from interval timing, batch quality, and equipment×operation ideal cycle.
+
+- Interval: (equipment_id, operation_id, batch_id) — actual run + idle gaps
+- Batch quality: (batch_id) — is_good_batch
+- Ideal cycle: (equipment_id, operation_id) — standard minutes; batch-independent
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Tuple
+
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql import functions as F
+
+from ..config import BatchAnalyticsConfig
+from ..extract import extract_table
+
+logger = logging.getLogger(__name__)
+
+_INTERVAL_REQUIRED = (
+    "equipment_id",
+    "batch_id",
+    "operation_id",
+    "planned_production_minutes",
+    "actual_run_minutes",
+    "idle_gap_minutes",
+)
+
+_BATCH_QUALITY_REQUIRED = ("batch_id", "is_good_batch")
+
+_IDEAL_CYCLE_REQUIRED = (
+    "equipment_id",
+    "operation_id",
+    "ideal_cycle_time_minutes",
+)
+
+
+def _require_columns(df: DataFrame, cols: Tuple[str, ...], label: str) -> None:
+    missing = [c for c in cols if c not in df.columns]
+    if missing:
+        raise ValueError(
+            f"equipment_oee {label} missing columns: {missing}. Available: {df.columns}"
+        )
+
+
+def _load_batch_quality(spark: SparkSession, config: BatchAnalyticsConfig) -> DataFrame:
+    table = (config.analytics.equipment_oee_batch_quality_table or "").strip()
+    if not table:
+        raise ValueError(
+            "equipment_oee requires batch-level quality. Set BATCH_OEE_BATCH_QUALITY_TABLE "
+            "to the ClickHouse table (e.g. etc1_executed_bpr_batch_quality)."
+        )
+    df = extract_table(spark, table, config)
+    _require_columns(df, _BATCH_QUALITY_REQUIRED, "batch quality table")
+    return df.select(
+        "batch_id",
+        F.col("is_good_batch").cast("double").alias("is_good_batch"),
+        *(
+            [F.col("batch_disposition")]
+            if "batch_disposition" in df.columns
+            else []
+        ),
+    ).dropDuplicates(["batch_id"])
+
+
+def _load_ideal_cycle(spark: SparkSession, config: BatchAnalyticsConfig) -> DataFrame:
+    table = (config.analytics.equipment_oee_ideal_cycle_table or "").strip()
+    if not table:
+        raise ValueError(
+            "equipment_oee requires equipment×operation ideal cycle standards. "
+            "Set BATCH_OEE_IDEAL_CYCLE_TABLE "
+            "(e.g. etc1_executed_bpr_equipment_ideal_cycle)."
+        )
+    df = extract_table(spark, table, config)
+    _require_columns(df, _IDEAL_CYCLE_REQUIRED, "ideal cycle table")
+    return df.select(
+        "equipment_id",
+        "operation_id",
+        F.col("ideal_cycle_time_minutes").cast("double").alias("ideal_cycle_time_minutes"),
+    ).dropDuplicates(["equipment_id", "operation_id"])
+
+
+def _staging_availability(df: DataFrame) -> DataFrame:
+    return (
+        df.groupBy("equipment_id", "batch_id")
+        .agg(
+            F.sum(F.col("actual_run_minutes").cast("double")).alias("actual_run_minutes"),
+            F.sum(F.col("idle_gap_minutes").cast("double")).alias("downtime_minutes"),
+        )
+        .withColumn(
+            "planned_production_minutes",
+            F.col("actual_run_minutes") + F.col("downtime_minutes"),
+        )
+    )
+
+
+def _interval_with_ideal(interval_df: DataFrame, ideal_df: DataFrame) -> DataFrame:
+    return interval_df.join(ideal_df, on=["equipment_id", "operation_id"], how="inner")
+
+
+def _staging_performance(interval_df: DataFrame, ideal_df: DataFrame) -> DataFrame:
+    joined = _interval_with_ideal(interval_df, ideal_df)
+    return joined.groupBy("equipment_id", "batch_id", "operation_id").agg(
+        F.first("ideal_cycle_time_minutes").alias("ideal_cycle_time_minutes"),
+        F.sum(F.col("actual_run_minutes").cast("double")).alias("actual_run_minutes"),
+    )
+
+
+def _staging_performance_totals(interval_df: DataFrame, ideal_df: DataFrame) -> DataFrame:
+    joined = _interval_with_ideal(interval_df, ideal_df)
+    return joined.groupBy("equipment_id", "batch_id").agg(
+        F.sum("ideal_cycle_time_minutes").alias("ideal_cycle_time_minutes"),
+        F.sum(F.col("actual_run_minutes").cast("double")).alias("actual_run_minutes"),
+    )
+
+
+def _rows_to_dicts(df: DataFrame) -> List[Dict[str, Any]]:
+    return [row.asDict() for row in df.collect()]
+
+
+def run_equipment_oee(
+    spark: SparkSession,
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> Dict[str, Any]:
+    """
+    Compute equipment × batch OEE.
+
+    Performance uses ideal_cycle_time_minutes from BATCH_OEE_IDEAL_CYCLE_TABLE
+    (equipment_id × operation_id, batch-independent), not batch-specific BPR durations.
+    """
+    _require_columns(df, _INTERVAL_REQUIRED, "interval staging")
+
+    batch_q = _load_batch_quality(spark, config)
+    ideal = _load_ideal_cycle(spark, config)
+    avail = _staging_availability(df)
+    perf_ops = _staging_performance(df, ideal)
+    perf_tot = _staging_performance_totals(df, ideal)
+
+    joined = (
+        avail.join(perf_tot, on=["equipment_id", "batch_id"], how="inner")
+        .join(batch_q, on="batch_id", how="inner")
+        .withColumn(
+            "availability_pct",
+            F.when(
+                F.col("planned_production_minutes") > 0,
+                F.col("actual_run_minutes") / F.col("planned_production_minutes"),
+            ).otherwise(F.lit(0.0)),
+        )
+        .withColumn(
+            "performance_pct",
+            F.when(
+                F.col("actual_run_minutes") > 0,
+                F.least(
+                    F.lit(1.0),
+                    F.col("ideal_cycle_time_minutes") / F.col("actual_run_minutes"),
+                ),
+            ).otherwise(F.lit(0.0)),
+        )
+        .withColumn("quality_pct", F.col("is_good_batch").cast("double"))
+        .withColumn(
+            "oee_pct",
+            F.col("availability_pct")
+            * F.col("performance_pct")
+            * F.col("quality_pct"),
+        )
+    )
+
+    out_cols = [
+        "equipment_id",
+        "batch_id",
+        "planned_production_minutes",
+        "actual_run_minutes",
+        "downtime_minutes",
+        "ideal_cycle_time_minutes",
+        "is_good_batch",
+        "availability_pct",
+        "performance_pct",
+        "quality_pct",
+        "oee_pct",
+    ]
+    if "batch_disposition" in joined.columns:
+        out_cols.insert(7, "batch_disposition")
+
+    result_df = joined.select(*out_cols).orderBy("equipment_id", "batch_id")
+
+    n = result_df.count()
+    logger.info("equipment_oee: %d equipment×batch rows", n)
+
+    return {
+        "row_count": n,
+        "staging_availability": _rows_to_dicts(avail.orderBy("equipment_id", "batch_id")),
+        "staging_batch_quality": _rows_to_dicts(batch_q.orderBy("batch_id")),
+        "staging_ideal_cycle": _rows_to_dicts(ideal.orderBy("equipment_id", "operation_id")),
+        "staging_performance_by_operation": _rows_to_dicts(
+            perf_ops.orderBy("equipment_id", "batch_id", "operation_id")
+        ),
+        "equipment_oee": _rows_to_dicts(result_df),
+    }

{batch_analytics-0.3.26 → batch_analytics-0.3.28}/src/batch_analytics/config.py

@@ -74,26 +74,47 @@ class TransformConfig:
 
     # Order: extract anchor_id from add_dimension(s) column, then dedupe by these keys.
     # Deduplication keys (comma-separated). Empty = dropDuplicates() on full row (all columns).
-    dedup_columns: str = os.environ.get("BATCH_DEDUP_COLUMNS", "")
+    # Use default_factory so values reflect os.environ when BatchAnalyticsConfig() is built, not at import time.
+    dedup_columns: str = field(
+        default_factory=lambda: os.environ.get("BATCH_DEDUP_COLUMNS", "")
+    )
     # Staging output path (local or S3)
-    staging_path: str = os.environ.get(
-        "BATCH_STAGING_PATH",
-        "/tmp/analytics_stage",
+    staging_path: str = field(
+        default_factory=lambda: os.environ.get(
+            "BATCH_STAGING_PATH",
+            "/tmp/analytics_stage",
+        )
     )
     # Output format for load_staged when reading (parquet/delta/clickhouse).
     # Stage job always writes to ClickHouse; use clickhouse for analytics to read from staged table.
-    staging_format: str = os.environ.get("BATCH_STAGING_FORMAT", "clickhouse")
+    staging_format: str = field(
+        default_factory=lambda: os.environ.get("BATCH_STAGING_FORMAT", "clickhouse")
+    )
     # Staging table name in ClickHouse (when format=clickhouse)
-    staging_table: str = os.environ.get("BATCH_STAGING_TABLE", "analytics_staging")
+    staging_table: str = field(
+        default_factory=lambda: os.environ.get("BATCH_STAGING_TABLE", "analytics_staging")
+    )
     # Spark save mode for ClickHouse staging (and path staging): overwrite | append
-    staging_write_mode: str = os.environ.get("BATCH_STAGING_WRITE_MODE", "overwrite")
+    staging_write_mode: str = field(
+        default_factory=lambda: os.environ.get("BATCH_STAGING_WRITE_MODE", "overwrite")
+    )
     # Source column holding a JSON object or Python dict string; every top-level key becomes a new String column
     # (see transform.expand_kv_blob_column). Example: add_dimensions {'anchor_id':'...','lot':'A1'}
-    add_dimension_column: str = os.environ.get("BATCH_ADD_DIMENSION_COLUMN", "add_dimension")
+    add_dimension_column: str = field(
+        default_factory=lambda: os.environ.get("BATCH_ADD_DIMENSION_COLUMN", "add_dimension")
+    )
     # Legacy: no longer used; output column names match JSON keys (e.g. anchor_id). Kept for env compatibility.
-    anchor_id_column: str = os.environ.get("BATCH_ANCHOR_ID_COLUMN", "anchor_id")
+    anchor_id_column: str = field(
+        default_factory=lambda: os.environ.get("BATCH_ANCHOR_ID_COLUMN", "anchor_id")
+    )
     # JSON object: {"new_col": "Spark SQL expression"} applied after KV expansion, before dedupe.
-    expr_columns_json: str = os.environ.get("BATCH_TRANSFORM_EXPR_COLUMNS", "").strip()
+    expr_columns_json: str = field(
+        default_factory=lambda: os.environ.get("BATCH_TRANSFORM_EXPR_COLUMNS", "").strip()
+    )
+    # JSON pivot spec (group_by, pivot_column, value_column, agg, optional pivot_values, column_name_prefix).
+    pivot_json: str = field(
+        default_factory=lambda: os.environ.get("BATCH_TRANSFORM_PIVOT_JSON", "").strip()
+    )
 
 
 @dataclass
@@ -156,6 +177,14 @@ class AnalyticsConfig:
     ttest_col_a: str = os.environ.get("BATCH_TTEST_COL_A", "")
     ttest_col_b: str = os.environ.get("BATCH_TTEST_COL_B", "")
 
+    # Equipment OEE: batch-level quality table (batch_id, is_good_batch, …)
+    equipment_oee_batch_quality_table: str = os.environ.get(
+        "BATCH_OEE_BATCH_QUALITY_TABLE", ""
+    ).strip()
+    equipment_oee_ideal_cycle_table: str = os.environ.get(
+        "BATCH_OEE_IDEAL_CYCLE_TABLE", ""
+    ).strip()
+
 
 @dataclass
 class SparkK8sConfig:

{batch_analytics-0.3.26 → batch_analytics-0.3.28}/src/batch_analytics/modules.py

@@ -10,6 +10,7 @@ from .analytics import (
     run_correlation,
     run_pca_clustering,
     run_t_test,
+    run_equipment_oee,
 )
 
 # module_arg -> (run_fn, result_key)
@@ -18,6 +19,7 @@ MODULE_REGISTRY = {
     "corr": (run_correlation, "correlation"),
     "pca": (run_pca_clustering, "pca_clustering"),
     "ttest": (run_t_test, "t_test"),
+    "oee": (run_equipment_oee, "equipment_oee"),
 }
 
 VALID_MODULES = list(MODULE_REGISTRY.keys())

{batch_analytics-0.3.26 → batch_analytics-0.3.28}/src/batch_analytics/transform.py

@@ -175,7 +175,11 @@ def apply_spark_expr_columns(
     Value must be a JSON object mapping **output column name** → **expression** (same dialect as
     ``selectExpr``), e.g. ``{"y": "cast(duration_minutes as double)", "x": "cast(regexp_extract(operation_id, '([0-9]+)$', 1) as double)"}`` (Java regex; prefer trailing digits to avoid hyphen/range issues in patterns like ``ETC-1-OP-…``).
     """
-    raw = (config.transform.expr_columns_json or "").strip()
+    # Prefer live os.environ (Spark driver env), same as BATCH_CLICKHOUSE_STAGING_ORDER_BY; config
+    # snapshot may lag if env was set after import (dataclass defaults used to freeze at class def).
+    raw = os.environ.get("BATCH_TRANSFORM_EXPR_COLUMNS", "").strip()
+    if not raw:
+        raw = (config.transform.expr_columns_json or "").strip()
     if not raw:
         return df
     try:
@@ -196,6 +200,98 @@ def apply_spark_expr_columns(
     return out
 
 
+def _pivot_output_column_name(prefix: str, pivot_value: str) -> str:
+    """Stable identifier for pivoted columns (e.g. imp_rm_001 from RM-001)."""
+    p = (prefix or "v").rstrip("_")
+    body = str(pivot_value).replace("-", "_").replace(" ", "_").lower()
+    return f"{p}_{body}"
+
+
+def apply_pivot(
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    """
+    GroupBy + pivot + aggregate (``BATCH_TRANSFORM_PIVOT_JSON``).
+
+    Example::
+
+        {
+          "group_by": "batch_id",
+          "pivot_column": "material_id",
+          "value_column": "total_impurities_pct",
+          "agg": "max",
+          "pivot_values": ["RM-001", "RM-002"],
+          "column_name_prefix": "imp",
+          "rename_pivot_columns": true
+        }
+
+    If ``pivot_values`` is omitted, distinct values are collected (sorted by string order).
+    """
+    raw = os.environ.get("BATCH_TRANSFORM_PIVOT_JSON", "").strip()
+    if not raw:
+        raw = (config.transform.pivot_json or "").strip()
+    if not raw:
+        return df
+    try:
+        spec = json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"BATCH_TRANSFORM_PIVOT_JSON must be valid JSON: {e}") from e
+    if not isinstance(spec, dict):
+        raise ValueError("BATCH_TRANSFORM_PIVOT_JSON must be a JSON object")
+
+    group_by = spec.get("group_by") or spec.get("groupBy")
+    pivot_col = (spec.get("pivot_column") or "").strip()
+    value_col = (spec.get("value_column") or "").strip()
+    if not group_by or not pivot_col or not value_col:
+        raise ValueError("pivot spec requires group_by, pivot_column, value_column")
+    agg_name = (spec.get("agg") or "max").strip().lower()
+    prefix = (spec.get("column_name_prefix") or "v").strip() or "v"
+    rename_pivot = spec.get("rename_pivot_columns", True)
+
+    from pyspark.sql import functions as F
+
+    gcols = [c.strip() for c in str(group_by).split(",") if c.strip()]
+    for c in gcols + [pivot_col, value_col]:
+        if c not in df.columns:
+            raise ValueError(f"pivot: column {c!r} not in dataframe; have {df.columns}")
+
+    vc = F.col(value_col)
+    if agg_name == "max":
+        agg_expr = F.max(vc)
+    elif agg_name == "min":
+        agg_expr = F.min(vc)
+    elif agg_name in ("sum",):
+        agg_expr = F.sum(vc)
+    elif agg_name in ("avg", "mean"):
+        agg_expr = F.avg(vc)
+    elif agg_name == "first":
+        agg_expr = F.first(vc)
+    else:
+        raise ValueError(
+            f"pivot agg must be one of max,min,sum,avg,mean,first; got {agg_name!r}"
+        )
+
+    pivot_values = spec.get("pivot_values")
+    if pivot_values is not None:
+        pv: List[str] = [str(v) for v in pivot_values]
+    else:
+        pv = [r[0] for r in df.select(pivot_col).distinct().sort(pivot_col).collect()]
+        logger.info("pivot: inferred %d distinct %s values", len(pv), pivot_col)
+
+    out = df.groupBy(*gcols).pivot(pivot_col, pv).agg(agg_expr)
+
+    if rename_pivot:
+        for v in pv:
+            old = v
+            if old not in out.columns:
+                continue
+            new_name = _pivot_output_column_name(prefix, old)
+            if new_name != old:
+                out = out.withColumnRenamed(old, new_name)
+    return out
+
+
 def transform(
     df: DataFrame,
     config: BatchAnalyticsConfig,
@@ -203,11 +299,13 @@ def transform(
     """
     Apply transformation only: (1) expand JSON/KV blob column into one column per top-level key,
     (2) optional Spark SQL expression columns (``BATCH_TRANSFORM_EXPR_COLUMNS``),
-    (3) deduplicate by BATCH_DEDUP_COLUMNS if set, else by full row.
+    (3) optional groupBy+pivot (``BATCH_TRANSFORM_PIVOT_JSON``),
+    (4) deduplicate by BATCH_DEDUP_COLUMNS if set, else by full row.
     Does not write anywhere. Use stage_to_clickhouse() separately to persist.
     """
     transformed = expand_kv_blob_column(df, config)
     transformed = apply_spark_expr_columns(transformed, config)
+    transformed = apply_pivot(transformed, config)
     dedup_cols = (
         [c.strip() for c in config.transform.dedup_columns.split(",") if c.strip()]
         if config.transform.dedup_columns
@@ -331,11 +429,15 @@ def stage_to_clickhouse(
         try:
             full_name = f"{cat}.{ch.database}.{tbl}"
             order_by = _merge_tree_order_by_for_staging(df, config)
+            # Helps compare local vs CH 25.x prod: missing ORDER BY cols are a schema issue here, not env.
             logger.info(
-                "Staging to ClickHouse via catalog %s (mode=%s, order_by=%s)",
+                "Staging to ClickHouse via catalog %s (mode=%s, order_by=%s); "
+                "dataframe columns (%d): %s",
                 full_name,
                 mode,
                 order_by,
+                len(df.columns),
+                sorted(df.columns),
             )
             # Plain MergeTree() only — SETTINGS belong in tableProperty("settings.*", ...) so the
             # connector emits them after ORDER BY; inline SETTINGS in ENGINE breaks CH 25.5 parsing.

{batch_analytics-0.3.26 → batch_analytics-0.3.28}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.26
+Version: 0.3.28
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.26 → batch_analytics-0.3.28}/src/batch_analytics.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ src/batch_analytics.egg-info/requires.txt
 src/batch_analytics.egg-info/top_level.txt
 src/batch_analytics/analytics/__init__.py
 src/batch_analytics/analytics/correlation.py
+src/batch_analytics/analytics/equipment_oee.py
 src/batch_analytics/analytics/gluon_autogluon_infer.py
 src/batch_analytics/analytics/gluon_autogluon_train.py
 src/batch_analytics/analytics/linear_regression.py