batch-analytics 0.3.27__tar.gz → 0.3.29__tar.gz

{batch_analytics-0.3.27 → batch_analytics-0.3.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.27
+Version: 0.3.29
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.27 → batch_analytics-0.3.29}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.27"
+version = "0.3.29"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.3.27 → batch_analytics-0.3.29}/src/batch_analytics/analytics/__init__.py

@@ -10,10 +10,12 @@ from .linear_regression import run_linear_regression
 from .correlation import run_correlation
 from .pca_clustering import run_pca_clustering
 from .t_test import run_t_test
+from .equipment_oee import run_equipment_oee
 
 __all__ = [
     "run_linear_regression",
     "run_correlation",
     "run_pca_clustering",
     "run_t_test",
+    "run_equipment_oee",
 ]

batch_analytics-0.3.29/src/batch_analytics/analytics/equipment_oee.py

@@ -0,0 +1,207 @@
+"""
+Equipment-level OEE from interval timing, batch quality, and equipment×operation ideal cycle.
+
+- Interval: (equipment_id, operation_id, batch_id) — actual run + idle gaps
+- Batch quality: (batch_id) — is_good_batch
+- Ideal cycle: (equipment_id, operation_id) — standard minutes; batch-independent
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Tuple
+
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql import functions as F
+
+from ..config import BatchAnalyticsConfig
+from ..extract import extract_table
+
+logger = logging.getLogger(__name__)
+
+_INTERVAL_REQUIRED = (
+    "equipment_id",
+    "batch_id",
+    "operation_id",
+    "planned_production_minutes",
+    "actual_run_minutes",
+    "idle_gap_minutes",
+)
+
+_BATCH_QUALITY_REQUIRED = ("batch_id", "is_good_batch")
+
+_IDEAL_CYCLE_REQUIRED = (
+    "equipment_id",
+    "operation_id",
+    "ideal_cycle_time_minutes",
+)
+
+
+def _require_columns(df: DataFrame, cols: Tuple[str, ...], label: str) -> None:
+    missing = [c for c in cols if c not in df.columns]
+    if missing:
+        raise ValueError(
+            f"equipment_oee {label} missing columns: {missing}. Available: {df.columns}"
+        )
+
+
+def _load_batch_quality(spark: SparkSession, config: BatchAnalyticsConfig) -> DataFrame:
+    table = (config.analytics.equipment_oee_batch_quality_table or "").strip()
+    if not table:
+        raise ValueError(
+            "equipment_oee requires batch-level quality. Set BATCH_OEE_BATCH_QUALITY_TABLE "
+            "to the ClickHouse table (e.g. etc1_executed_bpr_batch_quality)."
+        )
+    df = extract_table(spark, table, config)
+    _require_columns(df, _BATCH_QUALITY_REQUIRED, "batch quality table")
+    return df.select(
+        "batch_id",
+        F.col("is_good_batch").cast("double").alias("is_good_batch"),
+        *(
+            [F.col("batch_disposition")]
+            if "batch_disposition" in df.columns
+            else []
+        ),
+    ).dropDuplicates(["batch_id"])
+
+
+def _load_ideal_cycle(spark: SparkSession, config: BatchAnalyticsConfig) -> DataFrame:
+    table = (config.analytics.equipment_oee_ideal_cycle_table or "").strip()
+    if not table:
+        raise ValueError(
+            "equipment_oee requires equipment×operation ideal cycle standards. "
+            "Set BATCH_OEE_IDEAL_CYCLE_TABLE "
+            "(e.g. etc1_executed_bpr_equipment_ideal_cycle)."
+        )
+    df = extract_table(spark, table, config)
+    _require_columns(df, _IDEAL_CYCLE_REQUIRED, "ideal cycle table")
+    return df.select(
+        "equipment_id",
+        "operation_id",
+        F.col("ideal_cycle_time_minutes").cast("double").alias("ideal_cycle_time_minutes"),
+    ).dropDuplicates(["equipment_id", "operation_id"])
+
+
+def _staging_availability(df: DataFrame) -> DataFrame:
+    return (
+        df.groupBy("equipment_id", "batch_id")
+        .agg(
+            F.sum(F.col("actual_run_minutes").cast("double")).alias("actual_run_minutes"),
+            F.sum(F.col("idle_gap_minutes").cast("double")).alias("downtime_minutes"),
+        )
+        .withColumn(
+            "planned_production_minutes",
+            F.col("actual_run_minutes") + F.col("downtime_minutes"),
+        )
+    )
+
+
+def _interval_with_ideal(interval_df: DataFrame, ideal_df: DataFrame) -> DataFrame:
+    return interval_df.join(ideal_df, on=["equipment_id", "operation_id"], how="inner")
+
+
+def _staging_performance(interval_df: DataFrame, ideal_df: DataFrame) -> DataFrame:
+    joined = _interval_with_ideal(interval_df, ideal_df)
+    return joined.groupBy("equipment_id", "batch_id", "operation_id").agg(
+        F.first("ideal_cycle_time_minutes").alias("ideal_cycle_time_minutes"),
+        F.sum(F.col("actual_run_minutes").cast("double")).alias("actual_run_minutes"),
+    )
+
+
+def _staging_performance_totals(interval_df: DataFrame, ideal_df: DataFrame) -> DataFrame:
+    joined = _interval_with_ideal(interval_df, ideal_df)
+    return joined.groupBy("equipment_id", "batch_id").agg(
+        F.sum("ideal_cycle_time_minutes").alias("ideal_cycle_time_minutes"),
+        F.sum(F.col("actual_run_minutes").cast("double")).alias("actual_run_minutes"),
+    )
+
+
+def _rows_to_dicts(df: DataFrame) -> List[Dict[str, Any]]:
+    return [row.asDict() for row in df.collect()]
+
+
+def run_equipment_oee(
+    spark: SparkSession,
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> Dict[str, Any]:
+    """
+    Compute equipment × batch OEE.
+
+    Performance uses ideal_cycle_time_minutes from BATCH_OEE_IDEAL_CYCLE_TABLE
+    (equipment_id × operation_id, batch-independent), not batch-specific BPR durations.
+    """
+    _require_columns(df, _INTERVAL_REQUIRED, "interval staging")
+
+    batch_q = _load_batch_quality(spark, config)
+    ideal = _load_ideal_cycle(spark, config)
+    avail = _staging_availability(df)
+    perf_ops = _staging_performance(df, ideal)
+    # Only ideal totals are joined here; avail already has actual_run_minutes.
+    # Keeping both would make F.col("actual_run_minutes") ambiguous after the join.
+    perf_tot = _staging_performance_totals(df, ideal).select(
+        "equipment_id",
+        "batch_id",
+        "ideal_cycle_time_minutes",
+    )
+
+    joined = (
+        avail.join(perf_tot, on=["equipment_id", "batch_id"], how="inner")
+        .join(batch_q, on="batch_id", how="inner")
+        .withColumn(
+            "availability_pct",
+            F.when(
+                F.col("planned_production_minutes") > 0,
+                F.col("actual_run_minutes") / F.col("planned_production_minutes"),
+            ).otherwise(F.lit(0.0)),
+        )
+        .withColumn(
+            "performance_pct",
+            F.when(
+                F.col("actual_run_minutes") > 0,
+                F.least(
+                    F.lit(1.0),
+                    F.col("ideal_cycle_time_minutes") / F.col("actual_run_minutes"),
+                ),
+            ).otherwise(F.lit(0.0)),
+        )
+        .withColumn("quality_pct", F.col("is_good_batch").cast("double"))
+        .withColumn(
+            "oee_pct",
+            F.col("availability_pct")
+            * F.col("performance_pct")
+            * F.col("quality_pct"),
+        )
+    )
+
+    out_cols = [
+        "equipment_id",
+        "batch_id",
+        "planned_production_minutes",
+        "actual_run_minutes",
+        "downtime_minutes",
+        "ideal_cycle_time_minutes",
+        "is_good_batch",
+        "availability_pct",
+        "performance_pct",
+        "quality_pct",
+        "oee_pct",
+    ]
+    if "batch_disposition" in joined.columns:
+        out_cols.insert(7, "batch_disposition")
+
+    result_df = joined.select(*out_cols).orderBy("equipment_id", "batch_id")
+
+    n = result_df.count()
+    logger.info("equipment_oee: %d equipment×batch rows", n)
+
+    return {
+        "row_count": n,
+        "staging_availability": _rows_to_dicts(avail.orderBy("equipment_id", "batch_id")),
+        "staging_batch_quality": _rows_to_dicts(batch_q.orderBy("batch_id")),
+        "staging_ideal_cycle": _rows_to_dicts(ideal.orderBy("equipment_id", "operation_id")),
+        "staging_performance_by_operation": _rows_to_dicts(
+            perf_ops.orderBy("equipment_id", "batch_id", "operation_id")
+        ),
+        "equipment_oee": _rows_to_dicts(result_df),
+    }

{batch_analytics-0.3.27 → batch_analytics-0.3.29}/src/batch_analytics/config.py

@@ -74,28 +74,47 @@ class TransformConfig:
 
     # Order: extract anchor_id from add_dimension(s) column, then dedupe by these keys.
     # Deduplication keys (comma-separated). Empty = dropDuplicates() on full row (all columns).
-    dedup_columns: str = os.environ.get("BATCH_DEDUP_COLUMNS", "")
+    # Use default_factory so values reflect os.environ when BatchAnalyticsConfig() is built, not at import time.
+    dedup_columns: str = field(
+        default_factory=lambda: os.environ.get("BATCH_DEDUP_COLUMNS", "")
+    )
     # Staging output path (local or S3)
-    staging_path: str = os.environ.get(
-        "BATCH_STAGING_PATH",
-        "/tmp/analytics_stage",
+    staging_path: str = field(
+        default_factory=lambda: os.environ.get(
+            "BATCH_STAGING_PATH",
+            "/tmp/analytics_stage",
+        )
     )
     # Output format for load_staged when reading (parquet/delta/clickhouse).
     # Stage job always writes to ClickHouse; use clickhouse for analytics to read from staged table.
-    staging_format: str = os.environ.get("BATCH_STAGING_FORMAT", "clickhouse")
+    staging_format: str = field(
+        default_factory=lambda: os.environ.get("BATCH_STAGING_FORMAT", "clickhouse")
+    )
     # Staging table name in ClickHouse (when format=clickhouse)
-    staging_table: str = os.environ.get("BATCH_STAGING_TABLE", "analytics_staging")
+    staging_table: str = field(
+        default_factory=lambda: os.environ.get("BATCH_STAGING_TABLE", "analytics_staging")
+    )
     # Spark save mode for ClickHouse staging (and path staging): overwrite | append
-    staging_write_mode: str = os.environ.get("BATCH_STAGING_WRITE_MODE", "overwrite")
+    staging_write_mode: str = field(
+        default_factory=lambda: os.environ.get("BATCH_STAGING_WRITE_MODE", "overwrite")
+    )
     # Source column holding a JSON object or Python dict string; every top-level key becomes a new String column
     # (see transform.expand_kv_blob_column). Example: add_dimensions {'anchor_id':'...','lot':'A1'}
-    add_dimension_column: str = os.environ.get("BATCH_ADD_DIMENSION_COLUMN", "add_dimension")
+    add_dimension_column: str = field(
+        default_factory=lambda: os.environ.get("BATCH_ADD_DIMENSION_COLUMN", "add_dimension")
+    )
     # Legacy: no longer used; output column names match JSON keys (e.g. anchor_id). Kept for env compatibility.
-    anchor_id_column: str = os.environ.get("BATCH_ANCHOR_ID_COLUMN", "anchor_id")
+    anchor_id_column: str = field(
+        default_factory=lambda: os.environ.get("BATCH_ANCHOR_ID_COLUMN", "anchor_id")
+    )
     # JSON object: {"new_col": "Spark SQL expression"} applied after KV expansion, before dedupe.
-    expr_columns_json: str = os.environ.get("BATCH_TRANSFORM_EXPR_COLUMNS", "").strip()
+    expr_columns_json: str = field(
+        default_factory=lambda: os.environ.get("BATCH_TRANSFORM_EXPR_COLUMNS", "").strip()
+    )
     # JSON pivot spec (group_by, pivot_column, value_column, agg, optional pivot_values, column_name_prefix).
-    pivot_json: str = os.environ.get("BATCH_TRANSFORM_PIVOT_JSON", "").strip()
+    pivot_json: str = field(
+        default_factory=lambda: os.environ.get("BATCH_TRANSFORM_PIVOT_JSON", "").strip()
+    )
 
 
 @dataclass
@@ -158,6 +177,14 @@ class AnalyticsConfig:
     ttest_col_a: str = os.environ.get("BATCH_TTEST_COL_A", "")
     ttest_col_b: str = os.environ.get("BATCH_TTEST_COL_B", "")
 
+    # Equipment OEE: batch-level quality table (batch_id, is_good_batch, …)
+    equipment_oee_batch_quality_table: str = os.environ.get(
+        "BATCH_OEE_BATCH_QUALITY_TABLE", ""
+    ).strip()
+    equipment_oee_ideal_cycle_table: str = os.environ.get(
+        "BATCH_OEE_IDEAL_CYCLE_TABLE", ""
+    ).strip()
+
 
 @dataclass
 class SparkK8sConfig:

{batch_analytics-0.3.27 → batch_analytics-0.3.29}/src/batch_analytics/modules.py

@@ -10,6 +10,7 @@ from .analytics import (
     run_correlation,
     run_pca_clustering,
     run_t_test,
+    run_equipment_oee,
 )
 
 # module_arg -> (run_fn, result_key)
@@ -18,6 +19,7 @@ MODULE_REGISTRY = {
     "corr": (run_correlation, "correlation"),
     "pca": (run_pca_clustering, "pca_clustering"),
     "ttest": (run_t_test, "t_test"),
+    "oee": (run_equipment_oee, "equipment_oee"),
 }
 
 VALID_MODULES = list(MODULE_REGISTRY.keys())

{batch_analytics-0.3.27 → batch_analytics-0.3.29}/src/batch_analytics/transform.py

@@ -175,7 +175,11 @@ def apply_spark_expr_columns(
     Value must be a JSON object mapping **output column name** → **expression** (same dialect as
     ``selectExpr``), e.g. ``{"y": "cast(duration_minutes as double)", "x": "cast(regexp_extract(operation_id, '([0-9]+)$', 1) as double)"}`` (Java regex; prefer trailing digits to avoid hyphen/range issues in patterns like ``ETC-1-OP-…``).
     """
-    raw = (config.transform.expr_columns_json or "").strip()
+    # Prefer live os.environ (Spark driver env), same as BATCH_CLICKHOUSE_STAGING_ORDER_BY; config
+    # snapshot may lag if env was set after import (dataclass defaults used to freeze at class def).
+    raw = os.environ.get("BATCH_TRANSFORM_EXPR_COLUMNS", "").strip()
+    if not raw:
+        raw = (config.transform.expr_columns_json or "").strip()
     if not raw:
         return df
     try:
@@ -224,7 +228,9 @@ def apply_pivot(
 
     If ``pivot_values`` is omitted, distinct values are collected (sorted by string order).
     """
-    raw = (config.transform.pivot_json or "").strip()
+    raw = os.environ.get("BATCH_TRANSFORM_PIVOT_JSON", "").strip()
+    if not raw:
+        raw = (config.transform.pivot_json or "").strip()
     if not raw:
         return df
     try:
@@ -423,11 +429,15 @@ def stage_to_clickhouse(
         try:
             full_name = f"{cat}.{ch.database}.{tbl}"
             order_by = _merge_tree_order_by_for_staging(df, config)
+            # Helps compare local vs CH 25.x prod: missing ORDER BY cols are a schema issue here, not env.
             logger.info(
-                "Staging to ClickHouse via catalog %s (mode=%s, order_by=%s)",
+                "Staging to ClickHouse via catalog %s (mode=%s, order_by=%s); "
+                "dataframe columns (%d): %s",
                 full_name,
                 mode,
                 order_by,
+                len(df.columns),
+                sorted(df.columns),
             )
             # Plain MergeTree() only — SETTINGS belong in tableProperty("settings.*", ...) so the
             # connector emits them after ORDER BY; inline SETTINGS in ENGINE breaks CH 25.5 parsing.

{batch_analytics-0.3.27 → batch_analytics-0.3.29}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.27
+Version: 0.3.29
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.27 → batch_analytics-0.3.29}/src/batch_analytics.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ src/batch_analytics.egg-info/requires.txt
 src/batch_analytics.egg-info/top_level.txt
 src/batch_analytics/analytics/__init__.py
 src/batch_analytics/analytics/correlation.py
+src/batch_analytics/analytics/equipment_oee.py
 src/batch_analytics/analytics/gluon_autogluon_infer.py
 src/batch_analytics/analytics/gluon_autogluon_train.py
 src/batch_analytics/analytics/linear_regression.py