batch-analytics 0.3.30__tar.gz → 0.3.34__tar.gz

{batch_analytics-0.3.30 → batch_analytics-0.3.34}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.30
-Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
+Version: 0.3.34
+Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test, LLM classification).
 Author: Litewave Analytics Team
 License: MIT
 Requires-Python: >=3.8

{batch_analytics-0.3.30 → batch_analytics-0.3.34}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.30"
-description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
+version = "0.3.34"
+description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test, LLM classification)."
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [

{batch_analytics-0.3.30 → batch_analytics-0.3.34}/src/batch_analytics/analytics/__init__.py

@@ -8,6 +8,7 @@ Analytics modules for batch analytics pipeline.
 
 from .linear_regression import run_linear_regression
 from .correlation import run_correlation
+from .pca import run_pca
 from .pca_clustering import run_pca_clustering
 from .t_test import run_t_test
 from .equipment_oee import run_equipment_oee
@@ -15,6 +16,7 @@ from .equipment_oee import run_equipment_oee
 __all__ = [
     "run_linear_regression",
     "run_correlation",
+    "run_pca",
     "run_pca_clustering",
     "run_t_test",
     "run_equipment_oee",

batch_analytics-0.3.34/src/batch_analytics/analytics/pca.py

@@ -0,0 +1,70 @@
+"""
+Module: MVDA PCA on staged data (litewave-analytics POST /pca compatible response).
+"""
+
+import logging
+from typing import Any, Dict
+
+from pyspark.sql import DataFrame, SparkSession
+
+from ..config import BatchAnalyticsConfig
+from .pca_core import (
+    build_mvda_pca_response,
+    empty_mvda_response,
+    ensure_target_in_features,
+    fit_pca_and_collect_scores,
+    prepare_pca_dataframe,
+    resolve_n_components,
+    resolve_pca_feature_cols,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def run_pca(
+    spark: SparkSession,
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> Dict[str, Any]:
+    """
+    Principal Component Analysis with MVDA response shape.
+
+    Returns flat dict: n_components, explained_variance, cumulative_variance,
+    dominant_features, loadings, scores, row_count, message.
+    """
+    del spark  # SparkSession kept for MODULE_REGISTRY signature consistency
+    min_rows = config.analytics.pca_min_rows
+    sample_limit = config.analytics.pca_sample_size
+    target_feature = (config.analytics.pca_target_feature or "").strip()
+
+    feature_cols = resolve_pca_feature_cols(df, config)
+    if len(feature_cols) < 2:
+        raise ValueError("At least 2 features required for PCA")
+
+    feature_cols, target_feature = ensure_target_in_features(
+        feature_cols, target_feature
+    )
+
+    df_scaled, _df_num, row_count = prepare_pca_dataframe(df, feature_cols)
+    if row_count < min_rows:
+        return empty_mvda_response(row_count, min_rows)
+
+    k = resolve_n_components(len(feature_cols), config)
+    explained, pc_matrix, _pca_model, pca_rows = fit_pca_and_collect_scores(
+        df_scaled,
+        feature_cols,
+        target_feature,
+        k,
+        row_count,
+        sample_limit,
+    )
+
+    return build_mvda_pca_response(
+        feature_cols,
+        target_feature,
+        explained,
+        pc_matrix,
+        pca_rows,
+        row_count,
+        min_rows,
+    )

batch_analytics-0.3.34/src/batch_analytics/analytics/pca_core.py

@@ -0,0 +1,124 @@
+"""
+Shared PCA Spark pipeline helpers.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import List, Tuple
+
+from pyspark.ml.feature import StandardScaler, VectorAssembler
+from pyspark.sql import DataFrame
+from pyspark.sql.functions import col
+from pyspark.sql.types import DoubleType
+
+from ..config import BatchAnalyticsConfig
+from .pca_mvda import (
+    build_mvda_pca_response,
+    empty_mvda_response,
+    ensure_target_in_features,
+    build_loadings_and_dominant,
+)
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "resolve_pca_feature_cols",
+    "ensure_target_in_features",
+    "empty_mvda_response",
+    "build_loadings_and_dominant",
+    "build_mvda_pca_response",
+    "prepare_pca_dataframe",
+    "resolve_n_components",
+    "fit_pca_and_collect_scores",
+]
+
+
+def resolve_pca_feature_cols(df: DataFrame, config: BatchAnalyticsConfig) -> List[str]:
+    """Resolve feature column names from config or auto-detect numerics."""
+    feature_cols = [
+        c.strip()
+        for c in config.analytics.pca_features.split(",")
+        if c.strip()
+    ]
+    if not feature_cols:
+        feature_cols = [
+            f.name
+            for f in df.schema.fields
+            if "double" in str(f.dataType).lower()
+            or "int" in str(f.dataType).lower()
+            or "long" in str(f.dataType).lower()
+            or "float" in str(f.dataType).lower()
+        ]
+        logger.info("Auto-selected %d numeric columns for PCA", len(feature_cols))
+    return feature_cols
+
+
+def prepare_pca_dataframe(
+    df: DataFrame, feature_cols: List[str]
+) -> Tuple[DataFrame, DataFrame, int]:
+    """Cast features, drop NA, assemble and scale. Returns (df_scaled, df_num, row_count)."""
+    missing = [c for c in feature_cols if c not in df.columns]
+    if missing:
+        raise ValueError(
+            f"PCA features not found: {missing}. Available: {df.columns[:15]}..."
+        )
+
+    df_num = df.select(
+        *[col(c).cast(DoubleType()).alias(c) for c in feature_cols]
+    ).dropna()
+    row_count = df_num.count()
+
+    assembler = VectorAssembler(
+        inputCols=feature_cols,
+        outputCol="features_raw",
+        handleInvalid="skip",
+    )
+    df_vec = assembler.transform(df_num)
+
+    scaler = StandardScaler(
+        inputCol="features_raw",
+        outputCol="features",
+        withStd=True,
+        withMean=True,
+    )
+    scaler_model = scaler.fit(df_vec)
+    df_scaled = scaler_model.transform(df_vec)
+    return df_scaled, df_num, row_count
+
+
+def resolve_n_components(n_features: int, config: BatchAnalyticsConfig) -> int:
+    raw = (config.analytics.pca_n_components or "").strip()
+    if not raw:
+        return n_features
+    return min(int(raw), n_features)
+
+
+def fit_pca_and_collect_scores(
+    df_scaled: DataFrame,
+    feature_cols: List[str],
+    target_feature: str,
+    k: int,
+    row_count: int,
+    sample_limit: int,
+):
+    """Fit Spark PCA and collect rows for scores scatter (pca_features + features)."""
+    from pyspark.ml.feature import PCA as SparkPCA
+
+    pca = SparkPCA(k=k, inputCol="features", outputCol="pca_features")
+    pca_model = pca.fit(df_scaled)
+
+    explained = pca_model.explainedVariance.toArray().tolist()
+    pc_matrix = pca_model.pc.toArray()
+
+    pca_rows = []
+    if sample_limit > 0:
+        if row_count > sample_limit:
+            fraction = min(sample_limit / row_count * 1.2, 1.0)
+            sample_df = df_scaled.sample(fraction=fraction, seed=42)
+        else:
+            sample_df = df_scaled
+        result_df = pca_model.transform(sample_df)
+        select_cols = ["pca_features"] + feature_cols
+        pca_rows = result_df.select(*select_cols).limit(sample_limit).collect()
+    return explained, pc_matrix, pca_model, pca_rows

batch_analytics-0.3.34/src/batch_analytics/analytics/pca_mvda.py

@@ -0,0 +1,126 @@
+"""
+MVDA PCA response builders (no PySpark). Compatible with litewave-analytics POST /pca.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Tuple
+
+INSUFFICIENT_DATA_TEMPLATE = (
+    "Insufficient data — only {row_count} data-points available (need at least {min_rows})"
+)
+
+
+def ensure_target_in_features(
+    feature_cols: List[str], target_feature: str
+) -> Tuple[List[str], str]:
+    """Match litewave: target is included in the PCA feature set."""
+    target = (target_feature or "").strip()
+    if not target and feature_cols:
+        target = feature_cols[0]
+    cols = list(feature_cols)
+    if target and target not in cols:
+        cols = [target] + cols
+    return cols, target
+
+
+def empty_mvda_response(
+    row_count: int, min_rows: int, message: Optional[str] = None
+) -> Dict[str, Any]:
+    return {
+        "n_components": 0,
+        "explained_variance": [],
+        "cumulative_variance": [],
+        "dominant_features": [],
+        "loadings": [],
+        "scores": {
+            "pc1_label": "",
+            "pc1": [],
+            "pc2_label": "",
+            "pc2": [],
+            "target_values": [],
+        },
+        "row_count": row_count,
+        "message": message
+        or INSUFFICIENT_DATA_TEMPLATE.format(row_count=row_count, min_rows=min_rows),
+    }
+
+
+def build_loadings_and_dominant(
+    feature_cols: List[str], pc_matrix, n_components: int
+) -> Tuple[List[dict], List[dict]]:
+    """pc_matrix shape (n_features, k) — Spark PCA pc layout."""
+    loadings: List[dict] = []
+    dominant_features: List[dict] = []
+    for pc_i in range(n_components):
+        vec = pc_matrix[:, pc_i]
+        max_idx = int(max(range(len(vec)), key=lambda j: abs(float(vec[j]))))
+        dominant_features.append({"pc": pc_i + 1, "feature": feature_cols[max_idx]})
+        for j, name in enumerate(feature_cols):
+            loadings.append(
+                {"pc": pc_i + 1, "feature": name, "value": round(float(vec[j]), 6)}
+            )
+    return loadings, dominant_features
+
+
+def build_mvda_pca_response(
+    feature_cols: List[str],
+    target_feature: str,
+    explained: List[float],
+    pc_matrix,
+    pca_rows: List,
+    row_count: int,
+    min_rows: int,
+) -> Dict[str, Any]:
+    """Build flat MVDA PCA payload (litewave POST /pca compatible)."""
+    if row_count < min_rows:
+        return empty_mvda_response(row_count, min_rows)
+
+    n_components = len(explained)
+    cumsum: List[float] = []
+    s = 0.0
+    for v in explained:
+        s += v
+        cumsum.append(round(s, 6))
+
+    loadings, dominant_features = build_loadings_and_dominant(
+        feature_cols, pc_matrix, n_components
+    )
+
+    def _pc_label(pc_idx: int) -> str:
+        feat = (
+            dominant_features[pc_idx]["feature"]
+            if pc_idx < len(dominant_features)
+            else ""
+        )
+        pct = round(explained[pc_idx] * 100, 1) if pc_idx < len(explained) else 0
+        return f"PC{pc_idx + 1} \u2014 {feat} ({pct}%)"
+
+    scores: Dict[str, Any] = {
+        "pc1_label": _pc_label(0),
+        "pc1": [],
+        "pc2_label": _pc_label(1) if n_components >= 2 else "",
+        "pc2": [],
+        "target_values": [],
+    }
+    for row in pca_rows:
+        pca_vec = row["pca_features"].toArray()
+        scores["pc1"].append(round(float(pca_vec[0]), 4))
+        if n_components >= 2 and len(pca_vec) > 1:
+            scores["pc2"].append(round(float(pca_vec[1]), 4))
+        if target_feature:
+            target_val = row[target_feature]
+            scores["target_values"].append(
+                round(float(target_val), 4) if target_val is not None else None
+            )
+
+    return {
+        "n_components": n_components,
+        "explained_variance": [round(v, 6) for v in explained],
+        "cumulative_variance": cumsum,
+        "dominant_features": dominant_features,
+        "loadings": loadings,
+        "scores": scores,
+        "row_count": row_count,
+        "message": None,
+    }

{batch_analytics-0.3.30 → batch_analytics-0.3.34}/src/batch_analytics/config.py

@@ -162,8 +162,12 @@ class AnalyticsConfig:
     )
     corr_threshold: float = float(os.environ.get("BATCH_CORR_THRESHOLD", "0.8"))
 
-    # Module 3: PCA + Clustering
+    # Module 3: PCA (MVDA) and PCA + clustering
     pca_features: str = os.environ.get("BATCH_PCA_FEATURES", "")
+    pca_target_feature: str = os.environ.get("BATCH_PCA_TARGET_FEATURE", "")
+    pca_n_components: str = os.environ.get("BATCH_PCA_N_COMPONENTS", "")
+    pca_sample_size: int = int(os.environ.get("BATCH_PCA_SAMPLE_SIZE", "5000"))
+    pca_min_rows: int = int(os.environ.get("BATCH_PCA_MIN_ROWS", "10"))
     pca_variance_threshold: float = float(
         os.environ.get("BATCH_PCA_VARIANCE_THRESHOLD", "0.95")
     )

{batch_analytics-0.3.30 → batch_analytics-0.3.34}/src/batch_analytics/modules.py

@@ -8,16 +8,20 @@ See analytics_runner/catalog/analytics_catalog.yaml.
 from .analytics import (
     run_linear_regression,
     run_correlation,
+    run_pca,
     run_pca_clustering,
     run_t_test,
     run_equipment_oee,
 )
 
 # module_arg -> (run_fn, result_key)
+# "pca" is unchanged for pca_clustering jobs (--modules pca → nested pca_clustering result).
+# "pca_mvda" is the litewave-compatible flat PCA response (method_id pca).
 MODULE_REGISTRY = {
     "lr": (run_linear_regression, "linear_regression"),
     "corr": (run_correlation, "correlation"),
     "pca": (run_pca_clustering, "pca_clustering"),
+    "pca_mvda": (run_pca, "pca"),
     "ttest": (run_t_test, "t_test"),
     "oee": (run_equipment_oee, "equipment_oee"),
 }

{batch_analytics-0.3.30 → batch_analytics-0.3.34}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.30
-Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
+Version: 0.3.34
+Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test, LLM classification).
 Author: Litewave Analytics Team
 License: MIT
 Requires-Python: >=3.8

{batch_analytics-0.3.30 → batch_analytics-0.3.34}/src/batch_analytics.egg-info/SOURCES.txt

@@ -20,7 +20,10 @@ src/batch_analytics/analytics/equipment_oee.py
 src/batch_analytics/analytics/gluon_autogluon_infer.py
 src/batch_analytics/analytics/gluon_autogluon_train.py
 src/batch_analytics/analytics/linear_regression.py
+src/batch_analytics/analytics/pca.py
 src/batch_analytics/analytics/pca_clustering.py
+src/batch_analytics/analytics/pca_core.py
+src/batch_analytics/analytics/pca_mvda.py
 src/batch_analytics/analytics/t_test.py
 src/batch_analytics/output/__init__.py
 src/batch_analytics/output/base.py
@@ -28,4 +31,5 @@ src/batch_analytics/output/clickhouse.py
 src/batch_analytics/output/local.py
 src/batch_analytics/output/s3.py
 src/batch_analytics/utils/__init__.py
-src/batch_analytics/utils/gluon_autogluon_common.py
+src/batch_analytics/utils/gluon_autogluon_common.py
+tests/test_pca_mvda.py

batch_analytics-0.3.34/tests/test_pca_mvda.py

@@ -0,0 +1,86 @@
+"""Unit tests for MVDA PCA response helpers (no Spark required)."""
+
+import importlib.util
+from pathlib import Path
+
+import numpy as np
+
+_SRC = Path(__file__).resolve().parents[1] / "src" / "batch_analytics" / "analytics" / "pca_mvda.py"
+_spec = importlib.util.spec_from_file_location("pca_mvda", _SRC)
+pca_mvda = importlib.util.module_from_spec(_spec)
+assert _spec.loader is not None
+_spec.loader.exec_module(pca_mvda)
+
+build_loadings_and_dominant = pca_mvda.build_loadings_and_dominant
+build_mvda_pca_response = pca_mvda.build_mvda_pca_response
+empty_mvda_response = pca_mvda.empty_mvda_response
+ensure_target_in_features = pca_mvda.ensure_target_in_features
+
+
+class _FakePcaVec:
+    def __init__(self, arr):
+        self._arr = arr
+
+    def toArray(self):
+        return self._arr
+
+
+class _FakeRow:
+    def __init__(self, pca_vec, target_val, feature_cols, target_feature):
+        self._data = {"pca_features": _FakePcaVec(pca_vec)}
+        for f in feature_cols:
+            self._data[f] = 1.0 if f != target_feature else target_val
+
+    def __getitem__(self, key):
+        return self._data[key]
+
+
+def test_ensure_target_prepended():
+    cols, target = ensure_target_in_features(
+        ["a", "b"], "actual_yield"
+    )
+    assert cols[0] == "actual_yield"
+    assert target == "actual_yield"
+
+
+def test_empty_mvda_insufficient_rows():
+    out = empty_mvda_response(5, 10)
+    assert out["n_components"] == 0
+    assert out["row_count"] == 5
+    assert "need at least 10" in out["message"]
+
+
+def test_build_loadings_shape():
+    features = ["x", "y", "z"]
+    pc = np.array([[0.9, 0.1], [0.2, 0.8], [0.5, 0.5]])
+    loadings, dominant = build_loadings_and_dominant(features, pc, 2)
+    assert len(loadings) == 6
+    assert len(dominant) == 2
+    assert all("pc" in d and "feature" in d for d in dominant)
+
+
+def test_build_mvda_response_scores_lengths():
+    features = ["actual_yield", "feat_b"]
+    explained = [0.6, 0.4]
+    pc_matrix = np.array([[0.7, 0.3], [0.5, 0.5]])
+    rows = [
+        _FakeRow([1.0, 0.5], 100.0, features, "actual_yield"),
+        _FakeRow([-0.5, 1.0], 200.0, features, "actual_yield"),
+    ]
+    out = build_mvda_pca_response(
+        features,
+        "actual_yield",
+        explained,
+        pc_matrix,
+        rows,
+        row_count=102,
+        min_rows=10,
+    )
+    assert out["n_components"] == 2
+    assert out["message"] is None
+    assert out["row_count"] == 102
+    assert len(out["scores"]["pc1"]) == 2
+    assert len(out["scores"]["pc2"]) == 2
+    assert len(out["scores"]["target_values"]) == 2
+    assert out["cumulative_variance"][-1] == 1.0
+    assert len(out["loadings"]) == 4