batch-analytics 0.3.30__tar.gz → 0.3.35__tar.gz

{batch_analytics-0.3.30 → batch_analytics-0.3.35}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.30
-Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
+Version: 0.3.35
+Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test, LLM classification).
 Author: Litewave Analytics Team
 License: MIT
 Requires-Python: >=3.8

{batch_analytics-0.3.30 → batch_analytics-0.3.35}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.30"
-description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
+version = "0.3.35"
+description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test, LLM classification)."
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [

{batch_analytics-0.3.30 → batch_analytics-0.3.35}/src/batch_analytics/analytics/__init__.py

@@ -8,6 +8,7 @@ Analytics modules for batch analytics pipeline.
 
 from .linear_regression import run_linear_regression
 from .correlation import run_correlation
+from .pca import run_pca
 from .pca_clustering import run_pca_clustering
 from .t_test import run_t_test
 from .equipment_oee import run_equipment_oee
@@ -15,6 +16,7 @@ from .equipment_oee import run_equipment_oee
 __all__ = [
     "run_linear_regression",
     "run_correlation",
+    "run_pca",
     "run_pca_clustering",
     "run_t_test",
     "run_equipment_oee",

batch_analytics-0.3.35/src/batch_analytics/analytics/pca.py

@@ -0,0 +1,70 @@
+"""
+Module: MVDA PCA on staged data (litewave-analytics POST /pca compatible response).
+"""
+
+import logging
+from typing import Any, Dict
+
+from pyspark.sql import DataFrame, SparkSession
+
+from ..config import BatchAnalyticsConfig
+from .pca_core import (
+    build_mvda_pca_response,
+    empty_mvda_response,
+    ensure_target_in_features,
+    fit_pca_and_collect_scores,
+    prepare_pca_dataframe,
+    resolve_n_components,
+    resolve_pca_feature_cols,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def run_pca(
+    spark: SparkSession,
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> Dict[str, Any]:
+    """
+    Principal Component Analysis with MVDA response shape.
+
+    Returns flat dict: n_components, explained_variance, cumulative_variance,
+    dominant_features, loadings, scores, row_count, message.
+    """
+    del spark  # SparkSession kept for MODULE_REGISTRY signature consistency
+    min_rows = config.analytics.pca_min_rows
+    sample_limit = config.analytics.pca_sample_size
+    target_feature = (config.analytics.pca_target_feature or "").strip()
+
+    feature_cols = resolve_pca_feature_cols(df, config)
+    if len(feature_cols) < 2:
+        raise ValueError("At least 2 features required for PCA")
+
+    feature_cols, target_feature = ensure_target_in_features(
+        feature_cols, target_feature
+    )
+
+    df_scaled, _df_num, row_count = prepare_pca_dataframe(df, feature_cols)
+    if row_count < min_rows:
+        return empty_mvda_response(row_count, min_rows)
+
+    k = resolve_n_components(len(feature_cols), config)
+    explained, pc_matrix, _pca_model, pca_rows = fit_pca_and_collect_scores(
+        df_scaled,
+        feature_cols,
+        target_feature,
+        k,
+        row_count,
+        sample_limit,
+    )
+
+    return build_mvda_pca_response(
+        feature_cols,
+        target_feature,
+        explained,
+        pc_matrix,
+        pca_rows,
+        row_count,
+        min_rows,
+    )

batch_analytics-0.3.35/src/batch_analytics/analytics/pca_core.py

@@ -0,0 +1,124 @@
+"""
+Shared PCA Spark pipeline helpers.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import List, Tuple
+
+from pyspark.ml.feature import StandardScaler, VectorAssembler
+from pyspark.sql import DataFrame
+from pyspark.sql.functions import col
+from pyspark.sql.types import DoubleType
+
+from ..config import BatchAnalyticsConfig
+from .pca_mvda import (
+    build_mvda_pca_response,
+    empty_mvda_response,
+    ensure_target_in_features,
+    build_loadings_and_dominant,
+)
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "resolve_pca_feature_cols",
+    "ensure_target_in_features",
+    "empty_mvda_response",
+    "build_loadings_and_dominant",
+    "build_mvda_pca_response",
+    "prepare_pca_dataframe",
+    "resolve_n_components",
+    "fit_pca_and_collect_scores",
+]
+
+
+def resolve_pca_feature_cols(df: DataFrame, config: BatchAnalyticsConfig) -> List[str]:
+    """Resolve feature column names from config or auto-detect numerics."""
+    feature_cols = [
+        c.strip()
+        for c in config.analytics.pca_features.split(",")
+        if c.strip()
+    ]
+    if not feature_cols:
+        feature_cols = [
+            f.name
+            for f in df.schema.fields
+            if "double" in str(f.dataType).lower()
+            or "int" in str(f.dataType).lower()
+            or "long" in str(f.dataType).lower()
+            or "float" in str(f.dataType).lower()
+        ]
+        logger.info("Auto-selected %d numeric columns for PCA", len(feature_cols))
+    return feature_cols
+
+
+def prepare_pca_dataframe(
+    df: DataFrame, feature_cols: List[str]
+) -> Tuple[DataFrame, DataFrame, int]:
+    """Cast features, drop NA, assemble and scale. Returns (df_scaled, df_num, row_count)."""
+    missing = [c for c in feature_cols if c not in df.columns]
+    if missing:
+        raise ValueError(
+            f"PCA features not found: {missing}. Available: {df.columns[:15]}..."
+        )
+
+    df_num = df.select(
+        *[col(c).cast(DoubleType()).alias(c) for c in feature_cols]
+    ).dropna()
+    row_count = df_num.count()
+
+    assembler = VectorAssembler(
+        inputCols=feature_cols,
+        outputCol="features_raw",
+        handleInvalid="skip",
+    )
+    df_vec = assembler.transform(df_num)
+
+    scaler = StandardScaler(
+        inputCol="features_raw",
+        outputCol="features",
+        withStd=True,
+        withMean=True,
+    )
+    scaler_model = scaler.fit(df_vec)
+    df_scaled = scaler_model.transform(df_vec)
+    return df_scaled, df_num, row_count
+
+
+def resolve_n_components(n_features: int, config: BatchAnalyticsConfig) -> int:
+    raw = (config.analytics.pca_n_components or "").strip()
+    if not raw:
+        return n_features
+    return min(int(raw), n_features)
+
+
+def fit_pca_and_collect_scores(
+    df_scaled: DataFrame,
+    feature_cols: List[str],
+    target_feature: str,
+    k: int,
+    row_count: int,
+    sample_limit: int,
+):
+    """Fit Spark PCA and collect rows for scores scatter (pca_features + features)."""
+    from pyspark.ml.feature import PCA as SparkPCA
+
+    pca = SparkPCA(k=k, inputCol="features", outputCol="pca_features")
+    pca_model = pca.fit(df_scaled)
+
+    explained = pca_model.explainedVariance.toArray().tolist()
+    pc_matrix = pca_model.pc.toArray()
+
+    pca_rows = []
+    if sample_limit > 0:
+        if row_count > sample_limit:
+            fraction = min(sample_limit / row_count * 1.2, 1.0)
+            sample_df = df_scaled.sample(fraction=fraction, seed=42)
+        else:
+            sample_df = df_scaled
+        result_df = pca_model.transform(sample_df)
+        select_cols = ["pca_features"] + feature_cols
+        pca_rows = result_df.select(*select_cols).limit(sample_limit).collect()
+    return explained, pc_matrix, pca_model, pca_rows

batch_analytics-0.3.35/src/batch_analytics/analytics/pca_mvda.py

@@ -0,0 +1,126 @@
+"""
+MVDA PCA response builders (no PySpark). Compatible with litewave-analytics POST /pca.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Tuple
+
+INSUFFICIENT_DATA_TEMPLATE = (
+    "Insufficient data — only {row_count} data-points available (need at least {min_rows})"
+)
+
+
+def ensure_target_in_features(
+    feature_cols: List[str], target_feature: str
+) -> Tuple[List[str], str]:
+    """Match litewave: target is included in the PCA feature set."""
+    target = (target_feature or "").strip()
+    if not target and feature_cols:
+        target = feature_cols[0]
+    cols = list(feature_cols)
+    if target and target not in cols:
+        cols = [target] + cols
+    return cols, target
+
+
+def empty_mvda_response(
+    row_count: int, min_rows: int, message: Optional[str] = None
+) -> Dict[str, Any]:
+    return {
+        "n_components": 0,
+        "explained_variance": [],
+        "cumulative_variance": [],
+        "dominant_features": [],
+        "loadings": [],
+        "scores": {
+            "pc1_label": "",
+            "pc1": [],
+            "pc2_label": "",
+            "pc2": [],
+            "target_values": [],
+        },
+        "row_count": row_count,
+        "message": message
+        or INSUFFICIENT_DATA_TEMPLATE.format(row_count=row_count, min_rows=min_rows),
+    }
+
+
+def build_loadings_and_dominant(
+    feature_cols: List[str], pc_matrix, n_components: int
+) -> Tuple[List[dict], List[dict]]:
+    """pc_matrix shape (n_features, k) — Spark PCA pc layout."""
+    loadings: List[dict] = []
+    dominant_features: List[dict] = []
+    for pc_i in range(n_components):
+        vec = pc_matrix[:, pc_i]
+        max_idx = int(max(range(len(vec)), key=lambda j: abs(float(vec[j]))))
+        dominant_features.append({"pc": pc_i + 1, "feature": feature_cols[max_idx]})
+        for j, name in enumerate(feature_cols):
+            loadings.append(
+                {"pc": pc_i + 1, "feature": name, "value": round(float(vec[j]), 6)}
+            )
+    return loadings, dominant_features
+
+
+def build_mvda_pca_response(
+    feature_cols: List[str],
+    target_feature: str,
+    explained: List[float],
+    pc_matrix,
+    pca_rows: List,
+    row_count: int,
+    min_rows: int,
+) -> Dict[str, Any]:
+    """Build flat MVDA PCA payload (litewave POST /pca compatible)."""
+    if row_count < min_rows:
+        return empty_mvda_response(row_count, min_rows)
+
+    n_components = len(explained)
+    cumsum: List[float] = []
+    s = 0.0
+    for v in explained:
+        s += v
+        cumsum.append(round(s, 6))
+
+    loadings, dominant_features = build_loadings_and_dominant(
+        feature_cols, pc_matrix, n_components
+    )
+
+    def _pc_label(pc_idx: int) -> str:
+        feat = (
+            dominant_features[pc_idx]["feature"]
+            if pc_idx < len(dominant_features)
+            else ""
+        )
+        pct = round(explained[pc_idx] * 100, 1) if pc_idx < len(explained) else 0
+        return f"PC{pc_idx + 1} \u2014 {feat} ({pct}%)"
+
+    scores: Dict[str, Any] = {
+        "pc1_label": _pc_label(0),
+        "pc1": [],
+        "pc2_label": _pc_label(1) if n_components >= 2 else "",
+        "pc2": [],
+        "target_values": [],
+    }
+    for row in pca_rows:
+        pca_vec = row["pca_features"].toArray()
+        scores["pc1"].append(round(float(pca_vec[0]), 4))
+        if n_components >= 2 and len(pca_vec) > 1:
+            scores["pc2"].append(round(float(pca_vec[1]), 4))
+        if target_feature:
+            target_val = row[target_feature]
+            scores["target_values"].append(
+                round(float(target_val), 4) if target_val is not None else None
+            )
+
+    return {
+        "n_components": n_components,
+        "explained_variance": [round(v, 6) for v in explained],
+        "cumulative_variance": cumsum,
+        "dominant_features": dominant_features,
+        "loadings": loadings,
+        "scores": scores,
+        "row_count": row_count,
+        "message": None,
+    }

{batch_analytics-0.3.30 → batch_analytics-0.3.35}/src/batch_analytics/config.py

@@ -85,15 +85,24 @@ class TransformConfig:
             "/tmp/analytics_stage",
         )
     )
-    # Output format for load_staged when reading (parquet/delta/clickhouse).
+    # Output format for load_staged when reading (parquet/delta/clickhouse/iceberg).
     # Stage job always writes to ClickHouse; use clickhouse for analytics to read from staged table.
+    # Use iceberg to read Nessie Iceberg gold/silver tables (BATCH_STAGING_TABLE = catalog.ns.table).
     staging_format: str = field(
         default_factory=lambda: os.environ.get("BATCH_STAGING_FORMAT", "clickhouse")
     )
-    # Staging table name in ClickHouse (when format=clickhouse)
+    # Staging table: ClickHouse table name, or Iceberg identifier e.g. nessie.gold.batch_yield_features
     staging_table: str = field(
         default_factory=lambda: os.environ.get("BATCH_STAGING_TABLE", "analytics_staging")
     )
+    # Optional SQL WHERE fragment when reading staging (no leading WHERE).
+    staging_filter: str = field(
+        default_factory=lambda: os.environ.get("BATCH_STAGING_FILTER", "").strip()
+    )
+    # Optional comma-separated columns when reading staging (empty = SELECT *).
+    staging_columns: str = field(
+        default_factory=lambda: os.environ.get("BATCH_STAGING_COLUMNS", "").strip()
+    )
     # Spark save mode for ClickHouse staging (and path staging): overwrite | append
     staging_write_mode: str = field(
         default_factory=lambda: os.environ.get("BATCH_STAGING_WRITE_MODE", "overwrite")
@@ -162,8 +171,12 @@ class AnalyticsConfig:
     )
     corr_threshold: float = float(os.environ.get("BATCH_CORR_THRESHOLD", "0.8"))
 
-    # Module 3: PCA + Clustering
+    # Module 3: PCA (MVDA) and PCA + clustering
     pca_features: str = os.environ.get("BATCH_PCA_FEATURES", "")
+    pca_target_feature: str = os.environ.get("BATCH_PCA_TARGET_FEATURE", "")
+    pca_n_components: str = os.environ.get("BATCH_PCA_N_COMPONENTS", "")
+    pca_sample_size: int = int(os.environ.get("BATCH_PCA_SAMPLE_SIZE", "5000"))
+    pca_min_rows: int = int(os.environ.get("BATCH_PCA_MIN_ROWS", "10"))
     pca_variance_threshold: float = float(
         os.environ.get("BATCH_PCA_VARIANCE_THRESHOLD", "0.95")
     )
@@ -207,11 +220,18 @@ class SparkK8sConfig:
     executor_cores: int = int(os.environ.get("SPARK_EXECUTOR_CORES", "1"))
     executor_memory: str = os.environ.get("SPARK_EXECUTOR_MEMORY", "512m")
     executor_memory_overhead: str = os.environ.get("SPARK_EXECUTOR_MEMORY_OVERHEAD", "128m")
-    # S3 (optional; set for s3a:// paths)
+    # S3 (optional; set for s3a:// paths / Iceberg warehouse)
     s3_access_key: str = os.environ.get("AWS_ACCESS_KEY_ID", "")
     s3_secret_key: str = os.environ.get("AWS_SECRET_ACCESS_KEY", "")
-    s3_endpoint: str = os.environ.get("AWS_ENDPOINT", "s3.amazonaws.com")
-    s3_region: str = os.environ.get("AWS_REGION", "us-east-2")
+    # Prefer S3_ENDPOINT (lakehouse MinIO); fall back to AWS_ENDPOINT for legacy.
+    s3_endpoint: str = os.environ.get(
+        "S3_ENDPOINT",
+        os.environ.get("AWS_ENDPOINT", ""),
+    )
+    s3_region: str = os.environ.get(
+        "AWS_DEFAULT_REGION",
+        os.environ.get("AWS_REGION", "us-east-2"),
+    )
 
 
 @dataclass

{batch_analytics-0.3.30 → batch_analytics-0.3.35}/src/batch_analytics/job_runner.py

@@ -161,6 +161,9 @@ def create_spark_session(
                 read_codec,
             )
 
+    builder = _apply_iceberg_nessie_catalog(builder, config)
+    builder = _apply_s3a_lakehouse(builder, config)
+
     if cfg.master.startswith("k8s://"):
         driver_host = socket.gethostbyname(socket.gethostname())
         builder = (
@@ -188,19 +191,91 @@ def create_spark_session(
             .config("spark.kubernetes.executor.serviceAccountName", cfg.service_account)
             .config("spark.kubernetes.container.image.pullPolicy", "IfNotPresent")
         )
-        if cfg.s3_access_key and cfg.s3_secret_key:
-            builder = (
-                builder.config("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
-                .config("spark.hadoop.fs.s3a.access.key", cfg.s3_access_key)
-                .config("spark.hadoop.fs.s3a.secret.key", cfg.s3_secret_key)
-                .config("spark.hadoop.fs.s3a.endpoint", cfg.s3_endpoint)
-                .config("spark.hadoop.fs.s3a.endpoint.region", cfg.s3_region)
-            )
         logger.info("Spark on Kubernetes: master=%s", cfg.master)
 
     return builder.getOrCreate()
 
 
+def _apply_iceberg_nessie_catalog(builder, config: BatchAnalyticsConfig):
+    """Register Nessie Iceberg catalog when lakehouse env is present."""
+    nessie_uri = (os.environ.get("NESSIE_URI") or "").strip()
+    warehouse = (os.environ.get("ICEBERG_WAREHOUSE") or "").strip()
+    if not nessie_uri and not warehouse:
+        return builder
+    catalog = (os.environ.get("ICEBERG_CATALOG") or "nessie").strip() or "nessie"
+    ref = (os.environ.get("NESSIE_REF") or "main").strip() or "main"
+    if not nessie_uri:
+        nessie_uri = "http://nessie:19120/api/v2"
+    if not warehouse:
+        warehouse = "s3a://lake/"
+    logger.info(
+        "Configuring Iceberg catalog %s uri=%s ref=%s warehouse=%s",
+        catalog,
+        nessie_uri,
+        ref,
+        warehouse,
+    )
+    return (
+        builder.config(
+            "spark.sql.extensions",
+            "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+        )
+        .config(f"spark.sql.catalog.{catalog}", "org.apache.iceberg.spark.SparkCatalog")
+        .config(
+            f"spark.sql.catalog.{catalog}.catalog-impl",
+            "org.apache.iceberg.nessie.NessieCatalog",
+        )
+        .config(f"spark.sql.catalog.{catalog}.uri", nessie_uri)
+        .config(f"spark.sql.catalog.{catalog}.ref", ref)
+        .config(f"spark.sql.catalog.{catalog}.warehouse", warehouse)
+        .config(
+            f"spark.sql.catalog.{catalog}.io-impl",
+            "org.apache.iceberg.hadoop.HadoopFileIO",
+        )
+    )
+
+
+def _apply_s3a_lakehouse(builder, config: BatchAnalyticsConfig):
+    """S3A settings for Iceberg warehouse / s3a staging (AWS or MinIO via S3_ENDPOINT)."""
+    cfg = config.spark_k8s
+    if not (cfg.s3_access_key or cfg.s3_secret_key or os.environ.get("ICEBERG_WAREHOUSE")):
+        return builder
+
+    builder = builder.config(
+        "spark.hadoop.fs.s3a.impl",
+        "org.apache.hadoop.fs.s3a.S3AFileSystem",
+    )
+    if cfg.s3_access_key:
+        builder = builder.config("spark.hadoop.fs.s3a.access.key", cfg.s3_access_key)
+    if cfg.s3_secret_key:
+        builder = builder.config("spark.hadoop.fs.s3a.secret.key", cfg.s3_secret_key)
+
+    endpoint = (cfg.s3_endpoint or "").strip()
+    region = (cfg.s3_region or "us-east-2").strip()
+    if endpoint:
+        # Custom/MinIO endpoint
+        ssl = not endpoint.startswith("http://")
+        builder = (
+            builder.config("spark.hadoop.fs.s3a.endpoint", endpoint)
+            .config("spark.hadoop.fs.s3a.path.style.access", "true")
+            .config(
+                "spark.hadoop.fs.s3a.connection.ssl.enabled",
+                "true" if ssl else "false",
+            )
+        )
+    else:
+        builder = (
+            builder.config(
+                "spark.hadoop.fs.s3a.endpoint",
+                f"https://s3.{region}.amazonaws.com",
+            )
+            .config("spark.hadoop.fs.s3a.endpoint.region", region)
+            .config("spark.hadoop.fs.s3a.path.style.access", "false")
+            .config("spark.hadoop.fs.s3a.connection.ssl.enabled", "true")
+        )
+    return builder
+
+
 def run_pipeline(
     config: Optional[BatchAnalyticsConfig] = None,
     spark: Optional[SparkSession] = None,

{batch_analytics-0.3.30 → batch_analytics-0.3.35}/src/batch_analytics/modules.py

@@ -8,16 +8,20 @@ See analytics_runner/catalog/analytics_catalog.yaml.
 from .analytics import (
     run_linear_regression,
     run_correlation,
+    run_pca,
     run_pca_clustering,
     run_t_test,
     run_equipment_oee,
 )
 
 # module_arg -> (run_fn, result_key)
+# "pca" is unchanged for pca_clustering jobs (--modules pca → nested pca_clustering result).
+# "pca_mvda" is the litewave-compatible flat PCA response (method_id pca).
 MODULE_REGISTRY = {
     "lr": (run_linear_regression, "linear_regression"),
     "corr": (run_correlation, "correlation"),
     "pca": (run_pca_clustering, "pca_clustering"),
+    "pca_mvda": (run_pca, "pca"),
     "ttest": (run_t_test, "t_test"),
     "oee": (run_equipment_oee, "equipment_oee"),
 }

{batch_analytics-0.3.30 → batch_analytics-0.3.35}/src/batch_analytics/transform.py

@@ -534,33 +534,108 @@ def load_staged(
 ) -> DataFrame:
     """
     Load previously staged data (e.g. when running only analytics modules).
+
+    Formats:
+      parquet / delta — path under BATCH_STAGING_PATH
+      clickhouse — BATCH_STAGING_TABLE in ClickHouse
+      iceberg — Nessie Iceberg table (BATCH_STAGING_TABLE = catalog.ns.table)
     """
     staging_path = config.transform.staging_path
-    fmt = config.transform.staging_format
+    fmt = (config.transform.staging_format or "").strip().lower()
 
     if fmt == "parquet":
         return spark.read.parquet(staging_path)
     if fmt == "delta":
         return spark.read.format("delta").load(staging_path)
+    if fmt == "iceberg":
+        return _load_staged_iceberg(spark, config)
     if fmt == "clickhouse":
-        ch = config.clickhouse
-        tbl = config.transform.staging_table
-        cat = os.environ.get("BATCH_CLICKHOUSE_CATALOG", "batch_ch").strip()
-        if cat:
-            try:
-                return spark.table(f"{cat}.{ch.database}.{tbl}")
-            except Exception as e:
-                logger.warning(
-                    "load_staged: catalog table %s.%s.%s failed (%s), using JDBC",
-                    cat,
-                    ch.database,
-                    tbl,
-                    e,
-                )
-        dbtable = f"(SELECT * FROM `{ch.database}`.`{tbl}`) AS _stg"
-        return spark.read.jdbc(
-            ch.jdbc_url,
-            dbtable,
-            properties=ch.jdbc_properties,
-        )
+        return _load_staged_clickhouse(spark, config)
     return spark.read.format(fmt).load(staging_path)
+
+
+def _staging_select_list(config: BatchAnalyticsConfig) -> str:
+    cols = (config.transform.staging_columns or "").strip()
+    if not cols:
+        return "*"
+    return ", ".join(c.strip() for c in cols.split(",") if c.strip())
+
+
+def _staging_where_clause(config: BatchAnalyticsConfig) -> str:
+    filt = (config.transform.staging_filter or "").strip()
+    if not filt:
+        return ""
+    if filt.lower().startswith("where "):
+        filt = filt[6:].strip()
+    return f" WHERE {filt}" if filt else ""
+
+
+def _resolve_iceberg_table_name(config: BatchAnalyticsConfig) -> str:
+    """Return fully-qualified Iceberg table id (catalog.namespace.table)."""
+    raw = (config.transform.staging_table or "").strip()
+    if not raw:
+        raise ValueError(
+            "BATCH_STAGING_TABLE is required when BATCH_STAGING_FORMAT=iceberg "
+            "(e.g. nessie.gold.batch_yield_features)"
+        )
+    catalog = (os.environ.get("ICEBERG_CATALOG") or "nessie").strip()
+    parts = [p for p in raw.replace("/", ".").split(".") if p]
+    if len(parts) == 1:
+        return f"{catalog}.gold.{parts[0]}"
+    if len(parts) == 2:
+        return f"{catalog}.{parts[0]}.{parts[1]}"
+    return ".".join(parts)
+
+
+def _load_staged_iceberg(
+    spark: SparkSession,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    table = _resolve_iceberg_table_name(config)
+    sql = (
+        f"SELECT {_staging_select_list(config)} FROM {table}"
+        f"{_staging_where_clause(config)}"
+    )
+    logger.info("load_staged iceberg: %s", sql)
+    return spark.sql(sql)
+
+
+def _load_staged_clickhouse(
+    spark: SparkSession,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    ch = config.clickhouse
+    tbl = config.transform.staging_table
+    where = _staging_where_clause(config)
+    select_list = _staging_select_list(config)
+    cat = os.environ.get("BATCH_CLICKHOUSE_CATALOG", "batch_ch").strip()
+    if cat and not where and select_list == "*":
+        try:
+            return spark.table(f"{cat}.{ch.database}.{tbl}")
+        except Exception as e:
+            logger.warning(
+                "load_staged: catalog table %s.%s.%s failed (%s), using JDBC",
+                cat,
+                ch.database,
+                tbl,
+                e,
+            )
+    elif cat and (where or select_list != "*"):
+        sql = f"SELECT {select_list} FROM {cat}.{ch.database}.{tbl}{where}"
+        try:
+            logger.info("load_staged clickhouse catalog: %s", sql)
+            return spark.sql(sql)
+        except Exception as e:
+            logger.warning(
+                "load_staged: catalog SQL failed (%s), using JDBC",
+                e,
+            )
+    dbtable = (
+        f"(SELECT {select_list} FROM `{ch.database}`.`{tbl}`{where}) AS _stg"
+    )
+    logger.info("load_staged clickhouse jdbc: %s", dbtable)
+    return spark.read.jdbc(
+        ch.jdbc_url,
+        dbtable,
+        properties=ch.jdbc_properties,
+    )

{batch_analytics-0.3.30 → batch_analytics-0.3.35}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.30
-Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
+Version: 0.3.35
+Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test, LLM classification).
 Author: Litewave Analytics Team
 License: MIT
 Requires-Python: >=3.8

{batch_analytics-0.3.30 → batch_analytics-0.3.35}/src/batch_analytics.egg-info/SOURCES.txt

@@ -20,7 +20,10 @@ src/batch_analytics/analytics/equipment_oee.py
 src/batch_analytics/analytics/gluon_autogluon_infer.py
 src/batch_analytics/analytics/gluon_autogluon_train.py
 src/batch_analytics/analytics/linear_regression.py
+src/batch_analytics/analytics/pca.py
 src/batch_analytics/analytics/pca_clustering.py
+src/batch_analytics/analytics/pca_core.py
+src/batch_analytics/analytics/pca_mvda.py
 src/batch_analytics/analytics/t_test.py
 src/batch_analytics/output/__init__.py
 src/batch_analytics/output/base.py
@@ -28,4 +31,6 @@ src/batch_analytics/output/clickhouse.py
 src/batch_analytics/output/local.py
 src/batch_analytics/output/s3.py
 src/batch_analytics/utils/__init__.py
-src/batch_analytics/utils/gluon_autogluon_common.py
+src/batch_analytics/utils/gluon_autogluon_common.py
+tests/test_iceberg_staging.py
+tests/test_pca_mvda.py

batch_analytics-0.3.35/tests/test_iceberg_staging.py

@@ -0,0 +1,85 @@
+"""Unit tests for Iceberg staging load helpers."""
+
+from __future__ import annotations
+
+import os
+import unittest
+from unittest.mock import MagicMock, patch
+
+from batch_analytics.config import BatchAnalyticsConfig, TransformConfig
+from batch_analytics.transform import _load_staged_iceberg, _resolve_iceberg_table_name
+
+
+class ResolveIcebergTableNameTests(unittest.TestCase):
+    def test_full_name_passthrough(self) -> None:
+        cfg = BatchAnalyticsConfig(
+            transform=TransformConfig(staging_table="nessie.gold.batch_yield_features")
+        )
+        self.assertEqual(
+            _resolve_iceberg_table_name(cfg),
+            "nessie.gold.batch_yield_features",
+        )
+
+    def test_two_part_gets_catalog(self) -> None:
+        cfg = BatchAnalyticsConfig(
+            transform=TransformConfig(staging_table="gold.batch_yield_features")
+        )
+        with patch.dict(os.environ, {"ICEBERG_CATALOG": "nessie"}, clear=False):
+            self.assertEqual(
+                _resolve_iceberg_table_name(cfg),
+                "nessie.gold.batch_yield_features",
+            )
+
+    def test_one_part_defaults_to_gold(self) -> None:
+        cfg = BatchAnalyticsConfig(
+            transform=TransformConfig(staging_table="batch_yield_features")
+        )
+        with patch.dict(os.environ, {"ICEBERG_CATALOG": "nessie"}, clear=False):
+            self.assertEqual(
+                _resolve_iceberg_table_name(cfg),
+                "nessie.gold.batch_yield_features",
+            )
+
+
+class LoadStagedIcebergTests(unittest.TestCase):
+    def test_filter_and_columns(self) -> None:
+        spark = MagicMock()
+        spark.sql.return_value = MagicMock(name="df")
+        cfg = BatchAnalyticsConfig(
+            transform=TransformConfig(
+                staging_table="nessie.gold.batch_yield_features",
+                staging_filter="product = 'Gabapentin'",
+                staging_columns="batch_no,actual_yield_qty,batch_seq",
+            )
+        )
+        _load_staged_iceberg(spark, cfg)
+        spark.sql.assert_called_once()
+        sql = spark.sql.call_args[0][0]
+        self.assertIn("SELECT batch_no, actual_yield_qty, batch_seq FROM", sql)
+        self.assertIn("WHERE product = 'Gabapentin'", sql)
+
+
+class LoadStagedClickhouseTests(unittest.TestCase):
+    def test_filter_pushed_to_jdbc(self) -> None:
+        from batch_analytics.transform import _load_staged_clickhouse
+
+        spark = MagicMock()
+        spark.read.jdbc.return_value = MagicMock(name="df")
+        cfg = BatchAnalyticsConfig(
+            transform=TransformConfig(
+                staging_table="analytics_staging",
+                staging_filter="batch_id = 'B1'",
+                staging_columns="batch_id,qty",
+            )
+        )
+        cfg.clickhouse.database = "example_db"
+        with patch.dict(os.environ, {"BATCH_CLICKHOUSE_CATALOG": ""}, clear=False):
+            _load_staged_clickhouse(spark, cfg)
+        spark.read.jdbc.assert_called_once()
+        dbtable = spark.read.jdbc.call_args[0][1]
+        self.assertIn("SELECT batch_id, qty FROM", dbtable)
+        self.assertIn("WHERE batch_id = 'B1'", dbtable)
+
+
+if __name__ == "__main__":
+    unittest.main()

batch_analytics-0.3.35/tests/test_pca_mvda.py

@@ -0,0 +1,86 @@
+"""Unit tests for MVDA PCA response helpers (no Spark required)."""
+
+import importlib.util
+from pathlib import Path
+
+import numpy as np
+
+_SRC = Path(__file__).resolve().parents[1] / "src" / "batch_analytics" / "analytics" / "pca_mvda.py"
+_spec = importlib.util.spec_from_file_location("pca_mvda", _SRC)
+pca_mvda = importlib.util.module_from_spec(_spec)
+assert _spec.loader is not None
+_spec.loader.exec_module(pca_mvda)
+
+build_loadings_and_dominant = pca_mvda.build_loadings_and_dominant
+build_mvda_pca_response = pca_mvda.build_mvda_pca_response
+empty_mvda_response = pca_mvda.empty_mvda_response
+ensure_target_in_features = pca_mvda.ensure_target_in_features
+
+
+class _FakePcaVec:
+    def __init__(self, arr):
+        self._arr = arr
+
+    def toArray(self):
+        return self._arr
+
+
+class _FakeRow:
+    def __init__(self, pca_vec, target_val, feature_cols, target_feature):
+        self._data = {"pca_features": _FakePcaVec(pca_vec)}
+        for f in feature_cols:
+            self._data[f] = 1.0 if f != target_feature else target_val
+
+    def __getitem__(self, key):
+        return self._data[key]
+
+
+def test_ensure_target_prepended():
+    cols, target = ensure_target_in_features(
+        ["a", "b"], "actual_yield"
+    )
+    assert cols[0] == "actual_yield"
+    assert target == "actual_yield"
+
+
+def test_empty_mvda_insufficient_rows():
+    out = empty_mvda_response(5, 10)
+    assert out["n_components"] == 0
+    assert out["row_count"] == 5
+    assert "need at least 10" in out["message"]
+
+
+def test_build_loadings_shape():
+    features = ["x", "y", "z"]
+    pc = np.array([[0.9, 0.1], [0.2, 0.8], [0.5, 0.5]])
+    loadings, dominant = build_loadings_and_dominant(features, pc, 2)
+    assert len(loadings) == 6
+    assert len(dominant) == 2
+    assert all("pc" in d and "feature" in d for d in dominant)
+
+
+def test_build_mvda_response_scores_lengths():
+    features = ["actual_yield", "feat_b"]
+    explained = [0.6, 0.4]
+    pc_matrix = np.array([[0.7, 0.3], [0.5, 0.5]])
+    rows = [
+        _FakeRow([1.0, 0.5], 100.0, features, "actual_yield"),
+        _FakeRow([-0.5, 1.0], 200.0, features, "actual_yield"),
+    ]
+    out = build_mvda_pca_response(
+        features,
+        "actual_yield",
+        explained,
+        pc_matrix,
+        rows,
+        row_count=102,
+        min_rows=10,
+    )
+    assert out["n_components"] == 2
+    assert out["message"] is None
+    assert out["row_count"] == 102
+    assert len(out["scores"]["pc1"]) == 2
+    assert len(out["scores"]["pc2"]) == 2
+    assert len(out["scores"]["target_values"]) == 2
+    assert out["cumulative_variance"][-1] == 1.0
+    assert len(out["loadings"]) == 4