batch-analytics 0.1.0__tar.gz

batch_analytics-0.1.0/PKG-INFO

@@ -0,0 +1,80 @@
+Metadata-Version: 2.4
+Name: batch-analytics
+Version: 0.1.0
+Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
+Author: Analytics Team
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pyspark<3.6,>=3.4
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Provides-Extra: s3
+Requires-Dist: boto3>=1.28; extra == "s3"
+Provides-Extra: clickhouse
+Requires-Dist: clickhouse-connect>=0.7; extra == "clickhouse"
+Provides-Extra: output
+Requires-Dist: boto3>=1.28; extra == "output"
+Requires-Dist: clickhouse-connect>=0.7; extra == "output"
+
+# Batch Analytics
+
+PySpark-based analytics pipeline for ClickHouse data: **Extract** → **Transform** → **Stage** → **Analytics**. Designed to run as the main application inside a Spark driver container (invoked by `analytics_runners` via SparkApplication CRD).
+
+## Bundle contents
+
+Only the files required for the batch analytics job runner:
+
+```
+analytics/
+├── pyproject.toml
+├── requirements-batch.txt
+├── README.md
+└── src/
+    └── batch_analytics/
+        ├── __init__.py
+        ├── __main__.py        # python -m batch_analytics
+        ├── job_runner.py      # Entry point
+        ├── config.py
+        ├── extract.py
+        ├── transform.py
+        ├── log.py
+        ├── README.md
+        └── analytics/
+            ├── __init__.py
+            ├── linear_regression.py
+            ├── correlation.py
+            ├── pca_clustering.py
+            └── t_test.py
+```
+
+## Install
+
+```bash
+pip install -e .
+# or: pip install -r requirements-batch.txt && pip install -e .
+```
+
+## Run
+
+```bash
+# Via module
+python -m batch_analytics
+
+# Via CLI (after pip install -e .)
+batch-analytics
+
+# Full pipeline
+batch-analytics
+
+# Analytics only (from staged ClickHouse table)
+batch-analytics --from-stage --modules lr corr pca ttest
+```
+
+## Configuration
+
+See `src/batch_analytics/README.md` for environment variables and usage.
+
+## Docker image
+
+For Spark on Kubernetes, build an image that includes this package and exposes `job_runner.py` at the path used by `mainApplicationFile` (e.g. `local:///opt/analytics/job_runner.py`).

batch_analytics-0.1.0/README.md

@@ -0,0 +1,61 @@
+# Batch Analytics
+
+PySpark-based analytics pipeline for ClickHouse data: **Extract** → **Transform** → **Stage** → **Analytics**. Designed to run as the main application inside a Spark driver container (invoked by `analytics_runners` via SparkApplication CRD).
+
+## Bundle contents
+
+Only the files required for the batch analytics job runner:
+
+```
+analytics/
+├── pyproject.toml
+├── requirements-batch.txt
+├── README.md
+└── src/
+    └── batch_analytics/
+        ├── __init__.py
+        ├── __main__.py        # python -m batch_analytics
+        ├── job_runner.py      # Entry point
+        ├── config.py
+        ├── extract.py
+        ├── transform.py
+        ├── log.py
+        ├── README.md
+        └── analytics/
+            ├── __init__.py
+            ├── linear_regression.py
+            ├── correlation.py
+            ├── pca_clustering.py
+            └── t_test.py
+```
+
+## Install
+
+```bash
+pip install -e .
+# or: pip install -r requirements-batch.txt && pip install -e .
+```
+
+## Run
+
+```bash
+# Via module
+python -m batch_analytics
+
+# Via CLI (after pip install -e .)
+batch-analytics
+
+# Full pipeline
+batch-analytics
+
+# Analytics only (from staged ClickHouse table)
+batch-analytics --from-stage --modules lr corr pca ttest
+```
+
+## Configuration
+
+See `src/batch_analytics/README.md` for environment variables and usage.
+
+## Docker image
+
+For Spark on Kubernetes, build an image that includes this package and exposes `job_runner.py` at the path used by `mainApplicationFile` (e.g. `local:///opt/analytics/job_runner.py`).

batch_analytics-0.1.0/pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "batch-analytics"
+version = "0.1.0"
+description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
+readme = "README.md"
+requires-python = ">=3.9"
+dependencies = [
+    "pyspark>=3.4,<3.6",
+]
+authors = [{ name = "Analytics Team" }]
+license = { text = "MIT" }
+
+[project.optional-dependencies]
+dev = ["pytest>=7.0"]
+s3 = ["boto3>=1.28"]
+clickhouse = ["clickhouse-connect>=0.7"]
+output = ["boto3>=1.28", "clickhouse-connect>=0.7"]
+
+[project.scripts]
+batch-analytics = "batch_analytics.job_runner:main"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["batch_analytics*"]

batch_analytics-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

batch_analytics-0.1.0/src/batch_analytics/__init__.py

@@ -0,0 +1,44 @@
+"""
+Batch analytics pipeline: Extract, Transform, Log stages + analytics modules.
+
+Stages:
+  - Extract: Load data from ClickHouse via Spark ClickHouse connector or JDBC
+  - Transform: Deduplicate and stage data (parquet/delta/clickhouse)
+  - Log: Persist run metadata and analytics results
+
+Analytics modules:
+  - Module 1: Linear regression (XY) with slope comparison across groups
+  - Module 2: Multi-feature correlation
+  - Module 3: PCA + KMeans clustering
+"""
+
+from .config import BatchAnalyticsConfig, SparkK8sConfig
+from .extract import extract_all, extract_table, extract_unified
+from .transform import (
+    extract_anchor_id,
+    load_staged,
+    remove_duplicates,
+    stage_to_clickhouse,
+    transform,
+    transform_and_stage,
+)
+from .log import log_analytics_artifacts, log_run
+from .job_runner import run_pipeline, create_spark_session
+
+__all__ = [
+    "BatchAnalyticsConfig",
+    "SparkK8sConfig",
+    "extract_anchor_id",
+    "extract_all",
+    "extract_table",
+    "extract_unified",
+    "remove_duplicates",
+    "stage_to_clickhouse",
+    "transform",
+    "transform_and_stage",
+    "load_staged",
+    "log_run",
+    "log_analytics_artifacts",
+    "run_pipeline",
+    "create_spark_session",
+]

batch_analytics-0.1.0/src/batch_analytics/__main__.py

@@ -0,0 +1,5 @@
+"""Allow python -m batch_analytics to run the job runner."""
+
+from .job_runner import main
+import sys
+sys.exit(main())

batch_analytics-0.1.0/src/batch_analytics/analytics/__init__.py

@@ -0,0 +1,19 @@
+"""
+Analytics modules for batch analytics pipeline.
+- Module 1: Linear regression (XY) and slope comparison
+- Module 2: Multi-feature correlation
+- Module 3: PCA and clustering
+- Module 4: T-test to compare means of two groups
+"""
+
+from .linear_regression import run_linear_regression
+from .correlation import run_correlation
+from .pca_clustering import run_pca_clustering
+from .t_test import run_t_test
+
+__all__ = [
+    "run_linear_regression",
+    "run_correlation",
+    "run_pca_clustering",
+    "run_t_test",
+]

batch_analytics-0.1.0/src/batch_analytics/analytics/correlation.py

@@ -0,0 +1,113 @@
+"""
+Module 2: Multi-feature correlation analysis.
+"""
+
+import logging
+from typing import Any
+
+from pyspark.ml.feature import VectorAssembler
+from pyspark.ml.stat import Correlation
+from pyspark.sql import DataFrame, SparkSession
+
+from ..config import BatchAnalyticsConfig
+
+logger = logging.getLogger(__name__)
+
+
+def run_correlation(
+    spark: SparkSession,
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> dict[str, Any]:
+    """
+    Compute correlation matrix over multiple numeric features.
+    Optionally identify pairs above a threshold (collinearity).
+
+    Returns:
+        - correlation_matrix: full matrix (list of lists)
+        - feature_names: column order
+        - high_corr_pairs: pairs with |corr| >= threshold
+        - threshold: used threshold
+    """
+    feature_cols = [
+        c.strip()
+        for c in config.analytics.corr_features.split(",")
+        if c.strip()
+    ]
+
+    if not feature_cols:
+        # Auto-select numeric columns
+        feature_cols = [
+            f.name
+            for f in df.schema.fields
+            if "double" in str(f.dataType).lower()
+            or "int" in str(f.dataType).lower()
+            or "long" in str(f.dataType).lower()
+            or "float" in str(f.dataType).lower()
+        ]
+        logger.info("Auto-selected %d numeric columns for correlation", len(feature_cols))
+
+    missing = [c for c in feature_cols if c not in df.columns]
+    if missing:
+        raise ValueError(
+            f"Correlation features not found: {missing}. "
+            f"Available: {df.columns[:15]}..."
+        )
+
+    if len(feature_cols) < 2:
+        return {
+            "correlation_matrix": [[1.0]],
+            "feature_names": feature_cols,
+            "high_corr_pairs": [],
+            "threshold": config.analytics.corr_threshold,
+        }
+
+    # Cast to double and drop nulls
+    from pyspark.sql.functions import col
+    from pyspark.sql.types import DoubleType
+
+    df_num = df.select(
+        *[col(c).cast(DoubleType()).alias(c) for c in feature_cols]
+    ).dropna()
+
+    assembler = VectorAssembler(
+        inputCols=feature_cols,
+        outputCol="features",
+        handleInvalid="skip",
+    )
+    df_vec = assembler.transform(df_num)
+
+    corr_matrix = Correlation.corr(df_vec, "features", "pearson")
+    corr_row = corr_matrix.head()
+    if corr_row is None:
+        return {
+            "correlation_matrix": [],
+            "feature_names": feature_cols,
+            "high_corr_pairs": [],
+            "threshold": config.analytics.corr_threshold,
+        }
+
+    import numpy as np
+
+    arr = corr_row[0].toArray()
+    matrix = np.asarray(arr)
+
+    threshold = config.analytics.corr_threshold
+    high_pairs: list[dict] = []
+    n = len(feature_cols)
+    for i in range(n):
+        for j in range(i + 1, n):
+            val = float(matrix[i, j])
+            if abs(val) >= threshold:
+                high_pairs.append({
+                    "feature_a": feature_cols[i],
+                    "feature_b": feature_cols[j],
+                    "correlation": val,
+                })
+
+    return {
+        "correlation_matrix": matrix.tolist(),
+        "feature_names": feature_cols,
+        "high_corr_pairs": high_pairs,
+        "threshold": threshold,
+    }

batch_analytics-0.1.0/src/batch_analytics/analytics/linear_regression.py

@@ -0,0 +1,136 @@
+"""
+Module 1: Simple linear regression on XY data with slope comparison across groups.
+"""
+
+import logging
+from typing import Any
+
+from pyspark.ml.feature import VectorAssembler
+from pyspark.ml.regression import LinearRegression
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.functions import col, concat_ws
+from pyspark.sql.types import DoubleType
+
+from ..config import BatchAnalyticsConfig
+
+logger = logging.getLogger(__name__)
+
+
+def _ensure_numeric(df: DataFrame, x_col: str, y_col: str) -> DataFrame:
+    """Cast X and Y columns to double."""
+    return df.select(
+        col(x_col).cast(DoubleType()).alias(x_col),
+        col(y_col).cast(DoubleType()).alias(y_col),
+        *[c for c in df.columns if c not in (x_col, y_col)],
+    ).filter(col(x_col).isNotNull() & col(y_col).isNotNull())
+
+
+def run_linear_regression(
+    spark: SparkSession,
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> dict[str, Any]:
+    """
+    Run simple linear regression: Y ~ X.
+    If group columns are configured, fit separate models per group and compare slopes.
+
+    Returns:
+        - slopes: per-group slope and intercept
+        - global: single regression over all data
+        - slope_comparison: pairwise slope differences (when multiple groups)
+    """
+    x_col = config.analytics.lr_x_column
+    y_col = config.analytics.lr_y_column
+    group_cols = [
+        c.strip()
+        for c in config.analytics.lr_group_columns.split(",")
+        if c.strip()
+    ]
+
+    # Ensure required columns exist
+    if x_col not in df.columns or y_col not in df.columns:
+        numeric_cols = [f.name for f in df.schema.fields if "double" in str(f.dataType).lower() or "int" in str(f.dataType).lower()]
+        if len(numeric_cols) >= 2:
+            x_col = numeric_cols[0]
+            y_col = numeric_cols[1]
+            logger.warning(
+                "LR columns not found, using first two numeric: %s, %s",
+                x_col,
+                y_col,
+            )
+        else:
+            raise ValueError(
+                f"Could not find LR columns (x={x_col}, y={y_col}). "
+                "Specify BATCH_LR_X_COLUMN and BATCH_LR_Y_COLUMN."
+            )
+
+    df = _ensure_numeric(df, x_col, y_col)
+
+    assembler = VectorAssembler(inputCols=[x_col], outputCol="features", handleInvalid="skip")
+    df_vec = assembler.transform(df).withColumnRenamed(y_col, "label")
+
+    # Global regression (all data)
+    lr = LinearRegression(featuresCol="features", labelCol="label")
+    global_model = lr.fit(df_vec)
+    global_slope = float(global_model.coefficients[0])
+    global_intercept = float(global_model.intercept)
+    global_r2 = float(global_model.summary.r2)
+
+    result: dict[str, Any] = {
+        "global": {
+            "slope": global_slope,
+            "intercept": global_intercept,
+            "r2": global_r2,
+        },
+        "slopes": {},
+        "slope_comparison": [],
+    }
+
+    if not group_cols or not all(g in df.columns for g in group_cols):
+        return result
+
+    # Per-group regression
+    group_key = "_group_key"
+    df_grouped = df_vec.withColumn(
+        group_key,
+        concat_ws("|", *[col(g).cast("string") for g in group_cols]),
+    )
+    groups = df_grouped.select(group_key).distinct().collect()
+
+    slopes_by_group: dict[str, dict] = {}
+    for row in groups:
+        key_str = row[group_key]
+        if key_str is None:
+            continue
+        key_parts = key_str.split("|")
+
+        sub_df = df_grouped.filter(col(group_key) == key_str).drop(group_key)
+        if sub_df.count() < 2:
+            continue
+
+        model = lr.fit(sub_df)
+        slopes_by_group[key_str] = {
+            "slope": float(model.coefficients[0]),
+            "intercept": float(model.intercept),
+            "r2": float(model.summary.r2),
+            "n": sub_df.count(),
+            "group": dict(zip(group_cols, key_parts)),
+        }
+
+    result["slopes"] = slopes_by_group
+
+    # Slope comparison: pairwise differences
+    keys = list(slopes_by_group.keys())
+    for i in range(len(keys)):
+        for j in range(i + 1, len(keys)):
+            s1 = slopes_by_group[keys[i]]["slope"]
+            s2 = slopes_by_group[keys[j]]["slope"]
+            result["slope_comparison"].append({
+                "group_a": keys[i],
+                "group_b": keys[j],
+                "slope_a": s1,
+                "slope_b": s2,
+                "slope_diff": s1 - s2,
+            })
+
+    return result

batch_analytics-0.1.0/src/batch_analytics/analytics/pca_clustering.py

@@ -0,0 +1,143 @@
+"""
+Module 3: PCA for key feature identification and clustering on staged data.
+"""
+
+import logging
+from typing import Any
+
+from pyspark.ml.clustering import KMeans
+from pyspark.ml.feature import PCA, StandardScaler, VectorAssembler
+from pyspark.sql import DataFrame, SparkSession
+
+from ..config import BatchAnalyticsConfig
+
+logger = logging.getLogger(__name__)
+
+
+def run_pca_clustering(
+    spark: SparkSession,
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> dict[str, Any]:
+    """
+    Run PCA to identify key features (principal components) and KMeans clustering.
+    Uses scaled features; PCA components capture variance; clustering groups similar rows.
+
+    Returns:
+        - pca: explained variance, cumulative variance, feature loadings per PC
+        - clustering: cluster assignments, centroids, sizes
+    """
+    feature_cols = [
+        c.strip()
+        for c in config.analytics.pca_features.split(",")
+        if c.strip()
+    ]
+
+    if not feature_cols:
+        feature_cols = [
+            f.name
+            for f in df.schema.fields
+            if "double" in str(f.dataType).lower()
+            or "int" in str(f.dataType).lower()
+            or "long" in str(f.dataType).lower()
+            or "float" in str(f.dataType).lower()
+        ]
+        logger.info("Auto-selected %d numeric columns for PCA/clustering", len(feature_cols))
+
+    missing = [c for c in feature_cols if c not in df.columns]
+    if missing:
+        raise ValueError(
+            f"PCA features not found: {missing}. Available: {df.columns[:15]}..."
+        )
+
+    if len(feature_cols) < 2:
+        return {
+            "pca": {"explained_variance": [1.0], "feature_names": feature_cols},
+            "clustering": {"k": config.analytics.cluster_k, "sizes": []},
+        }
+
+    from pyspark.sql.functions import col
+    from pyspark.sql.types import DoubleType
+
+    df_num = df.select(
+        *[col(c).cast(DoubleType()).alias(c) for c in feature_cols]
+    ).dropna()
+
+    assembler = VectorAssembler(
+        inputCols=feature_cols,
+        outputCol="features_raw",
+        handleInvalid="skip",
+    )
+    df_vec = assembler.transform(df_num)
+
+    scaler = StandardScaler(
+        inputCol="features_raw",
+        outputCol="features",
+        withStd=True,
+        withMean=True,
+    )
+    scaler_model = scaler.fit(df_vec)
+    df_scaled = scaler_model.transform(df_vec)
+
+    # PCA - keep enough components for target variance
+    variance_threshold = config.analytics.pca_variance_threshold
+    n_comp_max = min(len(feature_cols), 20)
+    pca = PCA(k=n_comp_max, inputCol="features", outputCol="pca_features")
+    pca_model = pca.fit(df_scaled)
+
+    explained = pca_model.explainedVariance.toArray().tolist()
+    cumsum = []
+    s = 0.0
+    for v in explained:
+        s += v
+        cumsum.append(s)
+
+    # Feature loadings per PC (which original features contribute most)
+    components = pca_model.pc.toArray()
+    loadings: list[dict] = []
+    for i, comp in enumerate(components):
+        ranked = sorted(
+            zip(feature_cols, comp.tolist()),
+            key=lambda x: abs(x[1]),
+            reverse=True,
+        )
+        loadings.append({
+            "pc": i + 1,
+            "top_features": [{"name": n, "loading": float(v)} for n, v in ranked[:5]],
+        })
+
+    k = config.analytics.cluster_k
+    kmeans = KMeans(k=k, seed=42, featuresCol="pca_features", predictionCol="cluster")
+    kmeans_model = kmeans.fit(df_scaled)
+
+    df_clustered = kmeans_model.transform(df_scaled)
+    cluster_sizes = (
+        df_clustered.groupBy("cluster")
+        .count()
+        .orderBy("cluster")
+        .collect()
+    )
+    sizes = {int(r["cluster"]): int(r["count"]) for r in cluster_sizes}
+
+    centroids = []
+    for i in range(k):
+        c = kmeans_model.clusterCenters()[i]
+        centroids.append({"cluster": i, "centroid": c.tolist()})
+
+    return {
+        "pca": {
+            "explained_variance": explained,
+            "cumulative_variance": cumsum,
+            "feature_names": feature_cols,
+            "component_loadings": loadings,
+            "n_components": len(explained),
+        },
+        "clustering": {
+            "k": k,
+            "sizes": sizes,
+            "centroids": centroids,
+            "within_cluster_sum_of_squared_distances": float(
+                kmeans_model.summary.trainingCost
+            ),
+        },
+    }