batch-analytics 0.3.18__tar.gz → 0.3.21__tar.gz

{batch_analytics-0.3.18 → batch_analytics-0.3.21}/PKG-INFO

@@ -1,16 +1,17 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.18
+Version: 0.3.21
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: pyspark<3.6,>=3.4
 Requires-Dist: numpy>=1.19.0
-Requires-Dist: scipy>=1.5.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
+Provides-Extra: spark
+Requires-Dist: pyspark<3.6,>=3.4; extra == "spark"
+Requires-Dist: scipy>=1.5.0; extra == "spark"
 Provides-Extra: ttest
 Requires-Dist: scipy>=1.5.0; extra == "ttest"
 Provides-Extra: s3
@@ -23,18 +24,19 @@ Requires-Dist: boto3>=1.28; extra == "output"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "output"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "output"
 Provides-Extra: autogluon
-Requires-Dist: autogluon<2.0,>=1.0; extra == "autogluon"
+Requires-Dist: autogluon-tabular[lightgbm]<2.0,>=1.0; extra == "autogluon"
 Requires-Dist: pandas>=1.3.0; extra == "autogluon"
 Requires-Dist: boto3>=1.28; extra == "autogluon"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "autogluon"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "autogluon"
 Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "autogluon"
 Provides-Extra: full
+Requires-Dist: pyspark<3.6,>=3.4; extra == "full"
 Requires-Dist: scipy>=1.5.0; extra == "full"
 Requires-Dist: boto3>=1.28; extra == "full"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "full"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "full"
-Requires-Dist: autogluon<2.0,>=1.0; extra == "full"
+Requires-Dist: autogluon-tabular[lightgbm]<2.0,>=1.0; extra == "full"
 Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "full"
 
 # Batch Analytics
@@ -71,12 +73,13 @@ analytics/
 
 ## Install
 
+- `pip install batch-analytics` — core (numpy only)
+- `pip install batch-analytics[spark]` — PySpark ETL + job runner + SciPy (t-test / ANOVA)
+- `pip install batch-analytics[autogluon]` — AutoGluon + I/O (Gluon image; no PySpark)
+- `pip install batch-analytics[full]` — PySpark + AutoGluon + I/O
+
 ```bash
-pip install -e .
-# or install every runtime dependency used anywhere in the package, then editable:
-pip install -r requirements.txt && pip install -e .
-# PyPI install includes numpy and scipy (t-test); extras: s3, clickhouse, output, full
-pip install "batch-analytics[full]"
+pip install -e ".[spark]"   # dev
 ```
 
 ## Run

{batch_analytics-0.3.18 → batch_analytics-0.3.21}/README.md

@@ -32,12 +32,13 @@ analytics/
 
 ## Install
 
+- `pip install batch-analytics` — core (numpy only)
+- `pip install batch-analytics[spark]` — PySpark ETL + job runner + SciPy (t-test / ANOVA)
+- `pip install batch-analytics[autogluon]` — AutoGluon + I/O (Gluon image; no PySpark)
+- `pip install batch-analytics[full]` — PySpark + AutoGluon + I/O
+
 ```bash
-pip install -e .
-# or install every runtime dependency used anywhere in the package, then editable:
-pip install -r requirements.txt && pip install -e .
-# PyPI install includes numpy and scipy (t-test); extras: s3, clickhouse, output, full
-pip install "batch-analytics[full]"
+pip install -e ".[spark]"   # dev
 ```
 
 ## Run

{batch_analytics-0.3.18 → batch_analytics-0.3.21}/pyproject.toml

@@ -4,22 +4,22 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.18"
+version = "0.3.21"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
-    "pyspark>=3.4,<3.6",
     "numpy>=1.19.0",
-    # Welch t-test (t_test.py); keep on core deps so `pip install batch-analytics` works in minimal driver images
-    "scipy>=1.5.0",
 ]
 authors = [{ name = "Litewave Analytics Team" }]
 license = { text = "MIT" }
 
 [project.optional-dependencies]
 dev = ["pytest>=7.0"]
-# Legacy: scipy is a core dependency; kept so `pip install "batch-analytics[ttest]"` still resolves.
+spark = [
+    "pyspark>=3.4,<3.6",
+    "scipy>=1.5.0",
+]
 ttest = ["scipy>=1.5.0"]
 s3 = ["boto3>=1.28"]
 clickhouse = [
@@ -31,22 +31,22 @@ output = [
     "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
     "clickhouse-connect>=0.7; python_version >= '3.9'",
 ]
-# AutoGluon Tabular train/infer (POST /submit/train, /submit/inf on analytics_runner). Requires Python 3.9+.
+# autogluon-tabular[lightgbm]: tabular AutoML with LightGBM only (no PyTorch from full autogluon metapackage)
 autogluon = [
-    "autogluon>=1.0,<2.0",
+    "autogluon-tabular[lightgbm]>=1.0,<2.0",
     "pandas>=1.3.0",
     "boto3>=1.28",
     "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
     "clickhouse-connect>=0.7; python_version >= '3.9'",
     "pyarrow>=10.0.0; python_version >= '3.8'",
 ]
-# Install all optional runtime deps used anywhere in the package
 full = [
+    "pyspark>=3.4,<3.6",
     "scipy>=1.5.0",
     "boto3>=1.28",
     "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
     "clickhouse-connect>=0.7; python_version >= '3.9'",
-    "autogluon>=1.0,<2.0",
+    "autogluon-tabular[lightgbm]>=1.0,<2.0",
     "pyarrow>=10.0.0; python_version >= '3.8'",
 ]
 

batch_analytics-0.3.21/src/batch_analytics/__init__.py

@@ -0,0 +1,63 @@
+"""
+Batch analytics pipeline: Extract, Transform, Log stages + analytics modules.
+
+PySpark is optional: install ``batch-analytics[spark]``. Gluon image: ``[autogluon]`` only.
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any
+
+from .config import BatchAnalyticsConfig, SparkK8sConfig
+
+__all__ = [
+    "BatchAnalyticsConfig",
+    "SparkK8sConfig",
+    "expand_kv_blob_column",
+    "extract_anchor_id",
+    "extract_all",
+    "extract_table",
+    "extract_unified",
+    "parse_extract_filter_values",
+    "remove_duplicates",
+    "stage_to_clickhouse",
+    "transform",
+    "transform_and_stage",
+    "load_staged",
+    "log_run",
+    "log_analytics_artifacts",
+    "run_pipeline",
+    "create_spark_session",
+]
+
+# Lazy imports so ``import batch_analytics`` works without PySpark (Gluon image).
+_LAZY = {
+    "expand_kv_blob_column": ("transform", "expand_kv_blob_column"),
+    "extract_anchor_id": ("transform", "extract_anchor_id"),
+    "extract_all": ("extract", "extract_all"),
+    "extract_table": ("extract", "extract_table"),
+    "extract_unified": ("extract", "extract_unified"),
+    "parse_extract_filter_values": ("extract", "parse_extract_filter_values"),
+    "remove_duplicates": ("transform", "remove_duplicates"),
+    "stage_to_clickhouse": ("transform", "stage_to_clickhouse"),
+    "transform": ("transform", "transform"),
+    "transform_and_stage": ("transform", "transform_and_stage"),
+    "load_staged": ("transform", "load_staged"),
+    "log_run": ("log", "log_run"),
+    "log_analytics_artifacts": ("log", "log_analytics_artifacts"),
+    "run_pipeline": ("job_runner", "run_pipeline"),
+    "create_spark_session": ("job_runner", "create_spark_session"),
+}
+
+
+def __getattr__(name: str) -> Any:
+    if name in _LAZY:
+        mod_name, attr = _LAZY[name]
+        mod = importlib.import_module(f".{mod_name}", __name__)
+        return getattr(mod, attr)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__() -> list[str]:
+    return sorted({*__all__, *globals().keys()})

{batch_analytics-0.3.18 → batch_analytics-0.3.21}/src/batch_analytics/__main__.py

@@ -2,4 +2,5 @@
 
 from .job_runner import main
 import sys
+
 sys.exit(main())

{batch_analytics-0.3.18 → batch_analytics-0.3.21}/src/batch_analytics/analytics/gluon_autogluon_train.py

@@ -4,11 +4,13 @@ Train AutoGluon TabularPredictor from ClickHouse staging data; upload artifacts
 Env (injected by analytics_runner Gluon job):
   CLICKHOUSE_*, BATCH_STAGING_TABLE, MODEL_S3_PREFIX, TASK_ID,
   AUTOGLUON_LABEL, AUTOGLUON_FEATURES, AUTOGLUON_PROBLEM_TYPE, AUTOGLUON_TIME_LIMIT,
-  optional AUTOGLUON_MAX_ROWS
+  optional AUTOGLUON_MAX_ROWS,
+  optional AUTOGLUON_HYPERPARAMETERS — JSON object of AutoGluon model hyperparameters; default {"GBM": {}} (LightGBM only, no PyTorch)
 """
 
 from __future__ import annotations
 
+import json
 import logging
 import os
 import shutil
@@ -44,6 +46,22 @@ def _require(name: str) -> str:
     return v
 
 
+def _hyperparameters_from_env() -> dict:
+    """Default: GBM (LightGBM) only — matches autogluon-tabular[lightgbm] and avoids NN/torch models."""
+    raw = os.environ.get("AUTOGLUON_HYPERPARAMETERS", "").strip()
+    if not raw:
+        return {"GBM": {}}
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError as e:
+        logger.error("AUTOGLUON_HYPERPARAMETERS must be valid JSON: %s", e)
+        sys.exit(2)
+    if not isinstance(parsed, dict):
+        logger.error("AUTOGLUON_HYPERPARAMETERS must be a JSON object, e.g. {\"GBM\": {}}")
+        sys.exit(2)
+    return parsed
+
+
 def main() -> None:
     model_prefix = _require("MODEL_S3_PREFIX")
     label = _require("AUTOGLUON_LABEL")
@@ -92,10 +110,12 @@ def main() -> None:
     os.makedirs(local_dir, exist_ok=True)
 
     train_df = df[feature_list + [label]]
+    hyper = _hyperparameters_from_env()
     logger.info(
-        "Fitting TabularPredictor problem_type=%s time_limit=%ss rows=%s",
+        "Fitting TabularPredictor problem_type=%s time_limit=%ss hyperparameters=%s rows=%s",
         problem_type,
         time_limit,
+        hyper,
         len(train_df),
     )
     predictor = TabularPredictor(
@@ -103,7 +123,7 @@ def main() -> None:
         problem_type=problem_type,
         path=local_dir,
     )
-    predictor.fit(train_df, time_limit=time_limit)
+    predictor.fit(train_df, time_limit=time_limit, hyperparameters=hyper)
 
     logger.info("Uploading model artifacts to %s", model_prefix)
     upload_directory_to_s3(local_dir, model_prefix)

{batch_analytics-0.3.18 → batch_analytics-0.3.21}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,16 +1,17 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.18
+Version: 0.3.21
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: pyspark<3.6,>=3.4
 Requires-Dist: numpy>=1.19.0
-Requires-Dist: scipy>=1.5.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
+Provides-Extra: spark
+Requires-Dist: pyspark<3.6,>=3.4; extra == "spark"
+Requires-Dist: scipy>=1.5.0; extra == "spark"
 Provides-Extra: ttest
 Requires-Dist: scipy>=1.5.0; extra == "ttest"
 Provides-Extra: s3
@@ -23,18 +24,19 @@ Requires-Dist: boto3>=1.28; extra == "output"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "output"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "output"
 Provides-Extra: autogluon
-Requires-Dist: autogluon<2.0,>=1.0; extra == "autogluon"
+Requires-Dist: autogluon-tabular[lightgbm]<2.0,>=1.0; extra == "autogluon"
 Requires-Dist: pandas>=1.3.0; extra == "autogluon"
 Requires-Dist: boto3>=1.28; extra == "autogluon"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "autogluon"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "autogluon"
 Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "autogluon"
 Provides-Extra: full
+Requires-Dist: pyspark<3.6,>=3.4; extra == "full"
 Requires-Dist: scipy>=1.5.0; extra == "full"
 Requires-Dist: boto3>=1.28; extra == "full"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "full"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "full"
-Requires-Dist: autogluon<2.0,>=1.0; extra == "full"
+Requires-Dist: autogluon-tabular[lightgbm]<2.0,>=1.0; extra == "full"
 Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "full"
 
 # Batch Analytics
@@ -71,12 +73,13 @@ analytics/
 
 ## Install
 
+- `pip install batch-analytics` — core (numpy only)
+- `pip install batch-analytics[spark]` — PySpark ETL + job runner + SciPy (t-test / ANOVA)
+- `pip install batch-analytics[autogluon]` — AutoGluon + I/O (Gluon image; no PySpark)
+- `pip install batch-analytics[full]` — PySpark + AutoGluon + I/O
+
 ```bash
-pip install -e .
-# or install every runtime dependency used anywhere in the package, then editable:
-pip install -r requirements.txt && pip install -e .
-# PyPI install includes numpy and scipy (t-test); extras: s3, clickhouse, output, full
-pip install "batch-analytics[full]"
+pip install -e ".[spark]"   # dev
 ```
 
 ## Run

{batch_analytics-0.3.18 → batch_analytics-0.3.21}/src/batch_analytics.egg-info/requires.txt

@@ -1,9 +1,7 @@
-pyspark<3.6,>=3.4
 numpy>=1.19.0
-scipy>=1.5.0
 
 [autogluon]
-autogluon<2.0,>=1.0
+autogluon-tabular[lightgbm]<2.0,>=1.0
 pandas>=1.3.0
 boto3>=1.28
 
@@ -28,9 +26,10 @@ clickhouse-connect>=0.7
 pytest>=7.0
 
 [full]
+pyspark<3.6,>=3.4
 scipy>=1.5.0
 boto3>=1.28
-autogluon<2.0,>=1.0
+autogluon-tabular[lightgbm]<2.0,>=1.0
 
 [full:python_version < "3.9"]
 clickhouse-connect<0.9,>=0.7
@@ -53,5 +52,9 @@ clickhouse-connect>=0.7
 [s3]
 boto3>=1.28
 
+[spark]
+pyspark<3.6,>=3.4
+scipy>=1.5.0
+
 [ttest]
 scipy>=1.5.0

batch_analytics-0.3.18/src/batch_analytics/__init__.py

@@ -1,52 +0,0 @@
-"""
-Batch analytics pipeline: Extract, Transform, Log stages + analytics modules.
-
-Stages:
-  - Extract: Load data from ClickHouse via Spark ClickHouse connector or JDBC
-  - Transform: Deduplicate and stage data (parquet/delta/clickhouse)
-  - Log: Persist run metadata and analytics results
-
-Analytics modules:
-  - Module 1: Linear regression (XY) with slope comparison across groups
-  - Module 2: Multi-feature correlation
-  - Module 3: PCA + KMeans clustering
-"""
-
-from .config import BatchAnalyticsConfig, SparkK8sConfig
-from .extract import (
-    extract_all,
-    extract_table,
-    extract_unified,
-    parse_extract_filter_values,
-)
-from .transform import (
-    expand_kv_blob_column,
-    extract_anchor_id,
-    load_staged,
-    remove_duplicates,
-    stage_to_clickhouse,
-    transform,
-    transform_and_stage,
-)
-from .log import log_analytics_artifacts, log_run
-from .job_runner import run_pipeline, create_spark_session
-
-__all__ = [
-    "BatchAnalyticsConfig",
-    "SparkK8sConfig",
-    "expand_kv_blob_column",
-    "extract_anchor_id",
-    "extract_all",
-    "extract_table",
-    "extract_unified",
-    "parse_extract_filter_values",
-    "remove_duplicates",
-    "stage_to_clickhouse",
-    "transform",
-    "transform_and_stage",
-    "load_staged",
-    "log_run",
-    "log_analytics_artifacts",
-    "run_pipeline",
-    "create_spark_session",
-]