batch-analytics 0.2.8__tar.gz → 0.3.0__tar.gz

{batch_analytics-0.2.8 → batch_analytics-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.8
+Version: 0.3.0
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
@@ -15,14 +15,17 @@ Requires-Dist: scipy>=1.5.0; extra == "ttest"
 Provides-Extra: s3
 Requires-Dist: boto3>=1.28; extra == "s3"
 Provides-Extra: clickhouse
-Requires-Dist: clickhouse-connect>=0.7; extra == "clickhouse"
+Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "clickhouse"
+Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "clickhouse"
 Provides-Extra: output
 Requires-Dist: boto3>=1.28; extra == "output"
-Requires-Dist: clickhouse-connect>=0.7; extra == "output"
+Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "output"
+Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "output"
 Provides-Extra: full
 Requires-Dist: scipy>=1.5.0; extra == "full"
 Requires-Dist: boto3>=1.28; extra == "full"
-Requires-Dist: clickhouse-connect>=0.7; extra == "full"
+Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "full"
+Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "full"
 
 # Batch Analytics
 

{batch_analytics-0.2.8 → batch_analytics-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.2.8"
+version = "0.3.0"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"
@@ -20,10 +20,23 @@ dev = ["pytest>=7.0"]
 # Welch t-test module (batch_analytics.analytics.t_test)
 ttest = ["scipy>=1.5.0"]
 s3 = ["boto3>=1.28"]
-clickhouse = ["clickhouse-connect>=0.7"]
-output = ["boto3>=1.28", "clickhouse-connect>=0.7"]
+# 0.9+ uses list[...] etc. and breaks on Python 3.8; 3.9+ can take current clickhouse-connect.
+clickhouse = [
+    "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
+    "clickhouse-connect>=0.7; python_version >= '3.9'",
+]
+output = [
+    "boto3>=1.28",
+    "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
+    "clickhouse-connect>=0.7; python_version >= '3.9'",
+]
 # Install all optional runtime deps used anywhere in the package
-full = ["scipy>=1.5.0", "boto3>=1.28", "clickhouse-connect>=0.7"]
+full = [
+    "scipy>=1.5.0",
+    "boto3>=1.28",
+    "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
+    "clickhouse-connect>=0.7; python_version >= '3.9'",
+]
 
 [project.scripts]
 batch-analytics = "batch_analytics.job_runner:main"

{batch_analytics-0.2.8 → batch_analytics-0.3.0}/src/batch_analytics/job_runner.py

@@ -13,7 +13,7 @@ from typing import Dict, List, Optional
 
 from pyspark.sql import SparkSession
 
-from .config import BatchAnalyticsConfig
+from .config import BatchAnalyticsConfig, SparkK8sConfig
 from .extract import extract_unified
 from .log import log_dataframe_summary, log_run
 from .modules import DEFAULT_MODULES, MODULE_REGISTRY, VALID_MODULES
@@ -27,6 +27,21 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
+def _spark_on_kubernetes_for_classpath(cfg: SparkK8sConfig) -> bool:
+    """
+    True when Spark executors run on Kubernetes.
+
+    Spark Operator cluster mode passes --master k8s://... to spark-submit; SPARK_MASTER is often
+    unset, so SparkK8sConfig.master defaults to local[*] while the JVM still uses Kubernetes.
+    Image-baked JARs must go on extraClassPath in that case (spark.jars breaks executors).
+    """
+    if cfg.master.startswith("k8s://"):
+        return True
+    if os.environ.get("SPARK_MASTER", "").strip().startswith("k8s://"):
+        return True
+    return bool(os.environ.get("KUBERNETES_SERVICE_HOST"))
+
+
 def create_spark_session(
     app_name: str = "BatchAnalytics",
     clickhouse_jars: Optional[str] = None,
@@ -64,13 +79,13 @@ def create_spark_session(
             else:
                 packages.append(part)
 
-    if cfg.master.startswith("k8s://"):
+    if _spark_on_kubernetes_for_classpath(cfg):
         packages.append("org.apache.hadoop:hadoop-aws:3.3.4")
 
     # Spark-on-K8s: absolute paths in spark.jars are re-sent to executors as ./basename.jar and
     # fail there ("Unable to create executor due to ./clickhouse-spark-runtime-..."). JARs baked
     # into the driver/executor image belong on the JVM classpath instead.
-    if cfg.master.startswith("k8s://") and jar_list:
+    if _spark_on_kubernetes_for_classpath(cfg) and jar_list:
         local_cp: List[str] = []
         remote_jars: List[str] = []
         for p in jar_list:
@@ -83,6 +98,10 @@ def create_spark_session(
                 remote_jars.append(p)
         if local_cp:
             joined = ":".join(local_cp)
+            logger.info(
+                "Spark on Kubernetes: image JARs on driver/executor extraClassPath (not spark.jars): %s",
+                joined,
+            )
             builder = (
                 builder.config("spark.driver.extraClassPath", joined)
                 .config("spark.executor.extraClassPath", joined)

{batch_analytics-0.2.8 → batch_analytics-0.3.0}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.8
+Version: 0.3.0
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
@@ -15,14 +15,17 @@ Requires-Dist: scipy>=1.5.0; extra == "ttest"
 Provides-Extra: s3
 Requires-Dist: boto3>=1.28; extra == "s3"
 Provides-Extra: clickhouse
-Requires-Dist: clickhouse-connect>=0.7; extra == "clickhouse"
+Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "clickhouse"
+Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "clickhouse"
 Provides-Extra: output
 Requires-Dist: boto3>=1.28; extra == "output"
-Requires-Dist: clickhouse-connect>=0.7; extra == "output"
+Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "output"
+Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "output"
 Provides-Extra: full
 Requires-Dist: scipy>=1.5.0; extra == "full"
 Requires-Dist: boto3>=1.28; extra == "full"
-Requires-Dist: clickhouse-connect>=0.7; extra == "full"
+Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "full"
+Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "full"
 
 # Batch Analytics
 

batch_analytics-0.3.0/src/batch_analytics.egg-info/requires.txt

@@ -0,0 +1,38 @@
+pyspark<3.6,>=3.4
+numpy>=1.19.0
+
+[clickhouse]
+
+[clickhouse:python_version < "3.9"]
+clickhouse-connect<0.9,>=0.7
+
+[clickhouse:python_version >= "3.9"]
+clickhouse-connect>=0.7
+
+[dev]
+pytest>=7.0
+
+[full]
+scipy>=1.5.0
+boto3>=1.28
+
+[full:python_version < "3.9"]
+clickhouse-connect<0.9,>=0.7
+
+[full:python_version >= "3.9"]
+clickhouse-connect>=0.7
+
+[output]
+boto3>=1.28
+
+[output:python_version < "3.9"]
+clickhouse-connect<0.9,>=0.7
+
+[output:python_version >= "3.9"]
+clickhouse-connect>=0.7
+
+[s3]
+boto3>=1.28
+
+[ttest]
+scipy>=1.5.0

batch_analytics-0.2.8/src/batch_analytics.egg-info/requires.txt

@@ -1,23 +0,0 @@
-pyspark<3.6,>=3.4
-numpy>=1.19.0
-
-[clickhouse]
-clickhouse-connect>=0.7
-
-[dev]
-pytest>=7.0
-
-[full]
-scipy>=1.5.0
-boto3>=1.28
-clickhouse-connect>=0.7
-
-[output]
-boto3>=1.28
-clickhouse-connect>=0.7
-
-[s3]
-boto3>=1.28
-
-[ttest]
-scipy>=1.5.0