batch-analytics 0.2.6__tar.gz → 0.2.8__tar.gz

{batch_analytics-0.2.6 → batch_analytics-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.6
+Version: 0.2.8
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.2.6 → batch_analytics-0.2.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.2.6"
+version = "0.2.8"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.2.6 → batch_analytics-0.2.8}/src/batch_analytics/job_runner.py

@@ -67,6 +67,30 @@ def create_spark_session(
     if cfg.master.startswith("k8s://"):
         packages.append("org.apache.hadoop:hadoop-aws:3.3.4")
 
+    # Spark-on-K8s: absolute paths in spark.jars are re-sent to executors as ./basename.jar and
+    # fail there ("Unable to create executor due to ./clickhouse-spark-runtime-..."). JARs baked
+    # into the driver/executor image belong on the JVM classpath instead.
+    if cfg.master.startswith("k8s://") and jar_list:
+        local_cp: List[str] = []
+        remote_jars: List[str] = []
+        for p in jar_list:
+            p = p.strip()
+            if not p:
+                continue
+            if p.startswith("/") and p.endswith(".jar"):
+                local_cp.append(p)
+            else:
+                remote_jars.append(p)
+        if local_cp:
+            joined = ":".join(local_cp)
+            builder = (
+                builder.config("spark.driver.extraClassPath", joined)
+                .config("spark.executor.extraClassPath", joined)
+                .config("spark.kubernetes.driver.extraClassPath", joined)
+                .config("spark.kubernetes.executor.extraClassPath", joined)
+            )
+        jar_list = remote_jars
+
     if jar_list:
         builder = builder.config("spark.jars", ",".join(jar_list))
     if packages:
@@ -176,10 +200,12 @@ def run_pipeline(
         # Override: BATCH_SPARK_CLICKHOUSE_PACKAGES=maven coords / https jar URLs (comma-sep) or "" for SPARK_JARS only.
         _raw_ch = os.environ.get("BATCH_SPARK_CLICKHOUSE_PACKAGES")
         if _raw_ch is None:
+            # Shaded *-all.jar; pin matches Docker image / SPARK_JARS (0.9.x; no 0.10.x on Central for this artifact).
+            _ch_jdbc = os.environ.get("BATCH_CLICKHOUSE_JDBC_VERSION", "0.9.8").strip()
             ch_pkgs = (
                 "com.clickhouse.spark:clickhouse-spark-runtime-3.5_2.12:0.8.0,"
-                "https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc/0.6.2/"
-                "clickhouse-jdbc-0.6.2-all.jar"
+                f"https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc/{_ch_jdbc}/"
+                f"clickhouse-jdbc-{_ch_jdbc}-all.jar"
             )
         elif not _raw_ch.strip():
             ch_pkgs = None

{batch_analytics-0.2.6 → batch_analytics-0.2.8}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.6
+Version: 0.2.8
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT