batch-analytics 0.2.9__tar.gz → 0.3.1__tar.gz

{batch_analytics-0.2.9 → batch_analytics-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.9
+Version: 0.3.1
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.2.9 → batch_analytics-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.2.9"
+version = "0.3.1"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.2.9 → batch_analytics-0.3.1}/src/batch_analytics/job_runner.py

@@ -27,6 +27,35 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
+def _omit_spark_distrib_jars(jar_list: List[str]) -> List[str]:
+    """
+    Drop paths under $SPARK_HOME/jars from spark.jars.
+
+    Those JARs are already on the driver and executor JVM classpath (Spark launch scripts add
+    $SPARK_HOME/jars/*). Listing them again in spark.jars makes Spark distribute them to executors
+    as ./basename.jar and breaks Kubernetes executors.
+    """
+    spark_home = os.environ.get("SPARK_HOME", "/opt/spark").rstrip("/")
+    prefix = f"{spark_home}/jars/"
+    out: List[str] = []
+    skipped: List[str] = []
+    for p in jar_list:
+        p = p.strip()
+        if not p:
+            continue
+        if p.startswith(prefix) and p.endswith(".jar"):
+            skipped.append(p)
+        else:
+            out.append(p)
+    if skipped:
+        logger.info(
+            "Omitting spark.jars for JARs already on Spark classpath (%s): %s",
+            prefix,
+            ",".join(skipped),
+        )
+    return out
+
+
 def create_spark_session(
     app_name: str = "BatchAnalytics",
     clickhouse_jars: Optional[str] = None,
@@ -64,32 +93,7 @@ def create_spark_session(
             else:
                 packages.append(part)
 
-    if cfg.master.startswith("k8s://"):
-        packages.append("org.apache.hadoop:hadoop-aws:3.3.4")
-
-    # Spark-on-K8s: absolute paths in spark.jars are re-sent to executors as ./basename.jar and
-    # fail there ("Unable to create executor due to ./clickhouse-spark-runtime-..."). JARs baked
-    # into the driver/executor image belong on the JVM classpath instead.
-    if cfg.master.startswith("k8s://") and jar_list:
-        local_cp: List[str] = []
-        remote_jars: List[str] = []
-        for p in jar_list:
-            p = p.strip()
-            if not p:
-                continue
-            if p.startswith("/") and p.endswith(".jar"):
-                local_cp.append(p)
-            else:
-                remote_jars.append(p)
-        if local_cp:
-            joined = ":".join(local_cp)
-            builder = (
-                builder.config("spark.driver.extraClassPath", joined)
-                .config("spark.executor.extraClassPath", joined)
-                .config("spark.kubernetes.driver.extraClassPath", joined)
-                .config("spark.kubernetes.executor.extraClassPath", joined)
-            )
-        jar_list = remote_jars
+    jar_list = _omit_spark_distrib_jars(jar_list)
 
     if jar_list:
         builder = builder.config("spark.jars", ",".join(jar_list))

{batch_analytics-0.2.9 → batch_analytics-0.3.1}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.9
+Version: 0.3.1
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT