batch-analytics 0.3.1__tar.gz → 0.3.3__tar.gz

{batch_analytics-0.3.1 → batch_analytics-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.1
+Version: 0.3.3
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.1 → batch_analytics-0.3.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.1"
+version = "0.3.3"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.3.1 → batch_analytics-0.3.3}/src/batch_analytics/job_runner.py

@@ -10,6 +10,7 @@ import sys
 import uuid
 from pathlib import Path
 from typing import Dict, List, Optional
+from urllib.parse import urlparse
 
 from pyspark.sql import SparkSession
 
@@ -27,6 +28,17 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
+def _local_jar_path_for_match(p: str) -> str:
+    """Normalize file:/path or /path for prefix checks."""
+    p = p.strip()
+    if not p:
+        return p
+    if p.startswith("file:"):
+        parsed = urlparse(p)
+        return parsed.path if parsed.path else p[5:]
+    return p
+
+
 def _omit_spark_distrib_jars(jar_list: List[str]) -> List[str]:
     """
     Drop paths under $SPARK_HOME/jars from spark.jars.
@@ -34,23 +46,27 @@ def _omit_spark_distrib_jars(jar_list: List[str]) -> List[str]:
     Those JARs are already on the driver and executor JVM classpath (Spark launch scripts add
     $SPARK_HOME/jars/*). Listing them again in spark.jars makes Spark distribute them to executors
     as ./basename.jar and breaks Kubernetes executors.
+
+    SPARK_HOME may be unset or empty in some pods; always treat /opt/spark/jars/ as Spark distro
+    (apache/spark images).
     """
-    spark_home = os.environ.get("SPARK_HOME", "/opt/spark").rstrip("/")
-    prefix = f"{spark_home}/jars/"
+    spark_home = (os.environ.get("SPARK_HOME") or "/opt/spark").rstrip("/")
+    prefixes = (f"{spark_home}/jars/", "/opt/spark/jars/")
     out: List[str] = []
     skipped: List[str] = []
     for p in jar_list:
         p = p.strip()
         if not p:
             continue
-        if p.startswith(prefix) and p.endswith(".jar"):
+        norm = _local_jar_path_for_match(p)
+        if norm.endswith(".jar") and any(norm.startswith(pref) for pref in prefixes):
             skipped.append(p)
         else:
             out.append(p)
     if skipped:
         logger.info(
             "Omitting spark.jars for JARs already on Spark classpath (%s): %s",
-            prefix,
+            prefixes[0],
             ",".join(skipped),
         )
     return out
@@ -97,6 +113,20 @@ def create_spark_session(
 
     if jar_list:
         builder = builder.config("spark.jars", ",".join(jar_list))
+    elif os.environ.get("BATCH_ALLOW_INHERITED_SPARK_JARS", "").strip().lower() not in (
+        "1",
+        "true",
+        "yes",
+    ):
+        # spark-submit / image ENV often set SPARK_JARS → --jars; Python never sees it in jar_list
+        # above, but Spark still adds them ("Added JAR /opt/spark/jars/...") and K8s executors fail.
+        # Empty spark.jars overrides inherited submit --jars; $SPARK_HOME/jars stay on JVM classpath.
+        builder = builder.config("spark.jars", "")
+        logger.info(
+            "spark.jars cleared (override spark-submit --jars / SPARK_JARS); "
+            "JARs under $SPARK_HOME/jars remain on the classpath. "
+            "Set BATCH_ALLOW_INHERITED_SPARK_JARS=1 to keep submit-inherited spark.jars."
+        )
     if packages:
         builder = builder.config("spark.jars.packages", ",".join(packages))
 
@@ -202,16 +232,12 @@ def run_pipeline(
         # Native format("clickhouse") needs clickhouse-spark-runtime; JDBC needs shaded clickhouse-jdbc (*-all),
         # not the thin Maven artifact: thin JAR lacks HttpClient 5 (ClassicHttpRequest).
         # Override: BATCH_SPARK_CLICKHOUSE_PACKAGES=maven coords / https jar URLs (comma-sep) or "" for SPARK_JARS only.
+        # Empty/unset: rely on $SPARK_HOME/jars (analytics-runner image). Do not add spark.jars /
+        # spark.jars.packages for ClickHouse here — that breaks K8s executors (./basename.jar).
+        # For ad-hoc runs without the image, set e.g.
+        # BATCH_SPARK_CLICKHOUSE_PACKAGES=com.clickhouse.spark:clickhouse-spark-runtime-3.5_2.12:0.8.0,https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc/0.9.8/clickhouse-jdbc-0.9.8-all.jar
         _raw_ch = os.environ.get("BATCH_SPARK_CLICKHOUSE_PACKAGES")
-        if _raw_ch is None:
-            # Shaded *-all.jar; pin matches Docker image / SPARK_JARS (0.9.x; no 0.10.x on Central for this artifact).
-            _ch_jdbc = os.environ.get("BATCH_CLICKHOUSE_JDBC_VERSION", "0.9.8").strip()
-            ch_pkgs = (
-                "com.clickhouse.spark:clickhouse-spark-runtime-3.5_2.12:0.8.0,"
-                f"https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc/{_ch_jdbc}/"
-                f"clickhouse-jdbc-{_ch_jdbc}-all.jar"
-            )
-        elif not _raw_ch.strip():
+        if _raw_ch is None or not _raw_ch.strip():
             ch_pkgs = None
         else:
             ch_pkgs = _raw_ch.strip()

{batch_analytics-0.3.1 → batch_analytics-0.3.3}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.1
+Version: 0.3.3
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT