batch-analytics 0.2.2__tar.gz → 0.2.4__tar.gz

{batch_analytics-0.2.2 → batch_analytics-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.2
+Version: 0.2.4
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.2.2 → batch_analytics-0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.2.2"
+version = "0.2.4"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.2.2 → batch_analytics-0.2.4}/src/batch_analytics/extract.py

@@ -18,7 +18,7 @@ def _read_via_format(spark: SparkSession, cfg: BatchAnalyticsConfig, table: str)
     Requires: com.clickhouse.spark:clickhouse-spark-runtime in spark.jars.packages
     """
     try:
-        df = (
+        rd = (
             spark.read.format("clickhouse")
             .option("host", cfg.clickhouse.host)
             .option("protocol", cfg.clickhouse.protocol)
@@ -26,8 +26,10 @@ def _read_via_format(spark: SparkSession, cfg: BatchAnalyticsConfig, table: str)
             .option("database", cfg.clickhouse.database)
             .option("table", table)
             .option("user", cfg.clickhouse.user)
-            .load()
         )
+        if cfg.clickhouse.password:
+            rd = rd.option("password", cfg.clickhouse.password)
+        df = rd.load()
         return df
     except Exception as e:
         logger.warning(

{batch_analytics-0.2.2 → batch_analytics-0.2.4}/src/batch_analytics/job_runner.py

@@ -44,12 +44,31 @@ def create_spark_session(
         .config("spark.sql.adaptive.coalescePartitions.enabled", "true")
     )
 
-    # JARs: ClickHouse JDBC + hadoop-aws (for S3 when on K8s)
-    packages = []
+    # JARs: spark.jars = local/https/file paths; spark.jars.packages = Maven coordinates
+    # SPARK_JARS can be comma-separated URLs (e.g. Maven Central https://repo1.maven.org/...jar)
+    jar_list: List[str] = []
+    sp_jars = os.environ.get("SPARK_JARS", "").strip()
+    if sp_jars:
+        jar_list.extend(p.strip() for p in sp_jars.split(",") if p.strip())
+
+    packages: List[str] = []
     if clickhouse_jars:
-        packages.append(clickhouse_jars)
+        for part in clickhouse_jars.split(","):
+            part = part.strip()
+            if not part:
+                continue
+            if part.startswith(("http://", "https://", "file:")):
+                jar_list.append(part)
+            elif part.endswith(".jar") and ":" not in part:
+                jar_list.append(part)
+            else:
+                packages.append(part)
+
     if cfg.master.startswith("k8s://"):
         packages.append("org.apache.hadoop:hadoop-aws:3.3.4")
+
+    if jar_list:
+        builder = builder.config("spark.jars", ",".join(jar_list))
     if packages:
         builder = builder.config("spark.jars.packages", ",".join(packages))
 
@@ -121,10 +140,23 @@ def run_pipeline(
     run_id = str(uuid.uuid4())[:8]
 
     if spark is None:
-        jars = "com.clickhouse:clickhouse-jdbc:0.4.6:all"
+        # Native format("clickhouse") needs clickhouse-spark-runtime; JDBC needs shaded clickhouse-jdbc (*-all),
+        # not the thin Maven artifact: thin JAR lacks HttpClient 5 (ClassicHttpRequest).
+        # Override: BATCH_SPARK_CLICKHOUSE_PACKAGES=maven coords / https jar URLs (comma-sep) or "" for SPARK_JARS only.
+        _raw_ch = os.environ.get("BATCH_SPARK_CLICKHOUSE_PACKAGES")
+        if _raw_ch is None:
+            ch_pkgs = (
+                "com.clickhouse.spark:clickhouse-spark-runtime-3.5_2.12:0.8.0,"
+                "https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc/0.6.2/"
+                "clickhouse-jdbc-0.6.2-all.jar"
+            )
+        elif not _raw_ch.strip():
+            ch_pkgs = None
+        else:
+            ch_pkgs = _raw_ch.strip()
         spark = create_spark_session(
             app_name="BatchAnalytics",
-            clickhouse_jars=jars,
+            clickhouse_jars=ch_pkgs,
             config=config,
         )
 

{batch_analytics-0.2.2 → batch_analytics-0.2.4}/src/batch_analytics/transform.py

@@ -92,16 +92,19 @@ def stage_to_clickhouse(
     """
     n = df.count()
     try:
+        ch = config.clickhouse
         writer = (
             df.write.format("clickhouse")
-            .option("host", config.clickhouse.host)
-            .option("database", config.clickhouse.database)
+            .option("host", ch.host)
+            .option("protocol", ch.protocol)
+            .option("http_port", str(ch.port))
+            .option("database", ch.database)
             .option("table", config.transform.staging_table)
-            .option("user", config.clickhouse.user)
+            .option("user", ch.user)
             .mode("overwrite")
         )
-        if config.clickhouse.password:
-            writer = writer.option("password", config.clickhouse.password)
+        if ch.password:
+            writer = writer.option("password", ch.password)
         writer.save()
     except Exception as e:
         logger.warning("ClickHouse connector failed (%s), using JDBC", e)
@@ -168,13 +171,19 @@ def load_staged(
         return spark.read.format("delta").load(staging_path)
     if fmt == "clickhouse":
         try:
-            return (
+            ch = config.clickhouse
+            rd = (
                 spark.read.format("clickhouse")
-                .option("host", config.clickhouse.host)
-                .option("database", config.clickhouse.database)
+                .option("host", ch.host)
+                .option("protocol", ch.protocol)
+                .option("http_port", str(ch.port))
+                .option("database", ch.database)
                 .option("table", config.transform.staging_table)
-                .load()
+                .option("user", ch.user)
             )
+            if ch.password:
+                rd = rd.option("password", ch.password)
+            return rd.load()
         except Exception:
             return spark.read.jdbc(
                 config.clickhouse.jdbc_url,

{batch_analytics-0.2.2 → batch_analytics-0.2.4}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.2
+Version: 0.2.4
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT