batch-analytics 0.2.3__tar.gz → 0.2.5__tar.gz

{batch_analytics-0.2.3 → batch_analytics-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.3
+Version: 0.2.5
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.2.3 → batch_analytics-0.2.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.2.3"
+version = "0.2.5"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.2.3 → batch_analytics-0.2.5}/src/batch_analytics/extract.py

@@ -3,6 +3,7 @@ Extract stage: Load data from ClickHouse using Spark ClickHouse connector or JDB
 """
 
 import logging
+import os
 from typing import Dict, List, Optional
 
 from pyspark.sql import DataFrame, SparkSession
@@ -59,7 +60,7 @@ def extract_table(
     Uses native connector if configured, otherwise JDBC.
     """
     if config.extract.use_native_connector:
-        df = _read_via_format(spark, config, table)
+        df = _read_via_catalog(spark, config, table)
         if df is None:
             df = _read_via_jdbc(spark, config, table)
     else:

{batch_analytics-0.2.3 → batch_analytics-0.2.5}/src/batch_analytics/job_runner.py

@@ -72,6 +72,27 @@ def create_spark_session(
     if packages:
         builder = builder.config("spark.jars.packages", ",".join(packages))
 
+    # clickhouse-spark-runtime does not register legacy clickhouse.DefaultSource; the connector
+    # expects a Spark catalog (see ClickHouse docs). Enables spark.table("catalog.db.table").
+    ch_cat = os.environ.get("BATCH_CLICKHOUSE_CATALOG", "batch_ch").strip()
+    if ch_cat:
+        ch = config.clickhouse
+        builder = (
+            builder.config(
+                f"spark.sql.catalog.{ch_cat}",
+                "com.clickhouse.spark.ClickHouseCatalog",
+            )
+            .config(f"spark.sql.catalog.{ch_cat}.host", ch.host)
+            .config(f"spark.sql.catalog.{ch_cat}.protocol", ch.protocol)
+            .config(f"spark.sql.catalog.{ch_cat}.http_port", str(ch.port))
+            .config(f"spark.sql.catalog.{ch_cat}.user", ch.user)
+            .config(f"spark.sql.catalog.{ch_cat}.database", ch.database)
+        )
+        if ch.password:
+            builder = builder.config(f"spark.sql.catalog.{ch_cat}.password", ch.password)
+        if ch.protocol.lower() == "https":
+            builder = builder.config(f"spark.sql.catalog.{ch_cat}.option.ssl", "true")
+
     if cfg.master.startswith("k8s://"):
         driver_host = socket.gethostbyname(socket.gethostname())
         builder = (
@@ -140,13 +161,15 @@ def run_pipeline(
     run_id = str(uuid.uuid4())[:8]
 
     if spark is None:
-        # Native format("clickhouse") needs clickhouse-spark-runtime; JDBC fallback needs clickhouse-jdbc.
-        # Override: BATCH_SPARK_CLICKHOUSE_PACKAGES=maven coords (comma-sep) or "" to use only SPARK_JARS.
+        # Native format("clickhouse") needs clickhouse-spark-runtime; JDBC needs shaded clickhouse-jdbc (*-all),
+        # not the thin Maven artifact: thin JAR lacks HttpClient 5 (ClassicHttpRequest).
+        # Override: BATCH_SPARK_CLICKHOUSE_PACKAGES=maven coords / https jar URLs (comma-sep) or "" for SPARK_JARS only.
         _raw_ch = os.environ.get("BATCH_SPARK_CLICKHOUSE_PACKAGES")
         if _raw_ch is None:
             ch_pkgs = (
                 "com.clickhouse.spark:clickhouse-spark-runtime-3.5_2.12:0.8.0,"
-                "com.clickhouse:clickhouse-jdbc:0.6.2"
+                "https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc/0.6.2/"
+                "clickhouse-jdbc-0.6.2-all.jar"
             )
         elif not _raw_ch.strip():
             ch_pkgs = None

{batch_analytics-0.2.3 → batch_analytics-0.2.5}/src/batch_analytics/transform.py

@@ -3,6 +3,7 @@ Transform stage: Clean data (remove duplicates), extract add_dimension, and stag
 """
 
 import logging
+import os
 from typing import Optional, Sequence
 
 from pyspark.sql import DataFrame, SparkSession
@@ -170,24 +171,24 @@ def load_staged(
     if fmt == "delta":
         return spark.read.format("delta").load(staging_path)
     if fmt == "clickhouse":
-        try:
-            ch = config.clickhouse
-            rd = (
-                spark.read.format("clickhouse")
-                .option("host", ch.host)
-                .option("protocol", ch.protocol)
-                .option("http_port", str(ch.port))
-                .option("database", ch.database)
-                .option("table", config.transform.staging_table)
-                .option("user", ch.user)
-            )
-            if ch.password:
-                rd = rd.option("password", ch.password)
-            return rd.load()
-        except Exception:
-            return spark.read.jdbc(
-                config.clickhouse.jdbc_url,
-                config.transform.staging_table,
-                properties=config.clickhouse.jdbc_properties,
-            )
+        ch = config.clickhouse
+        tbl = config.transform.staging_table
+        cat = os.environ.get("BATCH_CLICKHOUSE_CATALOG", "batch_ch").strip()
+        if cat:
+            try:
+                return spark.table(f"{cat}.{ch.database}.{tbl}")
+            except Exception as e:
+                logger.warning(
+                    "load_staged: catalog table %s.%s.%s failed (%s), using JDBC",
+                    cat,
+                    ch.database,
+                    tbl,
+                    e,
+                )
+        dbtable = f"(SELECT * FROM `{ch.database}`.`{tbl}`) AS _stg"
+        return spark.read.jdbc(
+            ch.jdbc_url,
+            dbtable,
+            properties=ch.jdbc_properties,
+        )
     return spark.read.format(fmt).load(staging_path)

{batch_analytics-0.2.3 → batch_analytics-0.2.5}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.3
+Version: 0.2.5
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT