batch-analytics 0.2.4__tar.gz → 0.2.6__tar.gz

{batch_analytics-0.2.4 → batch_analytics-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.4
+Version: 0.2.6
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.2.4 → batch_analytics-0.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.2.4"
+version = "0.2.6"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.2.4 → batch_analytics-0.2.6}/src/batch_analytics/config.py

@@ -29,7 +29,14 @@ class ClickHouseConfig:
 
     @property
     def jdbc_properties(self) -> dict:
-        props = {"user": self.user, "driver": "com.clickhouse.jdbc.ClickHouseDriver"}
+        props = {
+            "user": self.user,
+            "driver": "com.clickhouse.jdbc.ClickHouseDriver",
+            # Match Spark read codec default: avoids JDBC LZ4/gzip mismatches with server HTTP compression
+            "compress_algorithm": os.environ.get(
+                "CLICKHOUSE_JDBC_COMPRESS_ALGORITHM", "none"
+            ),
+        }
         if self.password:
             props["password"] = self.password
         return props

{batch_analytics-0.2.4 → batch_analytics-0.2.6}/src/batch_analytics/extract.py

@@ -3,6 +3,7 @@ Extract stage: Load data from ClickHouse using Spark ClickHouse connector or JDB
 """
 
 import logging
+import os
 from typing import Dict, List, Optional
 
 from pyspark.sql import DataFrame, SparkSession
@@ -59,7 +60,7 @@ def extract_table(
     Uses native connector if configured, otherwise JDBC.
     """
     if config.extract.use_native_connector:
-        df = _read_via_format(spark, config, table)
+        df = _read_via_catalog(spark, config, table)
         if df is None:
             df = _read_via_jdbc(spark, config, table)
     else:

{batch_analytics-0.2.4 → batch_analytics-0.2.6}/src/batch_analytics/job_runner.py

@@ -72,6 +72,37 @@ def create_spark_session(
     if packages:
         builder = builder.config("spark.jars.packages", ",".join(packages))
 
+    # clickhouse-spark-runtime does not register legacy clickhouse.DefaultSource; the connector
+    # expects a Spark catalog (see ClickHouse docs). Enables spark.table("catalog.db.table").
+    ch_cat = os.environ.get("BATCH_CLICKHOUSE_CATALOG", "batch_ch").strip()
+    if ch_cat:
+        ch = config.clickhouse
+        builder = (
+            builder.config(
+                f"spark.sql.catalog.{ch_cat}",
+                "com.clickhouse.spark.ClickHouseCatalog",
+            )
+            .config(f"spark.sql.catalog.{ch_cat}.host", ch.host)
+            .config(f"spark.sql.catalog.{ch_cat}.protocol", ch.protocol)
+            .config(f"spark.sql.catalog.{ch_cat}.http_port", str(ch.port))
+            .config(f"spark.sql.catalog.{ch_cat}.user", ch.user)
+            .config(f"spark.sql.catalog.{ch_cat}.database", ch.database)
+        )
+        if ch.password:
+            builder = builder.config(f"spark.sql.catalog.{ch_cat}.password", ch.password)
+        if ch.protocol.lower() == "https":
+            builder = builder.config(f"spark.sql.catalog.{ch_cat}.option.ssl", "true")
+        # Avoid Lz4InputStream "Magic is not correct" when server HTTP compression != client expectation
+        # (see clickhouse-java#1449 / server enable_http_compression user defaults).
+        read_codec = os.environ.get(
+            "SPARK_CLICKHOUSE_READ_COMPRESSION_CODEC", "none"
+        ).strip()
+        if read_codec:
+            builder = builder.config(
+                "spark.clickhouse.read.compression.codec",
+                read_codec,
+            )
+
     if cfg.master.startswith("k8s://"):
         driver_host = socket.gethostbyname(socket.gethostname())
         builder = (

{batch_analytics-0.2.4 → batch_analytics-0.2.6}/src/batch_analytics/transform.py

@@ -3,6 +3,7 @@ Transform stage: Clean data (remove duplicates), extract add_dimension, and stag
 """
 
 import logging
+import os
 from typing import Optional, Sequence
 
 from pyspark.sql import DataFrame, SparkSession
@@ -170,24 +171,24 @@ def load_staged(
     if fmt == "delta":
         return spark.read.format("delta").load(staging_path)
     if fmt == "clickhouse":
-        try:
-            ch = config.clickhouse
-            rd = (
-                spark.read.format("clickhouse")
-                .option("host", ch.host)
-                .option("protocol", ch.protocol)
-                .option("http_port", str(ch.port))
-                .option("database", ch.database)
-                .option("table", config.transform.staging_table)
-                .option("user", ch.user)
-            )
-            if ch.password:
-                rd = rd.option("password", ch.password)
-            return rd.load()
-        except Exception:
-            return spark.read.jdbc(
-                config.clickhouse.jdbc_url,
-                config.transform.staging_table,
-                properties=config.clickhouse.jdbc_properties,
-            )
+        ch = config.clickhouse
+        tbl = config.transform.staging_table
+        cat = os.environ.get("BATCH_CLICKHOUSE_CATALOG", "batch_ch").strip()
+        if cat:
+            try:
+                return spark.table(f"{cat}.{ch.database}.{tbl}")
+            except Exception as e:
+                logger.warning(
+                    "load_staged: catalog table %s.%s.%s failed (%s), using JDBC",
+                    cat,
+                    ch.database,
+                    tbl,
+                    e,
+                )
+        dbtable = f"(SELECT * FROM `{ch.database}`.`{tbl}`) AS _stg"
+        return spark.read.jdbc(
+            ch.jdbc_url,
+            dbtable,
+            properties=ch.jdbc_properties,
+        )
     return spark.read.format(fmt).load(staging_path)

{batch_analytics-0.2.4 → batch_analytics-0.2.6}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.2.4
+Version: 0.2.6
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT