batch-analytics 0.3.13__tar.gz → 0.3.14__tar.gz

{batch_analytics-0.3.13 → batch_analytics-0.3.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.13
+Version: 0.3.14
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.13 → batch_analytics-0.3.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.13"
+version = "0.3.14"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.3.13 → batch_analytics-0.3.14}/src/batch_analytics/config.py

@@ -29,16 +29,23 @@ class ClickHouseConfig:
 
     @property
     def jdbc_properties(self) -> dict:
+        """JDBC connection properties for Spark.
+
+        clickhouse-jdbc (clickhouse-java v0.6+) rejects legacy keys such as
+        ``compress_algorithm`` (ClientMisconfigurationException). Prefer JDBC URL
+        query parameters for compression behavior. To force the old property for
+        legacy stacks, set CLICKHOUSE_JDBC_LEGACY_COMPRESS_ALGORITHM (e.g. ``none``).
+        """
         props = {
             "user": self.user,
             "driver": "com.clickhouse.jdbc.ClickHouseDriver",
-            # Match Spark read codec default: avoids JDBC LZ4/gzip mismatches with server HTTP compression
-            "compress_algorithm": os.environ.get(
-                "CLICKHOUSE_JDBC_COMPRESS_ALGORITHM", "none"
-            ),
         }
         if self.password:
             props["password"] = self.password
+        # Opt-in legacy property for older shaded JDBC stacks only.
+        legacy = os.environ.get("CLICKHOUSE_JDBC_LEGACY_COMPRESS_ALGORITHM", "").strip()
+        if legacy:
+            props["compress_algorithm"] = legacy
         return props
 
 
@@ -78,6 +85,8 @@ class TransformConfig:
     staging_format: str = os.environ.get("BATCH_STAGING_FORMAT", "clickhouse")
     # Staging table name in ClickHouse (when format=clickhouse)
     staging_table: str = os.environ.get("BATCH_STAGING_TABLE", "analytics_staging")
+    # Spark save mode for ClickHouse staging (and path staging): overwrite | append
+    staging_write_mode: str = os.environ.get("BATCH_STAGING_WRITE_MODE", "overwrite")
     # Source column holding a JSON object or Python dict string; every top-level key becomes a new String column
     # (see transform.expand_kv_blob_column). Example: add_dimensions {'anchor_id':'...','lot':'A1'}
     add_dimension_column: str = os.environ.get("BATCH_ADD_DIMENSION_COLUMN", "add_dimension")

{batch_analytics-0.3.13 → batch_analytics-0.3.14}/src/batch_analytics/extract.py

@@ -4,7 +4,8 @@ Extract stage: Load data from ClickHouse using Spark ClickHouse connector or JDB
 
 import json
 import logging
-from typing import Dict, List, Optional
+import os
+from typing import Dict, List, Optional, Tuple
 
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.functions import col
@@ -59,6 +60,48 @@ def _apply_extract_filter(df: DataFrame, config: BatchAnalyticsConfig) -> DataFr
     return filtered
 
 
+def _parse_ch_database_table(table: str, default_database: str) -> Tuple[str, str]:
+    """
+    Resolve ``table`` reference to (database, table_name).
+
+    ``batch_metric_facts`` → (default_database, batch_metric_facts)
+    ``analytics.batch_metric_facts`` → (analytics, batch_metric_facts)
+    """
+    t = (table or "").strip()
+    if not t:
+        return default_database, t
+    if "." in t and not t.startswith("("):
+        db, tbl = t.split(".", 1)
+        db, tbl = db.strip(), tbl.strip()
+        if db and tbl:
+            return db, tbl
+    return default_database, t
+
+
+def _read_via_catalog(spark: SparkSession, cfg: BatchAnalyticsConfig, table: str) -> Optional[DataFrame]:
+    """
+    Read via Spark SQL catalog (ClickHouseCatalog), registered in job_runner.create_spark_session.
+
+    clickhouse-spark-runtime does **not** register legacy short name ``format(\"clickhouse\")`` /
+    ``clickhouse.DefaultSource``; catalog + ``spark.table(catalog.db.table)`` is the supported path.
+    """
+    cat = os.environ.get("BATCH_CLICKHOUSE_CATALOG", "batch_ch").strip()
+    if not cat:
+        return None
+    db, tbl = _parse_ch_database_table(table, cfg.clickhouse.database)
+    ident = f"{cat}.{db}.{tbl}"
+    try:
+        return spark.table(ident)
+    except Exception as e:
+        logger.warning(
+            "Catalog read failed for %s (%s): %s. Trying other readers.",
+            ident,
+            table,
+            e,
+        )
+        return None
+
+
 def _read_via_format(spark: SparkSession, cfg: BatchAnalyticsConfig, table: str) -> Optional[DataFrame]:
     """
     Read from ClickHouse using the native format API (clickhouse-spark-runtime).
@@ -106,7 +149,10 @@ def extract_table(
     Uses native connector if configured, otherwise JDBC.
     """
     if config.extract.use_native_connector:
-        df = _read_via_format(spark, config, table)
+        # Prefer catalog (matches clickhouse-spark-runtime); avoid legacy DefaultSource path.
+        df = _read_via_catalog(spark, config, table)
+        if df is None:
+            df = _read_via_format(spark, config, table)
         if df is None:
             df = _read_via_jdbc(spark, config, table)
     else:

{batch_analytics-0.3.13 → batch_analytics-0.3.14}/src/batch_analytics/transform.py

@@ -183,6 +183,15 @@ def transform(
     return remove_duplicates(transformed, key_columns=dedup_cols)
 
 
+def _normalize_staging_write_mode(raw: str) -> str:
+    """Spark DataFrameWriter mode: overwrite (replace table contents) or append."""
+    m = (raw or "overwrite").strip().lower()
+    if m in ("overwrite", "append"):
+        return m
+    logger.warning("Invalid BATCH_STAGING_WRITE_MODE=%r; using overwrite", raw)
+    return "overwrite"
+
+
 def stage_to_clickhouse(
     spark: SparkSession,
     df: DataFrame,
@@ -192,8 +201,10 @@ def stage_to_clickhouse(
     Write transformed data to ClickHouse staging table.
     Separate job from transform; must complete before analytics can run.
     Uses native connector if available, else JDBC.
+    Write mode from BATCH_STAGING_WRITE_MODE (default overwrite = full replace).
     """
     n = df.count()
+    mode = _normalize_staging_write_mode(config.transform.staging_write_mode)
     try:
         ch = config.clickhouse
         writer = (
@@ -204,7 +215,7 @@ def stage_to_clickhouse(
             .option("database", ch.database)
             .option("table", config.transform.staging_table)
             .option("user", ch.user)
-            .mode("overwrite")
+            .mode(mode)
         )
         if ch.password:
             writer = writer.option("password", ch.password)
@@ -214,7 +225,7 @@ def stage_to_clickhouse(
         df.write.jdbc(
             config.clickhouse.jdbc_url,
             config.transform.staging_table,
-            mode="overwrite",
+            mode=mode,
             properties=config.clickhouse.jdbc_properties,
         )
     logger.info(
@@ -233,14 +244,15 @@ def stage_to_path(
     """Write transformed data to parquet/delta (for local dev or intermediate storage)."""
     path = config.transform.staging_path
     fmt = config.transform.staging_format
+    mode = _normalize_staging_write_mode(config.transform.staging_write_mode)
     if fmt == "parquet":
-        df.write.mode("overwrite").parquet(path)
+        df.write.mode(mode).parquet(path)
         logger.info("Staged data to %s (parquet)", path)
     elif fmt == "delta":
-        df.write.format("delta").mode("overwrite").save(path)
+        df.write.format("delta").mode(mode).save(path)
         logger.info("Staged data to %s (delta)", path)
     else:
-        df.write.format(fmt).mode("overwrite").save(path)
+        df.write.format(fmt).mode(mode).save(path)
         logger.info("Staged data to %s (%s)", path, fmt)
 
 

{batch_analytics-0.3.13 → batch_analytics-0.3.14}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.13
+Version: 0.3.14
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT