batch-analytics 0.3.21__tar.gz → 0.3.23__tar.gz

{batch_analytics-0.3.21 → batch_analytics-0.3.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.21
+Version: 0.3.23
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
@@ -25,6 +25,7 @@ Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra ==
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "output"
 Provides-Extra: autogluon
 Requires-Dist: autogluon-tabular[lightgbm]<2.0,>=1.0; extra == "autogluon"
+Requires-Dist: typing-extensions>=4.8.0; extra == "autogluon"
 Requires-Dist: pandas>=1.3.0; extra == "autogluon"
 Requires-Dist: boto3>=1.28; extra == "autogluon"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "autogluon"
@@ -37,6 +38,7 @@ Requires-Dist: boto3>=1.28; extra == "full"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "full"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "full"
 Requires-Dist: autogluon-tabular[lightgbm]<2.0,>=1.0; extra == "full"
+Requires-Dist: typing-extensions>=4.8.0; extra == "full"
 Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "full"
 
 # Batch Analytics

{batch_analytics-0.3.21 → batch_analytics-0.3.23}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.21"
+version = "0.3.23"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"
@@ -34,6 +34,7 @@ output = [
 # autogluon-tabular[lightgbm]: tabular AutoML with LightGBM only (no PyTorch from full autogluon metapackage)
 autogluon = [
     "autogluon-tabular[lightgbm]>=1.0,<2.0",
+    "typing-extensions>=4.8.0",
     "pandas>=1.3.0",
     "boto3>=1.28",
     "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
@@ -47,6 +48,7 @@ full = [
     "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
     "clickhouse-connect>=0.7; python_version >= '3.9'",
     "autogluon-tabular[lightgbm]>=1.0,<2.0",
+    "typing-extensions>=4.8.0",
     "pyarrow>=10.0.0; python_version >= '3.8'",
 ]
 

{batch_analytics-0.3.21 → batch_analytics-0.3.23}/src/batch_analytics/analytics/gluon_autogluon_train.py

@@ -97,8 +97,13 @@ def main() -> None:
 
     try:
         from autogluon.tabular import TabularPredictor
-    except ImportError:
-        logger.exception("autogluon is not installed; use pip install 'batch-analytics[autogluon]'")
+    except ImportError as e:
+        if isinstance(e, ModuleNotFoundError) and e.name == "typing_extensions":
+            logger.exception(
+                "Install typing-extensions (required by AutoGluon): pip install 'typing-extensions>=4.8'"
+            )
+        else:
+            logger.exception("autogluon is not installed; use pip install 'batch-analytics[autogluon]'")
         sys.exit(4)
 
     problem_type = os.environ.get("AUTOGLUON_PROBLEM_TYPE", "binary").strip() or "binary"

{batch_analytics-0.3.21 → batch_analytics-0.3.23}/src/batch_analytics/transform.py

@@ -192,6 +192,69 @@ def _normalize_staging_write_mode(raw: str) -> str:
     return "overwrite"
 
 
+def _ch_quoted_ident(name: str) -> str:
+    """ClickHouse identifier in ORDER BY (escape backticks)."""
+    return "`" + name.replace("`", "``") + "`"
+
+
+def _ch_order_by_key_expr(col: str, df: DataFrame) -> str:
+    """
+    Build one ORDER BY key expression. Spark nullable columns map to Nullable in CH; CH 25
+    rejects nullable sort keys unless ``allow_nullable_key`` is applied — the Spark connector
+    often does not forward ``tableProperty("settings.allow_nullable_key")`` into DDL.
+
+    Using ``assumeNotNull(col)`` yields a non-nullable key expression so CREATE TABLE succeeds.
+    Only wrap when Spark marks the field nullable (``assumeNotNull`` requires Nullable input).
+    """
+    q = _ch_quoted_ident(col)
+    for f in df.schema.fields:
+        if f.name == col:
+            if f.nullable:
+                return f"assumeNotNull({q})"
+            return q
+    return q
+
+
+def _strip_outer_tuple_parens(s: str) -> str:
+    """clickhouse-spark-runtime wraps ``order_by`` in parentheses; do not also wrap here."""
+    s = s.strip()
+    if len(s) >= 2 and s[0] == "(" and s[-1] == ")":
+        return s[1:-1].strip()
+    return s
+
+
+def _merge_tree_order_by_for_staging(df: DataFrame, config: BatchAnalyticsConfig) -> str:
+    """
+    ClickHouse 25+ rejects MergeTree DDL without ORDER BY. The Spark ClickHouse catalog
+    passes this via DataFrameWriterV2.tableProperty("order_by", ...).
+
+    **Important:** The connector emits ``ORDER BY (<order_by>)``. Do **not** wrap the value
+    in an extra ``(...)`` or ClickHouse sees ``ORDER BY ((...))`` and raises a syntax error.
+
+    Resolution order:
+    1. BATCH_CLICKHOUSE_STAGING_ORDER_BY — comma-separated key expressions (no outer parens);
+       one layer of surrounding ``(...)`` is stripped if present.
+    2. BATCH_DEDUP_COLUMNS / transform.dedup_columns — comma-separated, ``assumeNotNull`` when nullable.
+    3. First column of the staging DataFrame schema (same nullable rule).
+    """
+    explicit = os.environ.get("BATCH_CLICKHOUSE_STAGING_ORDER_BY", "").strip()
+    if explicit:
+        return _strip_outer_tuple_parens(explicit)
+    dedup = (config.transform.dedup_columns or "").strip()
+    if dedup:
+        cols = [c.strip() for c in dedup.split(",") if c.strip()]
+        if cols:
+            parts = [_ch_order_by_key_expr(c, df) for c in cols]
+            return ", ".join(parts)
+    names = [f.name for f in df.schema.fields]
+    if names:
+        return _ch_order_by_key_expr(names[0], df)
+    raise ValueError(
+        "Cannot derive MergeTree ORDER BY for staging: set BATCH_CLICKHOUSE_STAGING_ORDER_BY, "
+        "or BATCH_DEDUP_COLUMNS / dedup_columns, or ensure the DataFrame has columns."
+    )
+
+
 def stage_to_clickhouse(
     spark: SparkSession,
     df: DataFrame,
@@ -200,20 +263,76 @@ def stage_to_clickhouse(
     """
     Write transformed data to ClickHouse staging table.
     Separate job from transform; must complete before analytics can run.
-    Uses native connector if available, else JDBC.
+
+    Preferred path: Spark SQL **catalog** API (``DataFrame.writeTo``), matching
+    ``job_runner.create_spark_session`` registration of ``ClickHouseCatalog``
+    (``BATCH_CLICKHOUSE_CATALOG``, default ``batch_ch``). The clickhouse-spark-runtime
+    0.8.x connector does **not** register the legacy short name ``format("clickhouse")``
+    / ``clickhouse.DefaultSource``.
+
+    Fallback: legacy ``format("clickhouse")`` (older stacks), then JDBC (may fail on
+    ClickHouse 25+ auto-DDL without ORDER BY).
+
+    **MergeTree ORDER BY** (required on ClickHouse 25+): set ``BATCH_CLICKHOUSE_STAGING_ORDER_BY``,
+    or rely on ``BATCH_DEDUP_COLUMNS`` / ``dedup_columns`` (comma-separated keys; no extra outer parens).
+
+    **Nullable sort keys**: (1) Dedup-derived ``ORDER BY`` uses ``assumeNotNull(`col`)`` when
+    Spark marks the field nullable. (2) The Spark connector often **does not** pass
+    ``tableProperty("settings.allow_nullable_key")`` into DDL, so we also set
+    ``engine`` to ``MergeTree() SETTINGS allow_nullable_key = 1``, which ClickHouse
+    applies to the created table.
+
     Write mode from BATCH_STAGING_WRITE_MODE (default overwrite = full replace).
     """
     n = df.count()
     mode = _normalize_staging_write_mode(config.transform.staging_write_mode)
+    ch = config.clickhouse
+    tbl = config.transform.staging_table
+    cat = os.environ.get("BATCH_CLICKHOUSE_CATALOG", "batch_ch").strip()
+
+    if cat:
+        try:
+            full_name = f"{cat}.{ch.database}.{tbl}"
+            order_by = _merge_tree_order_by_for_staging(df, config)
+            logger.info(
+                "Staging to ClickHouse via catalog %s (mode=%s, order_by=%s)",
+                full_name,
+                mode,
+                order_by,
+            )
+            # MergeTree SETTINGS in ENGINE: connector often ignores tableProperty("settings.*")
+            # for CREATE TABLE; CH 25 then reports allow_nullable_key disabled.
+            _mt_engine = "MergeTree() SETTINGS allow_nullable_key = 1"
+            w2 = (
+                df.writeTo(full_name)
+                .tableProperty("engine", _mt_engine)
+                .tableProperty("order_by", order_by)
+            )
+            if mode == "overwrite":
+                w2.createOrReplace()
+            else:
+                w2.append()
+            logger.info(
+                "Staged data to ClickHouse %s.%s (%d rows)",
+                ch.database,
+                tbl,
+                n,
+            )
+            return
+        except Exception as e:
+            logger.warning(
+                "ClickHouse catalog write failed (%s), trying legacy format/jdbc",
+                e,
+            )
+
     try:
-        ch = config.clickhouse
         writer = (
             df.write.format("clickhouse")
             .option("host", ch.host)
             .option("protocol", ch.protocol)
             .option("http_port", str(ch.port))
             .option("database", ch.database)
-            .option("table", config.transform.staging_table)
+            .option("table", tbl)
             .option("user", ch.user)
             .mode(mode)
         )
@@ -221,17 +340,17 @@ def stage_to_clickhouse(
             writer = writer.option("password", ch.password)
         writer.save()
     except Exception as e:
-        logger.warning("ClickHouse connector failed (%s), using JDBC", e)
+        logger.warning("ClickHouse legacy format failed (%s), using JDBC", e)
         df.write.jdbc(
             config.clickhouse.jdbc_url,
-            config.transform.staging_table,
+            tbl,
             mode=mode,
             properties=config.clickhouse.jdbc_properties,
         )
     logger.info(
         "Staged data to ClickHouse %s.%s (%d rows)",
-        config.clickhouse.database,
-        config.transform.staging_table,
+        ch.database,
+        tbl,
         n,
     )
 

{batch_analytics-0.3.21 → batch_analytics-0.3.23}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.21
+Version: 0.3.23
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
@@ -25,6 +25,7 @@ Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra ==
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "output"
 Provides-Extra: autogluon
 Requires-Dist: autogluon-tabular[lightgbm]<2.0,>=1.0; extra == "autogluon"
+Requires-Dist: typing-extensions>=4.8.0; extra == "autogluon"
 Requires-Dist: pandas>=1.3.0; extra == "autogluon"
 Requires-Dist: boto3>=1.28; extra == "autogluon"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "autogluon"
@@ -37,6 +38,7 @@ Requires-Dist: boto3>=1.28; extra == "full"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "full"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "full"
 Requires-Dist: autogluon-tabular[lightgbm]<2.0,>=1.0; extra == "full"
+Requires-Dist: typing-extensions>=4.8.0; extra == "full"
 Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "full"
 
 # Batch Analytics

{batch_analytics-0.3.21 → batch_analytics-0.3.23}/src/batch_analytics.egg-info/requires.txt

@@ -2,6 +2,7 @@ numpy>=1.19.0
 
 [autogluon]
 autogluon-tabular[lightgbm]<2.0,>=1.0
+typing-extensions>=4.8.0
 pandas>=1.3.0
 boto3>=1.28
 
@@ -30,6 +31,7 @@ pyspark<3.6,>=3.4
 scipy>=1.5.0
 boto3>=1.28
 autogluon-tabular[lightgbm]<2.0,>=1.0
+typing-extensions>=4.8.0
 
 [full:python_version < "3.9"]
 clickhouse-connect<0.9,>=0.7