batch-analytics 0.3.7__tar.gz → 0.3.14__tar.gz

{batch_analytics-0.3.7 → batch_analytics-0.3.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.7
+Version: 0.3.14
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.7 → batch_analytics-0.3.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.7"
+version = "0.3.14"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.3.7 → batch_analytics-0.3.14}/src/batch_analytics/__init__.py

@@ -13,8 +13,14 @@ Analytics modules:
 """
 
 from .config import BatchAnalyticsConfig, SparkK8sConfig
-from .extract import extract_all, extract_table, extract_unified
+from .extract import (
+    extract_all,
+    extract_table,
+    extract_unified,
+    parse_extract_filter_values,
+)
 from .transform import (
+    expand_kv_blob_column,
     extract_anchor_id,
     load_staged,
     remove_duplicates,
@@ -28,10 +34,12 @@ from .job_runner import run_pipeline, create_spark_session
 __all__ = [
     "BatchAnalyticsConfig",
     "SparkK8sConfig",
+    "expand_kv_blob_column",
     "extract_anchor_id",
     "extract_all",
     "extract_table",
     "extract_unified",
+    "parse_extract_filter_values",
     "remove_duplicates",
     "stage_to_clickhouse",
     "transform",

{batch_analytics-0.3.7 → batch_analytics-0.3.14}/src/batch_analytics/config.py

@@ -29,16 +29,23 @@ class ClickHouseConfig:
 
     @property
     def jdbc_properties(self) -> dict:
+        """JDBC connection properties for Spark.
+
+        clickhouse-jdbc (clickhouse-java v0.6+) rejects legacy keys such as
+        ``compress_algorithm`` (ClientMisconfigurationException). Prefer JDBC URL
+        query parameters for compression behavior. To force the old property for
+        legacy stacks, set CLICKHOUSE_JDBC_LEGACY_COMPRESS_ALGORITHM (e.g. ``none``).
+        """
         props = {
             "user": self.user,
             "driver": "com.clickhouse.jdbc.ClickHouseDriver",
-            # Match Spark read codec default: avoids JDBC LZ4/gzip mismatches with server HTTP compression
-            "compress_algorithm": os.environ.get(
-                "CLICKHOUSE_JDBC_COMPRESS_ALGORITHM", "none"
-            ),
         }
         if self.password:
             props["password"] = self.password
+        # Opt-in legacy property for older shaded JDBC stacks only.
+        legacy = os.environ.get("CLICKHOUSE_JDBC_LEGACY_COMPRESS_ALGORITHM", "").strip()
+        if legacy:
+            props["compress_algorithm"] = legacy
         return props
 
 
@@ -55,13 +62,18 @@ class ExtractConfig:
     use_native_connector: bool = os.environ.get(
         "BATCH_USE_NATIVE_CONNECTOR", "false"
     ).lower() == "true"
+    # Optional WHERE col IN (...) after read. Empty filter_column = no filter (full table).
+    # filter_values: comma-separated list, or JSON array e.g. ["a","b"] for values containing commas.
+    filter_column: str = os.environ.get("BATCH_EXTRACT_FILTER_COLUMN", "").strip()
+    filter_values: str = os.environ.get("BATCH_EXTRACT_FILTER_VALUES", "").strip()
 
 
 @dataclass
 class TransformConfig:
     """Transform stage configuration."""
 
-    # Columns to use for deduplication (comma-separated); empty = use all columns
+    # Order: extract anchor_id from add_dimension(s) column, then dedupe by these keys.
+    # Deduplication keys (comma-separated). Empty = dropDuplicates() on full row (all columns).
     dedup_columns: str = os.environ.get("BATCH_DEDUP_COLUMNS", "")
     # Staging output path (local or S3)
     staging_path: str = os.environ.get(
@@ -73,8 +85,12 @@ class TransformConfig:
     staging_format: str = os.environ.get("BATCH_STAGING_FORMAT", "clickhouse")
     # Staging table name in ClickHouse (when format=clickhouse)
     staging_table: str = os.environ.get("BATCH_STAGING_TABLE", "analytics_staging")
-    # Extract anchor_id from add_dimension column (e.g. {'anchor_id':'GP/GPH(D)/II(W)/250019'})
+    # Spark save mode for ClickHouse staging (and path staging): overwrite | append
+    staging_write_mode: str = os.environ.get("BATCH_STAGING_WRITE_MODE", "overwrite")
+    # Source column holding a JSON object or Python dict string; every top-level key becomes a new String column
+    # (see transform.expand_kv_blob_column). Example: add_dimensions {'anchor_id':'...','lot':'A1'}
     add_dimension_column: str = os.environ.get("BATCH_ADD_DIMENSION_COLUMN", "add_dimension")
+    # Legacy: no longer used; output column names match JSON keys (e.g. anchor_id). Kept for env compatibility.
     anchor_id_column: str = os.environ.get("BATCH_ANCHOR_ID_COLUMN", "anchor_id")
 
 

{batch_analytics-0.3.7 → batch_analytics-0.3.14}/src/batch_analytics/extract.py

@@ -2,17 +2,106 @@
 Extract stage: Load data from ClickHouse using Spark ClickHouse connector or JDBC.
 """
 
+import json
 import logging
 import os
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Tuple
 
 from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.functions import col
 
 from .config import BatchAnalyticsConfig
 
 logger = logging.getLogger(__name__)
 
 
+def parse_extract_filter_values(raw: str) -> List[str]:
+    """
+    Parse BATCH_EXTRACT_FILTER_VALUES: comma-separated tokens, or JSON array string.
+
+    Examples:
+      a,b,c -> ["a","b","c"]
+      ["GP/A","GP/B"] -> JSON list (values may contain commas)
+    """
+    text = (raw or "").strip()
+    if not text:
+        return []
+    if text.startswith("["):
+        try:
+            data = json.loads(text)
+            if isinstance(data, list):
+                out = [str(x).strip() for x in data if str(x).strip()]
+                return out
+        except json.JSONDecodeError:
+            logger.warning("BATCH_EXTRACT_FILTER_VALUES looks like JSON but failed to parse; using comma split")
+    return [p.strip() for p in text.split(",") if p.strip()]
+
+
+def _apply_extract_filter(df: DataFrame, config: BatchAnalyticsConfig) -> DataFrame:
+    """Apply col IN (values) when filter_column is set; empty column = no filter."""
+    col_name = (config.extract.filter_column or "").strip()
+    if not col_name:
+        return df
+    if col_name not in df.columns:
+        logger.warning(
+            "BATCH_EXTRACT_FILTER_COLUMN=%r not in extracted columns %s; skipping filter",
+            col_name,
+            df.columns,
+        )
+        return df
+    values = parse_extract_filter_values(config.extract.filter_values)
+    if not values:
+        logger.warning(
+            "BATCH_EXTRACT_FILTER_COLUMN=%r set but BATCH_EXTRACT_FILTER_VALUES is empty; skipping IN filter",
+            col_name,
+        )
+        return df
+    filtered = df.filter(col(col_name).isin(values))
+    return filtered
+
+
+def _parse_ch_database_table(table: str, default_database: str) -> Tuple[str, str]:
+    """
+    Resolve ``table`` reference to (database, table_name).
+
+    ``batch_metric_facts`` → (default_database, batch_metric_facts)
+    ``analytics.batch_metric_facts`` → (analytics, batch_metric_facts)
+    """
+    t = (table or "").strip()
+    if not t:
+        return default_database, t
+    if "." in t and not t.startswith("("):
+        db, tbl = t.split(".", 1)
+        db, tbl = db.strip(), tbl.strip()
+        if db and tbl:
+            return db, tbl
+    return default_database, t
+
+
+def _read_via_catalog(spark: SparkSession, cfg: BatchAnalyticsConfig, table: str) -> Optional[DataFrame]:
+    """
+    Read via Spark SQL catalog (ClickHouseCatalog), registered in job_runner.create_spark_session.
+
+    clickhouse-spark-runtime does **not** register legacy short name ``format(\"clickhouse\")`` /
+    ``clickhouse.DefaultSource``; catalog + ``spark.table(catalog.db.table)`` is the supported path.
+    """
+    cat = os.environ.get("BATCH_CLICKHOUSE_CATALOG", "batch_ch").strip()
+    if not cat:
+        return None
+    db, tbl = _parse_ch_database_table(table, cfg.clickhouse.database)
+    ident = f"{cat}.{db}.{tbl}"
+    try:
+        return spark.table(ident)
+    except Exception as e:
+        logger.warning(
+            "Catalog read failed for %s (%s): %s. Trying other readers.",
+            ident,
+            table,
+            e,
+        )
+        return None
+
+
 def _read_via_format(spark: SparkSession, cfg: BatchAnalyticsConfig, table: str) -> Optional[DataFrame]:
     """
     Read from ClickHouse using the native format API (clickhouse-spark-runtime).
@@ -60,12 +149,16 @@ def extract_table(
     Uses native connector if configured, otherwise JDBC.
     """
     if config.extract.use_native_connector:
+        # Prefer catalog (matches clickhouse-spark-runtime); avoid legacy DefaultSource path.
         df = _read_via_catalog(spark, config, table)
+        if df is None:
+            df = _read_via_format(spark, config, table)
         if df is None:
             df = _read_via_jdbc(spark, config, table)
     else:
         df = _read_via_jdbc(spark, config, table)
 
+    df = _apply_extract_filter(df, config)
     logger.info("Extracted table %s: %d rows", table, df.count())
     return df
 

{batch_analytics-0.3.7 → batch_analytics-0.3.14}/src/batch_analytics/job_runner.py

@@ -270,9 +270,9 @@ def run_pipeline(
     else:
         df_transformed = df_raw  # df_raw already loaded from staged when not run_extract
         if run_extract and run_analytics:
-            from .transform import extract_anchor_id, remove_duplicates
+            from .transform import expand_kv_blob_column, remove_duplicates
 
-            df_transformed = extract_anchor_id(df_raw, config)
+            df_transformed = expand_kv_blob_column(df_raw, config)
             dedup_cols = (
                 [c.strip() for c in config.transform.dedup_columns.split(",") if c.strip()]
                 if config.transform.dedup_columns

{batch_analytics-0.3.7 → batch_analytics-0.3.14}/src/batch_analytics/transform.py

@@ -1,42 +1,143 @@
 """
-Transform stage: Clean data (remove duplicates), extract add_dimension, and stage.
+Transform stage: Clean data (remove duplicates), expand JSON/KV blob column, and stage.
 """
 
+import ast
+import json
 import logging
 import os
-from typing import Optional, Sequence
+import re
+from typing import Any, Dict, List, Optional, Sequence, Set
 
 from pyspark.sql import DataFrame, SparkSession
-from pyspark.sql.functions import coalesce, col, get_json_object, regexp_extract
+from pyspark.sql.functions import col, explode, map_keys, udf
+from pyspark.sql.types import MapType, StringType
 
 from .config import BatchAnalyticsConfig
 
 logger = logging.getLogger(__name__)
 
 
-def extract_anchor_id(
+def _stringify_leaf(v: Any) -> str:
+    if v is None:
+        return ""
+    if isinstance(v, (dict, list)):
+        return json.dumps(v, separators=(",", ":"))
+    return str(v)
+
+
+def parse_blob_to_strmap(s: Any) -> Dict[str, str]:
+    """
+    Parse a cell value into a flat string map (top-level keys only).
+
+    Accepts standard JSON objects or Python repr dicts (e.g. single-quoted).
+    Non-dict / unparsable input yields an empty map.
+    """
+    if s is None:
+        return {}
+    text = str(s).strip()
+    if not text:
+        return {}
+    obj: Any = None
+    try:
+        obj = json.loads(text)
+    except json.JSONDecodeError:
+        pass
+    if obj is None:
+        try:
+            obj = ast.literal_eval(text)
+        except (ValueError, SyntaxError, MemoryError):
+            return {}
+    if not isinstance(obj, dict):
+        return {}
+    out: Dict[str, str] = {}
+    for k, v in obj.items():
+        key = str(k).strip()
+        if not key:
+            continue
+        out[key] = _stringify_leaf(v)
+    return out
+
+
+def _spark_safe_base_name(key: str) -> str:
+    """Sanitize JSON key to a usable Spark column name."""
+    s = re.sub(r"[^0-9a-zA-Z_]", "_", key.strip())
+    s = re.sub(r"_+", "_", s).strip("_")
+    if not s:
+        return "kv_key"
+    if s[0].isdigit():
+        s = "c_" + s
+    return s
+
+
+def _unique_column_name(base: str, used: Set[str]) -> str:
+    name = base
+    n = 1
+    while name in used:
+        n += 1
+        name = f"{base}_{n}"
+    used.add(name)
+    return name
+
+
+def expand_kv_blob_column(
     df: DataFrame,
     config: BatchAnalyticsConfig,
 ) -> DataFrame:
     """
-    Extract anchor_id from add_dimension column.
-    Supports JSON format {"anchor_id":"value"} or Python-dict {"anchor_id":"value"}.
-    Creates a new column (anchor_id by default) with the extracted value.
+    Parse the configured blob column into top-level key/value pairs and add one String column per key.
+
+    No per-key user configuration: every distinct key observed in the column (across the dataset)
+    becomes a column; values are strings (nested dict/list serialized as JSON). Empty / null cells
+    yield nulls in those columns.
+
+    Source column: ``config.transform.add_dimension_column`` (env ``BATCH_ADD_DIMENSION_COLUMN``).
     """
     col_name = config.transform.add_dimension_column
-    out_col = config.transform.anchor_id_column
-
     if col_name not in df.columns:
-        logger.debug("Column %s not found, skipping anchor_id extraction", col_name)
+        logger.debug("KV blob column %r not found, skipping expansion", col_name)
         return df
 
-    # Valid JSON: {"anchor_id":"GP/GPH(D)/II(W)/250019"}
-    json_extract = get_json_object(col(col_name), "$.anchor_id")
-    # Python-dict style: {'anchor_id':'GP/GPH(D)/II(W)/250019'}
-    regex_extract = regexp_extract(col(col_name), r"'anchor_id'\s*:\s*'([^']*)'", 1)
+    parse_udf = udf(parse_blob_to_strmap, MapType(StringType(), StringType()))
+    with_map = df.withColumn("_kv_blob_map", parse_udf(col(col_name)))
+
+    key_rows = (
+        with_map.select(explode(map_keys(col("_kv_blob_map"))).alias("_k"))
+        .where(col("_k").isNotNull())
+        .distinct()
+        .collect()
+    )
+    all_keys: List[str] = sorted({str(r._k).strip() for r in key_rows if r._k and str(r._k).strip()})
+
+    if not all_keys:
+        logger.info("No keys found in KV blob column %r; dropping temporary map only", col_name)
+        return with_map.drop("_kv_blob_map")
 
-    extracted = coalesce(json_extract, regex_extract)
-    return df.withColumn(out_col, extracted)
+    used: Set[str] = set(with_map.columns)
+    out = with_map
+    added: List[str] = []
+    for k in all_keys:
+        base = _spark_safe_base_name(k)
+        target = _unique_column_name(base, used)
+        added.append(target)
+        out = out.withColumn(target, col("_kv_blob_map").getItem(k))
+
+    out = out.drop("_kv_blob_map")
+    logger.info(
+        "Expanded KV blob column %r into %d columns: %s",
+        col_name,
+        len(added),
+        ", ".join(added),
+    )
+    return out
+
+
+def extract_anchor_id(
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    """Backward-compatible name: expands all keys from the blob column (not only ``anchor_id``)."""
+    return expand_kv_blob_column(df, config)
 
 
 def remove_duplicates(
@@ -69,10 +170,11 @@ def transform(
     config: BatchAnalyticsConfig,
 ) -> DataFrame:
     """
-    Apply transformation only: extract anchor_id, remove duplicates.
+    Apply transformation only: (1) expand JSON/KV blob column into one column per top-level key,
+    (2) deduplicate by BATCH_DEDUP_COLUMNS if set, else by full row.
     Does not write anywhere. Use stage_to_clickhouse() separately to persist.
     """
-    transformed = extract_anchor_id(df, config)
+    transformed = expand_kv_blob_column(df, config)
     dedup_cols = (
         [c.strip() for c in config.transform.dedup_columns.split(",") if c.strip()]
         if config.transform.dedup_columns
@@ -81,6 +183,15 @@ def transform(
     return remove_duplicates(transformed, key_columns=dedup_cols)
 
 
+def _normalize_staging_write_mode(raw: str) -> str:
+    """Spark DataFrameWriter mode: overwrite (replace table contents) or append."""
+    m = (raw or "overwrite").strip().lower()
+    if m in ("overwrite", "append"):
+        return m
+    logger.warning("Invalid BATCH_STAGING_WRITE_MODE=%r; using overwrite", raw)
+    return "overwrite"
+
+
 def stage_to_clickhouse(
     spark: SparkSession,
     df: DataFrame,
@@ -90,8 +201,10 @@ def stage_to_clickhouse(
     Write transformed data to ClickHouse staging table.
     Separate job from transform; must complete before analytics can run.
     Uses native connector if available, else JDBC.
+    Write mode from BATCH_STAGING_WRITE_MODE (default overwrite = full replace).
     """
     n = df.count()
+    mode = _normalize_staging_write_mode(config.transform.staging_write_mode)
     try:
         ch = config.clickhouse
         writer = (
@@ -102,7 +215,7 @@ def stage_to_clickhouse(
             .option("database", ch.database)
             .option("table", config.transform.staging_table)
             .option("user", ch.user)
-            .mode("overwrite")
+            .mode(mode)
         )
         if ch.password:
             writer = writer.option("password", ch.password)
@@ -112,7 +225,7 @@ def stage_to_clickhouse(
         df.write.jdbc(
             config.clickhouse.jdbc_url,
             config.transform.staging_table,
-            mode="overwrite",
+            mode=mode,
             properties=config.clickhouse.jdbc_properties,
         )
     logger.info(
@@ -131,14 +244,15 @@ def stage_to_path(
     """Write transformed data to parquet/delta (for local dev or intermediate storage)."""
     path = config.transform.staging_path
     fmt = config.transform.staging_format
+    mode = _normalize_staging_write_mode(config.transform.staging_write_mode)
     if fmt == "parquet":
-        df.write.mode("overwrite").parquet(path)
+        df.write.mode(mode).parquet(path)
         logger.info("Staged data to %s (parquet)", path)
     elif fmt == "delta":
-        df.write.format("delta").mode("overwrite").save(path)
+        df.write.format("delta").mode(mode).save(path)
         logger.info("Staged data to %s (delta)", path)
     else:
-        df.write.format(fmt).mode("overwrite").save(path)
+        df.write.format(fmt).mode(mode).save(path)
         logger.info("Staged data to %s (%s)", path, fmt)
 
 

{batch_analytics-0.3.7 → batch_analytics-0.3.14}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.7
+Version: 0.3.14
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT