batch-analytics 0.3.15__tar.gz → 0.3.16__tar.gz

{batch_analytics-0.3.15 → batch_analytics-0.3.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.15
+Version: 0.3.16
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.15 → batch_analytics-0.3.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.15"
+version = "0.3.16"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.3.15 → batch_analytics-0.3.16}/src/batch_analytics/analytics/gluon_autogluon_infer.py

@@ -2,7 +2,8 @@
 Load AutoGluon TabularPredictor from S3; score ClickHouse rows; write predictions to output.
 
 Env: CLICKHOUSE_*, BATCH_STAGING_TABLE (inference feature table), MODEL_S3_PREFIX,
-  OUTPUT_TYPE, OUTPUT_CLICKHOUSE_DATABASE, OUTPUT_CLICKHOUSE_TABLE (clickhouse),
+  OUTPUT_TYPE (from OutputConfig.type), OUTPUT_CLICKHOUSE_DATABASE, OUTPUT_CLICKHOUSE_TABLE (clickhouse),
+  OUTPUT_CLICKHOUSE_AUTO_CREATE (optional; default true when unset — CREATE TABLE IF NOT EXISTS for clickhouse),
   or OUTPUT_S3_PATH (s3 parquet)
 """
 
@@ -37,6 +38,54 @@ logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
 logger = logging.getLogger(__name__)
 
 
+def _env_truthy(name: str, *, default: bool) -> bool:
+    raw = (os.environ.get(name) or "").strip().lower()
+    if not raw:
+        return default
+    return raw in ("1", "true", "yes", "on")
+
+
+def _sql_ident(name: str) -> str:
+    return "`" + str(name).replace("`", "``") + "`"
+
+
+def _pandas_col_ch_type(series: pd.Series) -> str:
+    """Map a pandas column to ClickHouse; Nullable(...) when the column has nulls."""
+    try:
+        kind = series.dtype.kind
+    except AttributeError:
+        kind = "O"
+    if kind == "b":
+        base = "Bool"
+    elif kind == "i":
+        sz = getattr(series.dtype, "itemsize", 8) or 8
+        base = {1: "Int8", 2: "Int16", 4: "Int32", 8: "Int64"}.get(sz, "Int64")
+    elif kind == "u":
+        sz = getattr(series.dtype, "itemsize", 8) or 8
+        base = {1: "UInt8", 2: "UInt16", 4: "UInt32", 8: "UInt64"}.get(sz, "UInt64")
+    elif kind == "f":
+        sz = getattr(series.dtype, "itemsize", 8) or 8
+        base = "Float32" if sz <= 4 else "Float64"
+    elif kind == "M":
+        base = "DateTime64(3)"
+    else:
+        base = "String"
+    if series.isna().any():
+        return f"Nullable({base})"
+    return base
+
+
+def _ensure_clickhouse_output_table(client, database: str, table: str, out_df: pd.DataFrame) -> None:
+    col_defs = []
+    for col in out_df.columns:
+        col_defs.append(f"  {_sql_ident(col)} {_pandas_col_ch_type(out_df[col])}")
+    body = ",\n".join(col_defs)
+    fq = f"{_sql_ident(database)}.{_sql_ident(table)}"
+    ddl = f"CREATE TABLE IF NOT EXISTS {fq} (\n{body}\n) ENGINE = MergeTree ORDER BY tuple()"
+    logger.info("Ensuring ClickHouse output table exists: %s.%s", database, table)
+    client.command(ddl)
+
+
 def _require(name: str) -> str:
     v = os.environ.get(name, "").strip()
     if not v:
@@ -93,6 +142,8 @@ def main() -> None:
             odb = _require("OUTPUT_CLICKHOUSE_DATABASE")
             otbl = _require("OUTPUT_CLICKHOUSE_TABLE")
             out_full = clickhouse_full_table(odb, otbl)
+            if _env_truthy("OUTPUT_CLICKHOUSE_AUTO_CREATE", default=True):
+                _ensure_clickhouse_output_table(client, odb, otbl, out_df)
             logger.info("Inserting %s rows into %s", len(out_df), out_full)
             client.insert_df(out_full, out_df)
         else:

{batch_analytics-0.3.15 → batch_analytics-0.3.16}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.15
+Version: 0.3.16
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT