batch-analytics 0.3.15__tar.gz → 0.3.17__tar.gz

{batch_analytics-0.3.15 → batch_analytics-0.3.17}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.15
+Version: 0.3.17
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: pyspark<3.6,>=3.4
-Requires-Dist: numpy>=1.19.0
+Requires-Dist: numpy>=1.22.0
 Requires-Dist: scipy>=1.5.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
@@ -16,26 +16,22 @@ Requires-Dist: scipy>=1.5.0; extra == "ttest"
 Provides-Extra: s3
 Requires-Dist: boto3>=1.28; extra == "s3"
 Provides-Extra: clickhouse
-Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "clickhouse"
-Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "clickhouse"
+Requires-Dist: clickhouse-connect>=0.7; extra == "clickhouse"
 Provides-Extra: output
 Requires-Dist: boto3>=1.28; extra == "output"
-Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "output"
-Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "output"
+Requires-Dist: clickhouse-connect>=0.7; extra == "output"
 Provides-Extra: autogluon
 Requires-Dist: autogluon<2.0,>=1.0; extra == "autogluon"
 Requires-Dist: pandas>=1.3.0; extra == "autogluon"
 Requires-Dist: boto3>=1.28; extra == "autogluon"
-Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "autogluon"
-Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "autogluon"
-Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "autogluon"
+Requires-Dist: clickhouse-connect>=0.7; extra == "autogluon"
+Requires-Dist: pyarrow>=10.0.0; extra == "autogluon"
 Provides-Extra: full
 Requires-Dist: scipy>=1.5.0; extra == "full"
 Requires-Dist: boto3>=1.28; extra == "full"
-Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "full"
-Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "full"
+Requires-Dist: clickhouse-connect>=0.7; extra == "full"
 Requires-Dist: autogluon<2.0,>=1.0; extra == "full"
-Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "full"
+Requires-Dist: pyarrow>=10.0.0; extra == "full"
 
 # Batch Analytics
 

{batch_analytics-0.3.15 → batch_analytics-0.3.17}/pyproject.toml

@@ -4,13 +4,14 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.15"
+version = "0.3.17"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
-requires-python = ">=3.8"
+# AutoGluon 1.x pulls thinc/blis which need NumPy 2.x; NumPy 2 does not support Python 3.8.
+requires-python = ">=3.9"
 dependencies = [
     "pyspark>=3.4,<3.6",
-    "numpy>=1.19.0",
+    "numpy>=1.22.0",
     # Welch t-test (t_test.py); keep on core deps so `pip install batch-analytics` works in minimal driver images
     "scipy>=1.5.0",
 ]
@@ -22,33 +23,28 @@ dev = ["pytest>=7.0"]
 # Legacy: scipy is a core dependency; kept so `pip install "batch-analytics[ttest]"` still resolves.
 ttest = ["scipy>=1.5.0"]
 s3 = ["boto3>=1.28"]
-# 0.9+ uses list[...] etc. and breaks on Python 3.8; 3.9+ can take current clickhouse-connect.
 clickhouse = [
-    "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
-    "clickhouse-connect>=0.7; python_version >= '3.9'",
+    "clickhouse-connect>=0.7",
 ]
 output = [
     "boto3>=1.28",
-    "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
-    "clickhouse-connect>=0.7; python_version >= '3.9'",
+    "clickhouse-connect>=0.7",
 ]
-# AutoGluon Tabular train/infer (POST /submit/train, /submit/inf on analytics_runner)
+# AutoGluon Tabular train/infer (POST /submit/train, /submit/inf on analytics_runner). Requires Python 3.9+.
 autogluon = [
     "autogluon>=1.0,<2.0",
     "pandas>=1.3.0",
     "boto3>=1.28",
-    "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
-    "clickhouse-connect>=0.7; python_version >= '3.9'",
-    "pyarrow>=10.0.0; python_version >= '3.8'",
+    "clickhouse-connect>=0.7",
+    "pyarrow>=10.0.0",
 ]
 # Install all optional runtime deps used anywhere in the package
 full = [
     "scipy>=1.5.0",
     "boto3>=1.28",
-    "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
-    "clickhouse-connect>=0.7; python_version >= '3.9'",
+    "clickhouse-connect>=0.7",
     "autogluon>=1.0,<2.0",
-    "pyarrow>=10.0.0; python_version >= '3.8'",
+    "pyarrow>=10.0.0",
 ]
 
 [project.scripts]

{batch_analytics-0.3.15 → batch_analytics-0.3.17}/src/batch_analytics/analytics/gluon_autogluon_infer.py

@@ -2,7 +2,8 @@
 Load AutoGluon TabularPredictor from S3; score ClickHouse rows; write predictions to output.
 
 Env: CLICKHOUSE_*, BATCH_STAGING_TABLE (inference feature table), MODEL_S3_PREFIX,
-  OUTPUT_TYPE, OUTPUT_CLICKHOUSE_DATABASE, OUTPUT_CLICKHOUSE_TABLE (clickhouse),
+  OUTPUT_TYPE (from OutputConfig.type), OUTPUT_CLICKHOUSE_DATABASE, OUTPUT_CLICKHOUSE_TABLE (clickhouse),
+  OUTPUT_CLICKHOUSE_AUTO_CREATE (optional; default true when unset — CREATE TABLE IF NOT EXISTS for clickhouse),
   or OUTPUT_S3_PATH (s3 parquet)
 """
 
@@ -37,6 +38,54 @@ logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
 logger = logging.getLogger(__name__)
 
 
+def _env_truthy(name: str, *, default: bool) -> bool:
+    raw = (os.environ.get(name) or "").strip().lower()
+    if not raw:
+        return default
+    return raw in ("1", "true", "yes", "on")
+
+
+def _sql_ident(name: str) -> str:
+    return "`" + str(name).replace("`", "``") + "`"
+
+
+def _pandas_col_ch_type(series: pd.Series) -> str:
+    """Map a pandas column to ClickHouse; Nullable(...) when the column has nulls."""
+    try:
+        kind = series.dtype.kind
+    except AttributeError:
+        kind = "O"
+    if kind == "b":
+        base = "Bool"
+    elif kind == "i":
+        sz = getattr(series.dtype, "itemsize", 8) or 8
+        base = {1: "Int8", 2: "Int16", 4: "Int32", 8: "Int64"}.get(sz, "Int64")
+    elif kind == "u":
+        sz = getattr(series.dtype, "itemsize", 8) or 8
+        base = {1: "UInt8", 2: "UInt16", 4: "UInt32", 8: "UInt64"}.get(sz, "UInt64")
+    elif kind == "f":
+        sz = getattr(series.dtype, "itemsize", 8) or 8
+        base = "Float32" if sz <= 4 else "Float64"
+    elif kind == "M":
+        base = "DateTime64(3)"
+    else:
+        base = "String"
+    if series.isna().any():
+        return f"Nullable({base})"
+    return base
+
+
+def _ensure_clickhouse_output_table(client, database: str, table: str, out_df: pd.DataFrame) -> None:
+    col_defs = []
+    for col in out_df.columns:
+        col_defs.append(f"  {_sql_ident(col)} {_pandas_col_ch_type(out_df[col])}")
+    body = ",\n".join(col_defs)
+    fq = f"{_sql_ident(database)}.{_sql_ident(table)}"
+    ddl = f"CREATE TABLE IF NOT EXISTS {fq} (\n{body}\n) ENGINE = MergeTree ORDER BY tuple()"
+    logger.info("Ensuring ClickHouse output table exists: %s.%s", database, table)
+    client.command(ddl)
+
+
 def _require(name: str) -> str:
     v = os.environ.get(name, "").strip()
     if not v:
@@ -93,6 +142,8 @@ def main() -> None:
             odb = _require("OUTPUT_CLICKHOUSE_DATABASE")
             otbl = _require("OUTPUT_CLICKHOUSE_TABLE")
             out_full = clickhouse_full_table(odb, otbl)
+            if _env_truthy("OUTPUT_CLICKHOUSE_AUTO_CREATE", default=True):
+                _ensure_clickhouse_output_table(client, odb, otbl, out_df)
             logger.info("Inserting %s rows into %s", len(out_df), out_full)
             client.insert_df(out_full, out_df)
         else:

{batch_analytics-0.3.15 → batch_analytics-0.3.17}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.15
+Version: 0.3.17
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: pyspark<3.6,>=3.4
-Requires-Dist: numpy>=1.19.0
+Requires-Dist: numpy>=1.22.0
 Requires-Dist: scipy>=1.5.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
@@ -16,26 +16,22 @@ Requires-Dist: scipy>=1.5.0; extra == "ttest"
 Provides-Extra: s3
 Requires-Dist: boto3>=1.28; extra == "s3"
 Provides-Extra: clickhouse
-Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "clickhouse"
-Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "clickhouse"
+Requires-Dist: clickhouse-connect>=0.7; extra == "clickhouse"
 Provides-Extra: output
 Requires-Dist: boto3>=1.28; extra == "output"
-Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "output"
-Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "output"
+Requires-Dist: clickhouse-connect>=0.7; extra == "output"
 Provides-Extra: autogluon
 Requires-Dist: autogluon<2.0,>=1.0; extra == "autogluon"
 Requires-Dist: pandas>=1.3.0; extra == "autogluon"
 Requires-Dist: boto3>=1.28; extra == "autogluon"
-Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "autogluon"
-Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "autogluon"
-Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "autogluon"
+Requires-Dist: clickhouse-connect>=0.7; extra == "autogluon"
+Requires-Dist: pyarrow>=10.0.0; extra == "autogluon"
 Provides-Extra: full
 Requires-Dist: scipy>=1.5.0; extra == "full"
 Requires-Dist: boto3>=1.28; extra == "full"
-Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "full"
-Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "full"
+Requires-Dist: clickhouse-connect>=0.7; extra == "full"
 Requires-Dist: autogluon<2.0,>=1.0; extra == "full"
-Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "full"
+Requires-Dist: pyarrow>=10.0.0; extra == "full"
 
 # Batch Analytics
 

batch_analytics-0.3.17/src/batch_analytics.egg-info/requires.txt

@@ -0,0 +1,33 @@
+pyspark<3.6,>=3.4
+numpy>=1.22.0
+scipy>=1.5.0
+
+[autogluon]
+autogluon<2.0,>=1.0
+pandas>=1.3.0
+boto3>=1.28
+clickhouse-connect>=0.7
+pyarrow>=10.0.0
+
+[clickhouse]
+clickhouse-connect>=0.7
+
+[dev]
+pytest>=7.0
+
+[full]
+scipy>=1.5.0
+boto3>=1.28
+clickhouse-connect>=0.7
+autogluon<2.0,>=1.0
+pyarrow>=10.0.0
+
+[output]
+boto3>=1.28
+clickhouse-connect>=0.7
+
+[s3]
+boto3>=1.28
+
+[ttest]
+scipy>=1.5.0

batch_analytics-0.3.15/src/batch_analytics.egg-info/requires.txt

@@ -1,57 +0,0 @@
-pyspark<3.6,>=3.4
-numpy>=1.19.0
-scipy>=1.5.0
-
-[autogluon]
-autogluon<2.0,>=1.0
-pandas>=1.3.0
-boto3>=1.28
-
-[autogluon:python_version < "3.9"]
-clickhouse-connect<0.9,>=0.7
-
-[autogluon:python_version >= "3.8"]
-pyarrow>=10.0.0
-
-[autogluon:python_version >= "3.9"]
-clickhouse-connect>=0.7
-
-[clickhouse]
-
-[clickhouse:python_version < "3.9"]
-clickhouse-connect<0.9,>=0.7
-
-[clickhouse:python_version >= "3.9"]
-clickhouse-connect>=0.7
-
-[dev]
-pytest>=7.0
-
-[full]
-scipy>=1.5.0
-boto3>=1.28
-autogluon<2.0,>=1.0
-
-[full:python_version < "3.9"]
-clickhouse-connect<0.9,>=0.7
-
-[full:python_version >= "3.8"]
-pyarrow>=10.0.0
-
-[full:python_version >= "3.9"]
-clickhouse-connect>=0.7
-
-[output]
-boto3>=1.28
-
-[output:python_version < "3.9"]
-clickhouse-connect<0.9,>=0.7
-
-[output:python_version >= "3.9"]
-clickhouse-connect>=0.7
-
-[s3]
-boto3>=1.28
-
-[ttest]
-scipy>=1.5.0