batch-analytics 0.3.14__tar.gz → 0.3.16__tar.gz

{batch_analytics-0.3.14 → batch_analytics-0.3.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.14
+Version: 0.3.16
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
@@ -22,11 +22,20 @@ Provides-Extra: output
 Requires-Dist: boto3>=1.28; extra == "output"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "output"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "output"
+Provides-Extra: autogluon
+Requires-Dist: autogluon<2.0,>=1.0; extra == "autogluon"
+Requires-Dist: pandas>=1.3.0; extra == "autogluon"
+Requires-Dist: boto3>=1.28; extra == "autogluon"
+Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "autogluon"
+Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "autogluon"
+Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "autogluon"
 Provides-Extra: full
 Requires-Dist: scipy>=1.5.0; extra == "full"
 Requires-Dist: boto3>=1.28; extra == "full"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "full"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "full"
+Requires-Dist: autogluon<2.0,>=1.0; extra == "full"
+Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "full"
 
 # Batch Analytics
 

{batch_analytics-0.3.14 → batch_analytics-0.3.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.14"
+version = "0.3.16"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"
@@ -32,12 +32,23 @@ output = [
     "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
     "clickhouse-connect>=0.7; python_version >= '3.9'",
 ]
+# AutoGluon Tabular train/infer (POST /submit/train, /submit/inf on analytics_runner)
+autogluon = [
+    "autogluon>=1.0,<2.0",
+    "pandas>=1.3.0",
+    "boto3>=1.28",
+    "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
+    "clickhouse-connect>=0.7; python_version >= '3.9'",
+    "pyarrow>=10.0.0; python_version >= '3.8'",
+]
 # Install all optional runtime deps used anywhere in the package
 full = [
     "scipy>=1.5.0",
     "boto3>=1.28",
     "clickhouse-connect>=0.7,<0.9; python_version < '3.9'",
     "clickhouse-connect>=0.7; python_version >= '3.9'",
+    "autogluon>=1.0,<2.0",
+    "pyarrow>=10.0.0; python_version >= '3.8'",
 ]
 
 [project.scripts]

batch_analytics-0.3.16/src/batch_analytics/analytics/gluon_autogluon_infer.py

@@ -0,0 +1,167 @@
+"""
+Load AutoGluon TabularPredictor from S3; score ClickHouse rows; write predictions to output.
+
+Env: CLICKHOUSE_*, BATCH_STAGING_TABLE (inference feature table), MODEL_S3_PREFIX,
+  OUTPUT_TYPE (from OutputConfig.type), OUTPUT_CLICKHOUSE_DATABASE, OUTPUT_CLICKHOUSE_TABLE (clickhouse),
+  OUTPUT_CLICKHOUSE_AUTO_CREATE (optional; default true when unset — CREATE TABLE IF NOT EXISTS for clickhouse),
+  or OUTPUT_S3_PATH (s3 parquet)
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import sys
+import tempfile
+
+import clickhouse_connect
+import pandas as pd
+
+try:
+    from batch_analytics.utils.gluon_autogluon_common import (
+        clickhouse_full_table,
+        download_s3_prefix_to_dir,
+        parse_s3_uri,
+    )
+except ImportError:
+    _pkg_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    if _pkg_root not in sys.path:
+        sys.path.insert(0, _pkg_root)
+    from utils.gluon_autogluon_common import (  # noqa: E402
+        clickhouse_full_table,
+        download_s3_prefix_to_dir,
+        parse_s3_uri,
+    )
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def _env_truthy(name: str, *, default: bool) -> bool:
+    raw = (os.environ.get(name) or "").strip().lower()
+    if not raw:
+        return default
+    return raw in ("1", "true", "yes", "on")
+
+
+def _sql_ident(name: str) -> str:
+    return "`" + str(name).replace("`", "``") + "`"
+
+
+def _pandas_col_ch_type(series: pd.Series) -> str:
+    """Map a pandas column to ClickHouse; Nullable(...) when the column has nulls."""
+    try:
+        kind = series.dtype.kind
+    except AttributeError:
+        kind = "O"
+    if kind == "b":
+        base = "Bool"
+    elif kind == "i":
+        sz = getattr(series.dtype, "itemsize", 8) or 8
+        base = {1: "Int8", 2: "Int16", 4: "Int32", 8: "Int64"}.get(sz, "Int64")
+    elif kind == "u":
+        sz = getattr(series.dtype, "itemsize", 8) or 8
+        base = {1: "UInt8", 2: "UInt16", 4: "UInt32", 8: "UInt64"}.get(sz, "UInt64")
+    elif kind == "f":
+        sz = getattr(series.dtype, "itemsize", 8) or 8
+        base = "Float32" if sz <= 4 else "Float64"
+    elif kind == "M":
+        base = "DateTime64(3)"
+    else:
+        base = "String"
+    if series.isna().any():
+        return f"Nullable({base})"
+    return base
+
+
+def _ensure_clickhouse_output_table(client, database: str, table: str, out_df: pd.DataFrame) -> None:
+    col_defs = []
+    for col in out_df.columns:
+        col_defs.append(f"  {_sql_ident(col)} {_pandas_col_ch_type(out_df[col])}")
+    body = ",\n".join(col_defs)
+    fq = f"{_sql_ident(database)}.{_sql_ident(table)}"
+    ddl = f"CREATE TABLE IF NOT EXISTS {fq} (\n{body}\n) ENGINE = MergeTree ORDER BY tuple()"
+    logger.info("Ensuring ClickHouse output table exists: %s.%s", database, table)
+    client.command(ddl)
+
+
+def _require(name: str) -> str:
+    v = os.environ.get(name, "").strip()
+    if not v:
+        logger.error("Missing required env var: %s", name)
+        sys.exit(2)
+    return v
+
+
+def main() -> None:
+    model_prefix = _require("MODEL_S3_PREFIX")
+    out_type = os.environ.get("OUTPUT_TYPE", "clickhouse").strip().lower()
+    if out_type not in ("clickhouse", "s3"):
+        logger.error("OUTPUT_TYPE must be clickhouse or s3, got %r", out_type)
+        sys.exit(2)
+
+    host = _require("CLICKHOUSE_HOST")
+    port = int(os.environ.get("CLICKHOUSE_HTTP_PORT", "8123"))
+    database = os.environ.get("CLICKHOUSE_DB", "default").strip() or "default"
+    user = os.environ.get("CLICKHOUSE_USER", "default")
+    password = os.environ.get("CLICKHOUSE_PASSWORD", "")
+    inference_table = _require("BATCH_STAGING_TABLE")
+    full_table = clickhouse_full_table(database, inference_table)
+
+    sql = f"SELECT * FROM {full_table}"
+    logger.info("Loading rows to score: %s", sql)
+    client = clickhouse_connect.get_client(
+        host=host,
+        port=port,
+        username=user,
+        password=password or None,
+        database=database,
+    )
+    df = client.query_df(sql)
+    if df.empty:
+        logger.warning("Inference input is empty; nothing to write")
+        return
+
+    try:
+        from autogluon.tabular import TabularPredictor
+    except ImportError:
+        logger.exception("autogluon is not installed; use pip install 'batch-analytics[autogluon]'")
+        sys.exit(4)
+
+    local_model = tempfile.mkdtemp(prefix="ag_infer_")
+    try:
+        logger.info("Downloading model from %s", model_prefix)
+        download_s3_prefix_to_dir(model_prefix, local_model)
+        predictor = TabularPredictor.load(local_model)
+        y_pred = predictor.predict(df)
+        out_df = df.copy()
+        out_df["prediction"] = pd.Series(y_pred, index=df.index)
+
+        if out_type == "clickhouse":
+            odb = _require("OUTPUT_CLICKHOUSE_DATABASE")
+            otbl = _require("OUTPUT_CLICKHOUSE_TABLE")
+            out_full = clickhouse_full_table(odb, otbl)
+            if _env_truthy("OUTPUT_CLICKHOUSE_AUTO_CREATE", default=True):
+                _ensure_clickhouse_output_table(client, odb, otbl, out_df)
+            logger.info("Inserting %s rows into %s", len(out_df), out_full)
+            client.insert_df(out_full, out_df)
+        else:
+            path = _require("OUTPUT_S3_PATH")
+            bucket, key = parse_s3_uri(path.rstrip("/") + "/")
+            key = key.rstrip("/")
+            if key:
+                key = key + "/"
+            parquet_key = key + "predictions.parquet"
+            tmp_parquet = os.path.join(local_model, "predictions.parquet")
+            out_df.to_parquet(tmp_parquet, index=False)
+            import boto3
+
+            boto3.client("s3").upload_file(tmp_parquet, bucket, parquet_key)
+            logger.info("Wrote s3://%s/%s", bucket, parquet_key)
+    finally:
+        shutil.rmtree(local_model, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()

batch_analytics-0.3.16/src/batch_analytics/analytics/gluon_autogluon_train.py

@@ -0,0 +1,114 @@
+"""
+Train AutoGluon TabularPredictor from ClickHouse staging data; upload artifacts to S3.
+
+Env (injected by analytics_runner Gluon job):
+  CLICKHOUSE_*, BATCH_STAGING_TABLE, MODEL_S3_PREFIX, TASK_ID,
+  AUTOGLUON_LABEL, AUTOGLUON_FEATURES, AUTOGLUON_PROBLEM_TYPE, AUTOGLUON_TIME_LIMIT,
+  optional AUTOGLUON_MAX_ROWS
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import sys
+
+import clickhouse_connect
+
+try:
+    from batch_analytics.utils.gluon_autogluon_common import (
+        clickhouse_full_table,
+        local_training_dir,
+        upload_directory_to_s3,
+    )
+except ImportError:
+    _pkg_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    if _pkg_root not in sys.path:
+        sys.path.insert(0, _pkg_root)
+    from utils.gluon_autogluon_common import (  # noqa: E402
+        clickhouse_full_table,
+        local_training_dir,
+        upload_directory_to_s3,
+    )
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def _require(name: str) -> str:
+    v = os.environ.get(name, "").strip()
+    if not v:
+        logger.error("Missing required env var: %s", name)
+        sys.exit(2)
+    return v
+
+
+def main() -> None:
+    model_prefix = _require("MODEL_S3_PREFIX")
+    label = _require("AUTOGLUON_LABEL")
+    features_raw = _require("AUTOGLUON_FEATURES")
+    feature_list = [c.strip() for c in features_raw.split(",") if c.strip()]
+
+    host = _require("CLICKHOUSE_HOST")
+    port = int(os.environ.get("CLICKHOUSE_HTTP_PORT", "8123"))
+    database = os.environ.get("CLICKHOUSE_DB", "default").strip() or "default"
+    user = os.environ.get("CLICKHOUSE_USER", "default")
+    password = os.environ.get("CLICKHOUSE_PASSWORD", "")
+    staging_table = _require("BATCH_STAGING_TABLE")
+    full_table = clickhouse_full_table(database, staging_table)
+
+    max_rows = os.environ.get("AUTOGLUON_MAX_ROWS", "").strip()
+    limit_sql = f" LIMIT {int(max_rows)}" if max_rows else ""
+    sql = f"SELECT * FROM {full_table}{limit_sql}"
+
+    logger.info("Loading training data: %s", sql)
+    client = clickhouse_connect.get_client(
+        host=host,
+        port=port,
+        username=user,
+        password=password or None,
+        database=database,
+    )
+    df = client.query_df(sql)
+
+    missing = [c for c in feature_list + [label] if c not in df.columns]
+    if missing:
+        logger.error("Columns missing from training data: %s (have: %s)", missing, list(df.columns))
+        sys.exit(3)
+
+    try:
+        from autogluon.tabular import TabularPredictor
+    except ImportError:
+        logger.exception("autogluon is not installed; use pip install 'batch-analytics[autogluon]'")
+        sys.exit(4)
+
+    problem_type = os.environ.get("AUTOGLUON_PROBLEM_TYPE", "binary").strip() or "binary"
+    time_limit = int(os.environ.get("AUTOGLUON_TIME_LIMIT", "300"))
+
+    local_dir = local_training_dir()
+    if os.path.isdir(local_dir):
+        shutil.rmtree(local_dir)
+    os.makedirs(local_dir, exist_ok=True)
+
+    train_df = df[feature_list + [label]]
+    logger.info(
+        "Fitting TabularPredictor problem_type=%s time_limit=%ss rows=%s",
+        problem_type,
+        time_limit,
+        len(train_df),
+    )
+    predictor = TabularPredictor(
+        label=label,
+        problem_type=problem_type,
+        path=local_dir,
+    )
+    predictor.fit(train_df, time_limit=time_limit)
+
+    logger.info("Uploading model artifacts to %s", model_prefix)
+    upload_directory_to_s3(local_dir, model_prefix)
+    logger.info("Train finished; task_id=%s", os.environ.get("TASK_ID", ""))
+
+
+if __name__ == "__main__":
+    main()

batch_analytics-0.3.16/src/batch_analytics/utils/__init__.py

@@ -0,0 +1 @@
+"""Utilities shared across batch_analytics (e.g. Gluon / AutoGluon helpers)."""

batch_analytics-0.3.16/src/batch_analytics/utils/gluon_autogluon_common.py

@@ -0,0 +1,83 @@
+"""Shared helpers for AutoGluon train/infer Gluon jobs (S3 + table naming)."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Tuple
+
+
+def parse_s3_uri(uri: str) -> Tuple[str, str]:
+    """
+    Split ``s3://bucket/key/prefix`` into bucket and key prefix (may be empty).
+
+    The key prefix does not include a leading slash; trailing slashes are preserved on the key side.
+    """
+    u = (uri or "").strip()
+    if not u.startswith("s3://"):
+        raise ValueError(f"Not an s3 URI: {uri!r}")
+    rest = u[5:]
+    if "/" not in rest:
+        return rest, ""
+    bucket, key = rest.split("/", 1)
+    return bucket, key
+
+
+def clickhouse_full_table(database: str, table: str) -> str:
+    """Build ``db.table`` when ``table`` is unqualified."""
+    t = (table or "").strip()
+    if not t:
+        return t
+    if "." in t and "'" not in t:
+        return t
+    db = (database or "").strip() or "default"
+    return f"{db}.{t}"
+
+
+def local_training_dir() -> str:
+    return os.environ.get("AUTOGLUON_LOCAL_MODEL_DIR", "/tmp/autogluon_model")
+
+
+def boto3_client():
+    import boto3
+
+    return boto3.client("s3")
+
+
+def upload_directory_to_s3(local_dir: str, s3_dir_uri: str) -> None:
+    """Upload every file under ``local_dir`` to ``s3_dir_uri`` (directory URI, trailing ``/`` optional)."""
+    bucket, prefix = parse_s3_uri(s3_dir_uri.rstrip("/") + "/")
+    prefix = prefix.rstrip("/")
+    if prefix:
+        prefix = prefix + "/"
+    cli = boto3_client()
+    root = Path(local_dir)
+    for path in root.rglob("*"):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(root).as_posix()
+        key = prefix + rel
+        cli.upload_file(str(path), bucket, key)
+
+
+def download_s3_prefix_to_dir(s3_dir_uri: str, local_dir: str) -> None:
+    """Download all objects under the S3 prefix implied by ``s3_dir_uri`` into ``local_dir``."""
+    bucket, pfx = parse_s3_uri(s3_dir_uri.rstrip("/") + "/")
+    pfx = pfx.rstrip("/")
+    if pfx:
+        pfx = pfx + "/"
+    cli = boto3_client()
+    os.makedirs(local_dir, exist_ok=True)
+    paginator = cli.get_paginator("list_objects_v2")
+    pages = paginator.paginate(Bucket=bucket, Prefix=pfx)
+    for page in pages:
+        for obj in page.get("Contents") or []:
+            key = obj["Key"]
+            if key.endswith("/"):
+                continue
+            rel = key[len(pfx) :] if key.startswith(pfx) else key
+            if not rel:
+                continue
+            dest = Path(local_dir) / rel
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            cli.download_file(bucket, key, str(dest))

{batch_analytics-0.3.14 → batch_analytics-0.3.16}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.14
+Version: 0.3.16
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT
@@ -22,11 +22,20 @@ Provides-Extra: output
 Requires-Dist: boto3>=1.28; extra == "output"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "output"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "output"
+Provides-Extra: autogluon
+Requires-Dist: autogluon<2.0,>=1.0; extra == "autogluon"
+Requires-Dist: pandas>=1.3.0; extra == "autogluon"
+Requires-Dist: boto3>=1.28; extra == "autogluon"
+Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "autogluon"
+Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "autogluon"
+Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "autogluon"
 Provides-Extra: full
 Requires-Dist: scipy>=1.5.0; extra == "full"
 Requires-Dist: boto3>=1.28; extra == "full"
 Requires-Dist: clickhouse-connect<0.9,>=0.7; python_version < "3.9" and extra == "full"
 Requires-Dist: clickhouse-connect>=0.7; python_version >= "3.9" and extra == "full"
+Requires-Dist: autogluon<2.0,>=1.0; extra == "full"
+Requires-Dist: pyarrow>=10.0.0; python_version >= "3.8" and extra == "full"
 
 # Batch Analytics
 

{batch_analytics-0.3.14 → batch_analytics-0.3.16}/src/batch_analytics.egg-info/SOURCES.txt

@@ -16,6 +16,8 @@ src/batch_analytics.egg-info/requires.txt
 src/batch_analytics.egg-info/top_level.txt
 src/batch_analytics/analytics/__init__.py
 src/batch_analytics/analytics/correlation.py
+src/batch_analytics/analytics/gluon_autogluon_infer.py
+src/batch_analytics/analytics/gluon_autogluon_train.py
 src/batch_analytics/analytics/linear_regression.py
 src/batch_analytics/analytics/pca_clustering.py
 src/batch_analytics/analytics/t_test.py
@@ -23,4 +25,6 @@ src/batch_analytics/output/__init__.py
 src/batch_analytics/output/base.py
 src/batch_analytics/output/clickhouse.py
 src/batch_analytics/output/local.py
-src/batch_analytics/output/s3.py
+src/batch_analytics/output/s3.py
+src/batch_analytics/utils/__init__.py
+src/batch_analytics/utils/gluon_autogluon_common.py

{batch_analytics-0.3.14 → batch_analytics-0.3.16}/src/batch_analytics.egg-info/requires.txt

@@ -2,6 +2,20 @@ pyspark<3.6,>=3.4
 numpy>=1.19.0
 scipy>=1.5.0
 
+[autogluon]
+autogluon<2.0,>=1.0
+pandas>=1.3.0
+boto3>=1.28
+
+[autogluon:python_version < "3.9"]
+clickhouse-connect<0.9,>=0.7
+
+[autogluon:python_version >= "3.8"]
+pyarrow>=10.0.0
+
+[autogluon:python_version >= "3.9"]
+clickhouse-connect>=0.7
+
 [clickhouse]
 
 [clickhouse:python_version < "3.9"]
@@ -16,10 +30,14 @@ pytest>=7.0
 [full]
 scipy>=1.5.0
 boto3>=1.28
+autogluon<2.0,>=1.0
 
 [full:python_version < "3.9"]
 clickhouse-connect<0.9,>=0.7
 
+[full:python_version >= "3.8"]
+pyarrow>=10.0.0
+
 [full:python_version >= "3.9"]
 clickhouse-connect>=0.7