batch-analytics 0.1.0__py3-none-any.whl

batch_analytics/modules.py

@@ -0,0 +1,24 @@
+"""
+Module registry for batch analytics. Maps --modules short names to run functions.
+
+Must stay in sync with analytics_runner catalog module_arg for each Spark method.
+See analytics_runner/catalog/analytics_catalog.yaml.
+"""
+
+from .analytics import (
+    run_linear_regression,
+    run_correlation,
+    run_pca_clustering,
+    run_t_test,
+)
+
+# module_arg -> (run_fn, result_key)
+MODULE_REGISTRY = {
+    "lr": (run_linear_regression, "linear_regression"),
+    "corr": (run_correlation, "correlation"),
+    "pca": (run_pca_clustering, "pca_clustering"),
+    "ttest": (run_t_test, "t_test"),
+}
+
+VALID_MODULES = list(MODULE_REGISTRY.keys())
+DEFAULT_MODULES = VALID_MODULES.copy()

batch_analytics/output/__init__.py

@@ -0,0 +1,22 @@
+"""
+Output drivers for analytics results: local, S3, ClickHouse.
+
+Configuration via env vars (injected by analytics_runner):
+- OUTPUT_TYPE: local | s3 | clickhouse
+- OUTPUT_S3_PATH: s3://bucket/prefix/ (when type=s3)
+- OUTPUT_CLICKHOUSE_DATABASE, OUTPUT_CLICKHOUSE_TABLE (when type=clickhouse)
+- TASK_ID: task identifier (injected by spark_runner)
+"""
+
+from .base import OutputDriver, write_analytics_output
+from .local import LocalOutputDriver
+from .s3 import S3OutputDriver
+from .clickhouse import ClickHouseOutputDriver
+
+__all__ = [
+    "OutputDriver",
+    "write_analytics_output",
+    "LocalOutputDriver",
+    "S3OutputDriver",
+    "ClickHouseOutputDriver",
+]

batch_analytics/output/base.py

@@ -0,0 +1,97 @@
+"""
+Base output driver interface and write orchestration.
+"""
+
+import json
+import logging
+from abc import ABC, abstractmethod
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def _serialize_for_json(obj: Any) -> Any:
+    """Convert numpy/Python types to JSON-serializable forms."""
+    import numpy as np
+
+    if isinstance(obj, dict):
+        return {k: _serialize_for_json(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [_serialize_for_json(x) for x in obj]
+    if isinstance(obj, np.ndarray):
+        return obj.tolist()
+    if isinstance(obj, (np.floating, np.float32, np.float64)):
+        return float(obj)
+    if isinstance(obj, (np.integer, np.int32, np.int64)):
+        return int(obj)
+    if isinstance(obj, np.bool_):
+        return bool(obj)
+    return obj
+
+
+class OutputDriver(ABC):
+    """Interface for writing analytics results to a destination."""
+
+    @abstractmethod
+    def write(
+        self,
+        run_id: str,
+        task_id: str,
+        artifacts: dict[str, Any],
+    ) -> list[str]:
+        """
+        Write analytics artifacts to the destination.
+
+        Args:
+            run_id: Unique run identifier
+            task_id: Task identifier from spark_runner
+            artifacts: Dict of module_name -> result (e.g. {"linear_regression": {...}})
+
+        Returns:
+            List of written locations (paths, keys, or identifiers)
+        """
+        pass
+
+
+def write_analytics_output(
+    run_id: str,
+    task_id: str,
+    artifacts: dict[str, Any],
+    output_type: str,
+    **driver_kwargs: Any,
+) -> list[str]:
+    """
+    Write analytics results using the configured output driver.
+
+    Args:
+        run_id: Run identifier
+        task_id: Task identifier (from TASK_ID env or run_id fallback)
+        artifacts: Analytics module outputs
+        output_type: local | s3 | clickhouse
+        **driver_kwargs: Driver-specific config (path, database, table, etc.)
+
+    Returns:
+        List of written locations
+    """
+    if not artifacts:
+        logger.debug("No analytics artifacts to write")
+        return []
+
+    output_type = (output_type or "local").lower().strip()
+    if output_type == "local":
+        driver = LocalOutputDriver(**driver_kwargs)
+    elif output_type == "s3":
+        driver = S3OutputDriver(**driver_kwargs)
+    elif output_type == "clickhouse":
+        driver = ClickHouseOutputDriver(**driver_kwargs)
+    else:
+        logger.warning("Unknown OUTPUT_TYPE=%r, falling back to local", output_type)
+        driver = LocalOutputDriver(**driver_kwargs)
+
+    try:
+        locations = driver.write(run_id, task_id, artifacts)
+        logger.info("Wrote analytics output to %s: %s", output_type, locations)
+        return locations
+    except Exception as e:
+        logger.exception("Failed to write analytics output: %s", e)
+        raise

batch_analytics/output/clickhouse.py

@@ -0,0 +1,89 @@
+"""
+ClickHouse output driver: inserts analytics results into a ClickHouse table.
+"""
+
+import json
+import logging
+from typing import Any
+
+from .base import OutputDriver, _serialize_for_json
+
+logger = logging.getLogger(__name__)
+
+
+def _create_table_sql(database: str, table: str) -> str:
+    """CREATE TABLE IF NOT EXISTS for analytics results."""
+    return f"""
+    CREATE TABLE IF NOT EXISTS {database}.{table} (
+        task_id String,
+        run_id String,
+        module String,
+        result String,
+        created_at DateTime DEFAULT now()
+    ) ENGINE = MergeTree()
+    ORDER BY (task_id, run_id, module)
+    """
+
+
+class ClickHouseOutputDriver(OutputDriver):
+    """Insert analytics artifacts into a ClickHouse table as JSON strings."""
+
+    def __init__(
+        self,
+        database: str,
+        table: str,
+        host: str | None = None,
+        port: int = 8123,
+        user: str = "default",
+        password: str = "",
+        **kwargs: Any,
+    ) -> None:
+        self.database = database
+        self.table = table
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+
+    def _client(self):
+        """Lazy clickhouse-connect client."""
+        try:
+            import clickhouse_connect
+        except ImportError as e:
+            raise ImportError(
+                "ClickHouse output requires clickhouse-connect. "
+                "Install with: pip install batch-analytics[clickhouse]"
+            ) from e
+
+        return clickhouse_connect.get_client(
+            host=self.host or "localhost",
+            port=self.port,
+            username=self.user,
+            password=self.password if self.password else None,
+        )
+
+    def write(
+        self,
+        run_id: str,
+        task_id: str,
+        artifacts: dict[str, Any],
+    ) -> list[str]:
+        client = self._client()
+
+        # Ensure table exists
+        client.command(_create_table_sql(self.database, self.table).strip())
+
+        rows: list[tuple[str, str, str, str]] = []
+        for module, data in artifacts.items():
+            result_json = json.dumps(_serialize_for_json(data))
+            rows.append((task_id, run_id, module, result_json))
+
+        client.insert(
+            f"{self.database}.{self.table}",
+            rows,
+            column_names=["task_id", "run_id", "module", "result"],
+        )
+
+        location = f"{self.database}.{self.table}"
+        logger.info("Inserted %d analytics rows into %s", len(rows), location)
+        return [location]

batch_analytics/output/local.py

@@ -0,0 +1,36 @@
+"""
+Local output driver: writes analytics results to a local directory (BATCH_LOG_PATH).
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from .base import OutputDriver, _serialize_for_json
+
+logger = logging.getLogger(__name__)
+
+
+class LocalOutputDriver(OutputDriver):
+    """Write analytics artifacts to local filesystem."""
+
+    def __init__(self, path: str | Path = "/tmp/analytics_logs", **kwargs: Any) -> None:
+        self.path = Path(path)
+        self.path.mkdir(parents=True, exist_ok=True)
+
+    def write(
+        self,
+        run_id: str,
+        task_id: str,
+        artifacts: dict[str, Any],
+    ) -> list[str]:
+        locations: list[str] = []
+        for name, data in artifacts.items():
+            filepath = self.path / f"{run_id}_analytics_{name}.json"
+            with open(filepath, "w") as f:
+                json.dump(_serialize_for_json(data), f, indent=2)
+            locations.append(str(filepath))
+
+        logger.info("Wrote %d analytics artifacts to %s", len(locations), self.path)
+        return locations

batch_analytics/output/s3.py

@@ -0,0 +1,82 @@
+"""
+S3 output driver: uploads analytics results as JSON to S3.
+"""
+
+import json
+import logging
+from typing import Any
+
+from .base import OutputDriver, _serialize_for_json
+
+logger = logging.getLogger(__name__)
+
+
+class S3OutputDriver(OutputDriver):
+    """Upload analytics artifacts to S3 as JSON objects."""
+
+    def __init__(
+        self,
+        path: str,
+        region: str | None = None,
+        endpoint: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """
+        Args:
+            path: S3 path (e.g. s3://bucket/prefix/) - must end with /
+            region: AWS region (default from env)
+            endpoint: Custom endpoint for S3-compatible storage
+        """
+        self.path = path.rstrip("/") + "/"
+        self.region = region
+        self.endpoint = endpoint
+
+    def _client(self):
+        """Lazy boto3 client to avoid import at module load."""
+        try:
+            import boto3
+            from botocore.config import Config
+        except ImportError as e:
+            raise ImportError(
+                "S3 output requires boto3. Install with: pip install batch-analytics[s3]"
+            ) from e
+
+        config = Config(signature_version="s3v4")
+        client_kwargs: dict[str, Any] = {}
+        if self.region:
+            client_kwargs["region_name"] = self.region
+        if self.endpoint:
+            client_kwargs["endpoint_url"] = self.endpoint
+
+        return boto3.client("s3", config=config, **client_kwargs)
+
+    def _parse_s3_path(self) -> tuple[str, str]:
+        """Parse s3://bucket/prefix into bucket and prefix."""
+        if not self.path.startswith("s3://"):
+            raise ValueError(f"Invalid S3 path: {self.path}")
+        parts = self.path[5:].split("/", 1)  # Remove s3://
+        bucket = parts[0]
+        prefix = parts[1].rstrip("/") + "/" if len(parts) > 1 else ""
+        return bucket, prefix
+
+    def write(
+        self,
+        run_id: str,
+        task_id: str,
+        artifacts: dict[str, Any],
+    ) -> list[str]:
+        bucket, prefix = self._parse_s3_path()
+        client = self._client()
+
+        # Use task_id for path: prefix/task_id/run_id_analytics_module.json
+        key_prefix = f"{prefix}{task_id}/"
+
+        locations: list[str] = []
+        for name, data in artifacts.items():
+            key = f"{key_prefix}{run_id}_analytics_{name}.json"
+            body = json.dumps(_serialize_for_json(data), indent=2)
+            client.put_object(Bucket=bucket, Key=key, Body=body, ContentType="application/json")
+            locations.append(f"s3://{bucket}/{key}")
+
+        logger.info("Uploaded %d analytics artifacts to s3://%s/%s", len(locations), bucket, key_prefix)
+        return locations

batch_analytics/transform.py

@@ -0,0 +1,184 @@
+"""
+Transform stage: Clean data (remove duplicates), extract add_dimension, and stage.
+"""
+
+import logging
+from typing import Sequence
+
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.functions import coalesce, col, get_json_object, regexp_extract
+
+from .config import BatchAnalyticsConfig
+
+logger = logging.getLogger(__name__)
+
+
+def extract_anchor_id(
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    """
+    Extract anchor_id from add_dimension column.
+    Supports JSON format {"anchor_id":"value"} or Python-dict {"anchor_id":"value"}.
+    Creates a new column (anchor_id by default) with the extracted value.
+    """
+    col_name = config.transform.add_dimension_column
+    out_col = config.transform.anchor_id_column
+
+    if col_name not in df.columns:
+        logger.debug("Column %s not found, skipping anchor_id extraction", col_name)
+        return df
+
+    # Valid JSON: {"anchor_id":"GP/GPH(D)/II(W)/250019"}
+    json_extract = get_json_object(col(col_name), "$.anchor_id")
+    # Python-dict style: {'anchor_id':'GP/GPH(D)/II(W)/250019'}
+    regex_extract = regexp_extract(col(col_name), r"'anchor_id'\s*:\s*'([^']*)'", 1)
+
+    extracted = coalesce(json_extract, regex_extract)
+    return df.withColumn(out_col, extracted)
+
+
+def remove_duplicates(
+    df: DataFrame,
+    key_columns: Sequence[str] | None = None,
+) -> DataFrame:
+    """
+    Remove duplicate rows.
+    If key_columns is provided, keeps first occurrence per key.
+    Otherwise, drops exact row duplicates.
+    """
+    before_count = df.count()
+    if key_columns:
+        df_cleaned = df.dropDuplicates(key_columns)
+    else:
+        df_cleaned = df.distinct()
+    after_count = df_cleaned.count()
+    removed = before_count - after_count
+    logger.info(
+        "Deduplication: %d -> %d rows (removed %d duplicates)",
+        before_count,
+        after_count,
+        removed,
+    )
+    return df_cleaned
+
+
+def transform(
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    """
+    Apply transformation only: extract anchor_id, remove duplicates.
+    Does not write anywhere. Use stage_to_clickhouse() separately to persist.
+    """
+    transformed = extract_anchor_id(df, config)
+    dedup_cols = (
+        [c.strip() for c in config.transform.dedup_columns.split(",") if c.strip()]
+        if config.transform.dedup_columns
+        else None
+    )
+    return remove_duplicates(transformed, key_columns=dedup_cols)
+
+
+def stage_to_clickhouse(
+    spark: SparkSession,
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> None:
+    """
+    Write transformed data to ClickHouse staging table.
+    Separate job from transform; must complete before analytics can run.
+    Uses native connector if available, else JDBC.
+    """
+    n = df.count()
+    try:
+        writer = (
+            df.write.format("clickhouse")
+            .option("host", config.clickhouse.host)
+            .option("database", config.clickhouse.database)
+            .option("table", config.transform.staging_table)
+            .option("user", config.clickhouse.user)
+            .mode("overwrite")
+        )
+        if config.clickhouse.password:
+            writer = writer.option("password", config.clickhouse.password)
+        writer.save()
+    except Exception as e:
+        logger.warning("ClickHouse connector failed (%s), using JDBC", e)
+        df.write.jdbc(
+            config.clickhouse.jdbc_url,
+            config.transform.staging_table,
+            mode="overwrite",
+            properties=config.clickhouse.jdbc_properties,
+        )
+    logger.info(
+        "Staged data to ClickHouse %s.%s (%d rows)",
+        config.clickhouse.database,
+        config.transform.staging_table,
+        n,
+    )
+
+
+def stage_to_path(
+    spark: SparkSession,
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> None:
+    """Write transformed data to parquet/delta (for local dev or intermediate storage)."""
+    path = config.transform.staging_path
+    fmt = config.transform.staging_format
+    if fmt == "parquet":
+        df.write.mode("overwrite").parquet(path)
+        logger.info("Staged data to %s (parquet)", path)
+    elif fmt == "delta":
+        df.write.format("delta").mode("overwrite").save(path)
+        logger.info("Staged data to %s (delta)", path)
+    else:
+        df.write.format(fmt).mode("overwrite").save(path)
+        logger.info("Staged data to %s (%s)", path, fmt)
+
+
+def transform_and_stage(
+    spark: SparkSession,
+    df: DataFrame,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    """
+    Transform and stage to ClickHouse. Kept for backward compatibility.
+    Prefer calling transform() then stage_to_clickhouse() separately.
+    """
+    cleaned = transform(df, config)
+    stage_to_clickhouse(spark, cleaned, config)
+    return cleaned
+
+
+def load_staged(
+    spark: SparkSession,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    """
+    Load previously staged data (e.g. when running only analytics modules).
+    """
+    staging_path = config.transform.staging_path
+    fmt = config.transform.staging_format
+
+    if fmt == "parquet":
+        return spark.read.parquet(staging_path)
+    if fmt == "delta":
+        return spark.read.format("delta").load(staging_path)
+    if fmt == "clickhouse":
+        try:
+            return (
+                spark.read.format("clickhouse")
+                .option("host", config.clickhouse.host)
+                .option("database", config.clickhouse.database)
+                .option("table", config.transform.staging_table)
+                .load()
+            )
+        except Exception:
+            return spark.read.jdbc(
+                config.clickhouse.jdbc_url,
+                config.transform.staging_table,
+                properties=config.clickhouse.jdbc_properties,
+            )
+    return spark.read.format(fmt).load(staging_path)

batch_analytics-0.1.0.dist-info/METADATA

@@ -0,0 +1,80 @@
+Metadata-Version: 2.4
+Name: batch-analytics
+Version: 0.1.0
+Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
+Author: Analytics Team
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pyspark<3.6,>=3.4
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Provides-Extra: s3
+Requires-Dist: boto3>=1.28; extra == "s3"
+Provides-Extra: clickhouse
+Requires-Dist: clickhouse-connect>=0.7; extra == "clickhouse"
+Provides-Extra: output
+Requires-Dist: boto3>=1.28; extra == "output"
+Requires-Dist: clickhouse-connect>=0.7; extra == "output"
+
+# Batch Analytics
+
+PySpark-based analytics pipeline for ClickHouse data: **Extract** → **Transform** → **Stage** → **Analytics**. Designed to run as the main application inside a Spark driver container (invoked by `analytics_runners` via SparkApplication CRD).
+
+## Bundle contents
+
+Only the files required for the batch analytics job runner:
+
+```
+analytics/
+├── pyproject.toml
+├── requirements-batch.txt
+├── README.md
+└── src/
+    └── batch_analytics/
+        ├── __init__.py
+        ├── __main__.py        # python -m batch_analytics
+        ├── job_runner.py      # Entry point
+        ├── config.py
+        ├── extract.py
+        ├── transform.py
+        ├── log.py
+        ├── README.md
+        └── analytics/
+            ├── __init__.py
+            ├── linear_regression.py
+            ├── correlation.py
+            ├── pca_clustering.py
+            └── t_test.py
+```
+
+## Install
+
+```bash
+pip install -e .
+# or: pip install -r requirements-batch.txt && pip install -e .
+```
+
+## Run
+
+```bash
+# Via module
+python -m batch_analytics
+
+# Via CLI (after pip install -e .)
+batch-analytics
+
+# Full pipeline
+batch-analytics
+
+# Analytics only (from staged ClickHouse table)
+batch-analytics --from-stage --modules lr corr pca ttest
+```
+
+## Configuration
+
+See `src/batch_analytics/README.md` for environment variables and usage.
+
+## Docker image
+
+For Spark on Kubernetes, build an image that includes this package and exposes `job_runner.py` at the path used by `mainApplicationFile` (e.g. `local:///opt/analytics/job_runner.py`).

batch_analytics-0.1.0.dist-info/RECORD

@@ -0,0 +1,23 @@
+batch_analytics/__init__.py,sha256=5tdHpnDS80B_cgQN1aj1DVj28z0n-PAQyNbw_rJpFOk,1214
+batch_analytics/__main__.py,sha256=lnQrJI_tQWnm1X0KjKwfwm5O9sIcg_SZNECbDanAFXM,119
+batch_analytics/config.py,sha256=PW2NNGifaeOFyEkJ-dV4OXHb4fn32LUskN1MVvnUh5g,6803
+batch_analytics/extract.py,sha256=qZKLebkJ9bn14wT6K04XgvNKwhjfGnDJ77MwQJckCoo,3555
+batch_analytics/job_runner.py,sha256=xznaYVTwkUUs_upAaO_lT9vsiI2vjTv7sac6tFm3oIw,10673
+batch_analytics/log.py,sha256=Hq3jFCdnMZzrHsFzO16lGZfKHxPYXkZOiZJpOgWxJmM,2743
+batch_analytics/modules.py,sha256=ND2fZRtwwlI-HvH4xFg3Cj5YGd31qUEohHmjuejJY0o,677
+batch_analytics/transform.py,sha256=KVlLUIpM2qnB877-QCm42qxPJMyhOcg7Rkd802jCsxs,5844
+batch_analytics/analytics/__init__.py,sha256=wyyAXs3Owu92mhixlViK3yWfqH4KWXmopmswEqrLP70,515
+batch_analytics/analytics/correlation.py,sha256=WmmZll8yfcB2rSSpoCOeTzYn4PDThupAOJKAkxsXzoo,3238
+batch_analytics/analytics/linear_regression.py,sha256=wpdsjyzl29umPD4hHBCcHaBwb1cx4qWfKBNfTxCAX8I,4439
+batch_analytics/analytics/pca_clustering.py,sha256=jdWwtU_G0Mkn50Lc_zOJv14clLE2a5-Gy0o-ynmLFvs,4375
+batch_analytics/analytics/t_test.py,sha256=aYoRTEXSxaxsIYgievHSkRZPogxCK5tObGNIHf5RV04,5497
+batch_analytics/output/__init__.py,sha256=79e5QJ9IJAHCk1e9HZi5g6WXUvnq5MKI9I0WhEaIbD4,665
+batch_analytics/output/base.py,sha256=-HVj5HA4jQ7Lvk0B9WR2kyk_1wu6-wvVXGV10WzxgPg,2852
+batch_analytics/output/clickhouse.py,sha256=gUI0LQqA6Lg4ZObDe3_9QJp_cguM9AbRCxdUeRgAfhY,2531
+batch_analytics/output/local.py,sha256=aDGa-riSEaZStSV4qE_zLfb4Af70O1G4XvHUiXbPMGk,1072
+batch_analytics/output/s3.py,sha256=oSyd8mL5nL-ryNddclpszFgfYChrogzgaNCJwdYZZYU,2695
+batch_analytics-0.1.0.dist-info/METADATA,sha256=HdahIx8c2NiwtPXLQPEisMeXBIrfoBFmrKruYcrNBPM,2308
+batch_analytics-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+batch_analytics-0.1.0.dist-info/entry_points.txt,sha256=v1Yx6LOlDBC0DmcdaaDM7KBROGe8GmBO7d6-Q2_z4dg,68
+batch_analytics-0.1.0.dist-info/top_level.txt,sha256=wpRlC_JZ_uyGxzP1P3HlpNAjL8qS8kjXCE03DE2Dlyk,16
+batch_analytics-0.1.0.dist-info/RECORD,,

batch_analytics-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

batch_analytics-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+batch-analytics = batch_analytics.job_runner:main

batch_analytics-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+batch_analytics