batch-analytics 0.1.0__py3-none-any.whl

batch_analytics/config.py

@@ -0,0 +1,169 @@
+"""
+Configuration for batch analytics pipeline: Extract, Transform, Log stages.
+"""
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class ClickHouseConfig:
+    """ClickHouse connection settings."""
+
+    host: str = field(
+        default_factory=lambda: os.environ.get(
+            "CLICKHOUSE_HOST",
+            "my-simple-cluster-clickhouse-headless.default.svc.cluster.local",
+        )
+    )
+    port: int = int(os.environ.get("CLICKHOUSE_HTTP_PORT", "8123"))
+    database: str = os.environ.get("CLICKHOUSE_DB", "example_db")
+    user: str = os.environ.get("CLICKHOUSE_USER", "default")
+    password: str = os.environ.get("CLICKHOUSE_PASSWORD", "")
+    protocol: str = os.environ.get("CLICKHOUSE_PROTOCOL", "http")
+
+    @property
+    def jdbc_url(self) -> str:
+        return f"jdbc:ch://{self.host}:{self.port}/{self.database}"
+
+    @property
+    def jdbc_properties(self) -> dict:
+        props = {"user": self.user, "driver": "com.clickhouse.jdbc.ClickHouseDriver"}
+        if self.password:
+            props["password"] = self.password
+        return props
+
+
+@dataclass
+class ExtractConfig:
+    """Extract stage configuration."""
+
+    # Source table(s) - comma-separated for multiple, or single table name
+    source_tables: str = os.environ.get(
+        "BATCH_SOURCE_TABLES",
+        "batch_details_table,manufacturing_table,yield_table,temperature_table",
+    )
+    # Use ClickHouse native connector (format) vs JDBC fallback
+    use_native_connector: bool = os.environ.get(
+        "BATCH_USE_NATIVE_CONNECTOR", "false"
+    ).lower() == "true"
+
+
+@dataclass
+class TransformConfig:
+    """Transform stage configuration."""
+
+    # Columns to use for deduplication (comma-separated); empty = use all columns
+    dedup_columns: str = os.environ.get("BATCH_DEDUP_COLUMNS", "")
+    # Staging output path (local or S3)
+    staging_path: str = os.environ.get(
+        "BATCH_STAGING_PATH",
+        "/tmp/analytics_stage",
+    )
+    # Output format for load_staged when reading (parquet/delta/clickhouse).
+    # Stage job always writes to ClickHouse; use clickhouse for analytics to read from staged table.
+    staging_format: str = os.environ.get("BATCH_STAGING_FORMAT", "clickhouse")
+    # Staging table name in ClickHouse (when format=clickhouse)
+    staging_table: str = os.environ.get("BATCH_STAGING_TABLE", "analytics_staging")
+    # Extract anchor_id from add_dimension column (e.g. {'anchor_id':'GP/GPH(D)/II(W)/250019'})
+    add_dimension_column: str = os.environ.get("BATCH_ADD_DIMENSION_COLUMN", "add_dimension")
+    anchor_id_column: str = os.environ.get("BATCH_ANCHOR_ID_COLUMN", "anchor_id")
+
+
+@dataclass
+class LogConfig:
+    """Log stage configuration."""
+
+    log_path: str = os.environ.get(
+        "BATCH_LOG_PATH",
+        "/tmp/analytics_logs",
+    )
+    # Retention: number of runs to keep
+    retention_runs: int = int(os.environ.get("BATCH_LOG_RETENTION", "30"))
+
+
+@dataclass
+class OutputConfig:
+    """
+    Output destination for analytics results (injected by analytics_runner).
+
+    Env vars: OUTPUT_TYPE, OUTPUT_S3_PATH, OUTPUT_CLICKHOUSE_*, TASK_ID
+    """
+
+    type: str = os.environ.get("OUTPUT_TYPE", "local")
+    s3_path: str = os.environ.get("OUTPUT_S3_PATH", "")
+    clickhouse_database: str = os.environ.get("OUTPUT_CLICKHOUSE_DATABASE", "example_db")
+    clickhouse_table: str = os.environ.get("OUTPUT_CLICKHOUSE_TABLE", "analytics_results")
+    # Task ID from spark_runner (for S3 key prefix, ClickHouse task_id column)
+    task_id: str = os.environ.get("TASK_ID", "")
+
+
+@dataclass
+class AnalyticsConfig:
+    """Analytics modules configuration."""
+
+    # Module 1: Linear regression - X and Y column names
+    lr_x_column: str = os.environ.get("BATCH_LR_X_COLUMN", "x")
+    lr_y_column: str = os.environ.get("BATCH_LR_Y_COLUMN", "y")
+    # Groups to compare slopes (e.g. "product,batch_no")
+    lr_group_columns: str = os.environ.get("BATCH_LR_GROUP_COLUMNS", "")
+
+    # Module 2: Correlation - feature columns (comma-separated)
+    corr_features: str = os.environ.get(
+        "BATCH_CORR_FEATURES",
+        "",
+    )
+    corr_threshold: float = float(os.environ.get("BATCH_CORR_THRESHOLD", "0.8"))
+
+    # Module 3: PCA + Clustering
+    pca_features: str = os.environ.get("BATCH_PCA_FEATURES", "")
+    pca_variance_threshold: float = float(
+        os.environ.get("BATCH_PCA_VARIANCE_THRESHOLD", "0.95")
+    )
+    cluster_k: int = int(os.environ.get("BATCH_CLUSTER_K", "3"))
+
+    # Module 4: T-test - compare means of two groups
+    # Mode 1: value column + group column (2 levels)
+    ttest_value_column: str = os.environ.get("BATCH_TTEST_VALUE_COLUMN", "")
+    ttest_group_column: str = os.environ.get("BATCH_TTEST_GROUP_COLUMN", "")
+    # Mode 2: two numeric columns
+    ttest_col_a: str = os.environ.get("BATCH_TTEST_COL_A", "")
+    ttest_col_b: str = os.environ.get("BATCH_TTEST_COL_B", "")
+
+
+@dataclass
+class SparkK8sConfig:
+    """Spark on Kubernetes configuration."""
+
+    # Set SPARK_MASTER=k8s://https://kubernetes.default.svc:443 to enable K8s
+    master: str = os.environ.get("SPARK_MASTER", "local[*]")
+    container_image: str = os.environ.get("SPARK_K8S_IMAGE", "sudhakso/spark:3.5.0")
+    namespace: str = os.environ.get("SPARK_K8S_NAMESPACE", "default")
+    service_account: str = os.environ.get("SPARK_K8S_SERVICE_ACCOUNT", "spark-sa")
+    deploy_mode: str = os.environ.get("SPARK_DEPLOY_MODE", "client")
+    # Driver/executor resources
+    driver_memory: str = os.environ.get("SPARK_DRIVER_MEMORY", "512m")
+    driver_memory_overhead: str = os.environ.get("SPARK_DRIVER_MEMORY_OVERHEAD", "256m")
+    executor_instances: int = int(os.environ.get("SPARK_EXECUTOR_INSTANCES", "1"))
+    executor_cores: int = int(os.environ.get("SPARK_EXECUTOR_CORES", "1"))
+    executor_memory: str = os.environ.get("SPARK_EXECUTOR_MEMORY", "512m")
+    executor_memory_overhead: str = os.environ.get("SPARK_EXECUTOR_MEMORY_OVERHEAD", "128m")
+    # S3 (optional; set for s3a:// paths)
+    s3_access_key: str = os.environ.get("AWS_ACCESS_KEY_ID", "")
+    s3_secret_key: str = os.environ.get("AWS_SECRET_ACCESS_KEY", "")
+    s3_endpoint: str = os.environ.get("AWS_ENDPOINT", "s3.amazonaws.com")
+    s3_region: str = os.environ.get("AWS_REGION", "us-east-2")
+
+
+@dataclass
+class BatchAnalyticsConfig:
+    """Aggregate configuration for the pipeline."""
+
+    clickhouse: ClickHouseConfig = field(default_factory=ClickHouseConfig)
+    spark_k8s: SparkK8sConfig = field(default_factory=SparkK8sConfig)
+    extract: ExtractConfig = field(default_factory=ExtractConfig)
+    transform: TransformConfig = field(default_factory=TransformConfig)
+    log: LogConfig = field(default_factory=LogConfig)
+    output: OutputConfig = field(default_factory=OutputConfig)
+    analytics: AnalyticsConfig = field(default_factory=AnalyticsConfig)

batch_analytics/extract.py

@@ -0,0 +1,118 @@
+"""
+Extract stage: Load data from ClickHouse using Spark ClickHouse connector or JDBC.
+"""
+
+import logging
+from typing import Any
+
+from pyspark.sql import DataFrame, SparkSession
+
+from .config import BatchAnalyticsConfig
+
+logger = logging.getLogger(__name__)
+
+
+def _read_via_format(spark: SparkSession, cfg: BatchAnalyticsConfig, table: str) -> DataFrame | None:
+    """
+    Read from ClickHouse using the native format API (clickhouse-spark-runtime).
+    Requires: com.clickhouse.spark:clickhouse-spark-runtime in spark.jars.packages
+    """
+    try:
+        df = (
+            spark.read.format("clickhouse")
+            .option("host", cfg.clickhouse.host)
+            .option("protocol", cfg.clickhouse.protocol)
+            .option("http_port", str(cfg.clickhouse.port))
+            .option("database", cfg.clickhouse.database)
+            .option("table", table)
+            .option("user", cfg.clickhouse.user)
+            .load()
+        )
+        return df
+    except Exception as e:
+        logger.warning(
+            "Native ClickHouse connector failed for table %s: %s. Falling back to JDBC.",
+            table,
+            e,
+        )
+        return None
+
+
+def _read_via_jdbc(spark: SparkSession, cfg: BatchAnalyticsConfig, table: str) -> DataFrame:
+    """Read from ClickHouse via JDBC."""
+    return spark.read.jdbc(
+        cfg.clickhouse.jdbc_url,
+        table,
+        properties=cfg.clickhouse.jdbc_properties,
+    )
+
+
+def extract_table(
+    spark: SparkSession,
+    table: str,
+    config: BatchAnalyticsConfig,
+) -> DataFrame:
+    """
+    Extract a single table from ClickHouse.
+    Uses native connector if configured, otherwise JDBC.
+    """
+    if config.extract.use_native_connector:
+        df = _read_via_format(spark, config, table)
+        if df is None:
+            df = _read_via_jdbc(spark, config, table)
+    else:
+        df = _read_via_jdbc(spark, config, table)
+
+    logger.info("Extracted table %s: %d rows", table, df.count())
+    return df
+
+
+def extract_all(
+    spark: SparkSession,
+    config: BatchAnalyticsConfig,
+) -> dict[str, DataFrame]:
+    """
+    Extract all configured source tables from ClickHouse.
+    Returns a dict mapping table name to DataFrame.
+    """
+    tables = [t.strip() for t in config.extract.source_tables.split(",") if t.strip()]
+    if not tables:
+        raise ValueError("No source tables configured in BATCH_SOURCE_TABLES")
+
+    result: dict[str, DataFrame] = {}
+    for table in tables:
+        df = extract_table(spark, table, config)
+        result[table] = df
+
+    return result
+
+
+def extract_unified(
+    spark: SparkSession,
+    config: BatchAnalyticsConfig,
+    join_keys: list[str] | None = None,
+    primary_table: str | None = None,
+) -> DataFrame:
+    """
+    Extract and unify source tables into one DataFrame.
+    - Single table: returns it directly.
+    - Multiple tables + join_keys: joins on those keys (left join, first table base).
+    - Multiple tables, no join_keys: returns the primary (or first) table.
+      Use primary_table to choose which table to use for analytics.
+    """
+    all_dfs = extract_all(spark, config)
+
+    if len(all_dfs) == 1:
+        return list(all_dfs.values())[0]
+
+    if join_keys:
+        dfs = list(all_dfs.values())
+        base = dfs[0]
+        for other in dfs[1:]:
+            base = base.join(other, on=join_keys, how="left")
+        return base
+
+    # Multiple tables, no join: use primary or first
+    if primary_table and primary_table in all_dfs:
+        return all_dfs[primary_table]
+    return list(all_dfs.values())[0]

batch_analytics/job_runner.py

@@ -0,0 +1,300 @@
+"""
+Batch analytics job runner: orchestrates Extract, Transform, Log stages and analytics modules.
+"""
+
+import argparse
+import logging
+import os
+import socket
+import sys
+import uuid
+from pathlib import Path
+
+from pyspark.sql import SparkSession
+
+from .config import BatchAnalyticsConfig
+from .extract import extract_unified
+from .log import log_dataframe_summary, log_run
+from .modules import DEFAULT_MODULES, MODULE_REGISTRY, VALID_MODULES
+from .output import write_analytics_output
+from .transform import load_staged, stage_to_clickhouse, transform
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+def create_spark_session(
+    app_name: str = "BatchAnalytics",
+    clickhouse_jars: str | None = None,
+    config: BatchAnalyticsConfig | None = None,
+) -> SparkSession:
+    """
+    Create SparkSession. Uses Kubernetes config when SPARK_MASTER starts with k8s://.
+    """
+    config = config or BatchAnalyticsConfig()
+    cfg = config.spark_k8s
+
+    builder = (
+        SparkSession.builder.appName(app_name)
+        .config("spark.sql.adaptive.enabled", "true")
+        .config("spark.sql.adaptive.coalescePartitions.enabled", "true")
+    )
+
+    # JARs: ClickHouse JDBC + hadoop-aws (for S3 when on K8s)
+    packages = []
+    if clickhouse_jars:
+        packages.append(clickhouse_jars)
+    if cfg.master.startswith("k8s://"):
+        packages.append("org.apache.hadoop:hadoop-aws:3.3.4")
+    if packages:
+        builder = builder.config("spark.jars.packages", ",".join(packages))
+
+    if cfg.master.startswith("k8s://"):
+        driver_host = socket.gethostbyname(socket.gethostname())
+        builder = (
+            builder.master(cfg.master)
+            .config("spark.kubernetes.container.image", cfg.container_image)
+            .config("spark.kubernetes.namespace", cfg.namespace)
+            .config("spark.kubernetes.authenticate.serviceAccountName", cfg.service_account)
+            .config("spark.driver.host", driver_host)
+            .config("spark.driver.bindAddress", "0.0.0.0")
+            .config(
+                "spark.kubernetes.driver.pod.name",
+                os.environ.get("HOSTNAME", socket.gethostname()),
+            )
+            .config("spark.submit.deployMode", cfg.deploy_mode)
+            .config(
+                "spark.driver.extraJavaOptions",
+                "-Djava.net.preferIPv4Stack=true -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError",
+            )
+            .config("spark.driver.memory", cfg.driver_memory)
+            .config("spark.driver.memoryOverhead", cfg.driver_memory_overhead)
+            .config("spark.executor.instances", str(cfg.executor_instances))
+            .config("spark.executor.cores", str(cfg.executor_cores))
+            .config("spark.executor.memory", cfg.executor_memory)
+            .config("spark.executor.memoryOverhead", cfg.executor_memory_overhead)
+            .config("spark.kubernetes.executor.serviceAccountName", cfg.service_account)
+            .config("spark.kubernetes.container.image.pullPolicy", "IfNotPresent")
+        )
+        if cfg.s3_access_key and cfg.s3_secret_key:
+            builder = (
+                builder.config("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
+                .config("spark.hadoop.fs.s3a.access.key", cfg.s3_access_key)
+                .config("spark.hadoop.fs.s3a.secret.key", cfg.s3_secret_key)
+                .config("spark.hadoop.fs.s3a.endpoint", cfg.s3_endpoint)
+                .config("spark.hadoop.fs.s3a.endpoint.region", cfg.s3_region)
+            )
+        logger.info("Spark on Kubernetes: master=%s", cfg.master)
+
+    return builder.getOrCreate()
+
+
+def run_pipeline(
+    config: BatchAnalyticsConfig | None = None,
+    spark: SparkSession | None = None,
+    run_extract: bool = True,
+    run_transform: bool = True,
+    run_stage: bool = True,
+    run_analytics: bool = True,
+    modules: list[str] | None = None,
+) -> dict:
+    """
+    Run the full pipeline or selected stages.
+
+    Args:
+        config: Pipeline config (default: BatchAnalyticsConfig())
+        spark: SparkSession (created if None)
+        run_extract: Whether to extract from ClickHouse
+        run_transform: Whether to transform (clean, dedupe, extract anchor_id)
+        run_stage: Whether to stage transformed data to ClickHouse (separate job)
+        run_analytics: Whether to run analytics (requires staged data in ClickHouse)
+        modules: Which analytics to run: ["lr", "corr", "pca", "ttest"] or None for all
+
+    Returns:
+        Dict with run_id, stage results, analytics outputs
+    """
+    config = config or BatchAnalyticsConfig()
+    run_id = str(uuid.uuid4())[:8]
+
+    if spark is None:
+        jars = "com.clickhouse:clickhouse-jdbc:0.4.6:all"
+        spark = create_spark_session(
+            app_name="BatchAnalytics",
+            clickhouse_jars=jars,
+            config=config,
+        )
+
+    result = {"run_id": run_id, "stages": {}, "analytics": {}}
+
+    # ----- Extract -----
+    if run_extract:
+        logger.info("Stage: Extract")
+        df_raw = extract_unified(spark, config)
+        result["stages"]["extract"] = {
+            "row_count": df_raw.count(),
+            "columns": [f.name for f in df_raw.schema.fields],
+        }
+    else:
+        logger.info("Stage: Extract (skipped, loading from stage)")
+        df_raw = load_staged(spark, config)
+        result["stages"]["extract"] = {"skipped": True, "loaded_from_stage": True}
+
+    # ----- Transform -----
+    if run_transform:
+        logger.info("Stage: Transform")
+        df_transformed = transform(df_raw, config)
+        result["stages"]["transform"] = {"row_count": df_transformed.count()}
+    else:
+        df_transformed = df_raw  # df_raw already loaded from staged when not run_extract
+        if run_extract and run_analytics:
+            from .transform import extract_anchor_id, remove_duplicates
+
+            df_transformed = extract_anchor_id(df_raw, config)
+            dedup_cols = (
+                [c.strip() for c in config.transform.dedup_columns.split(",") if c.strip()]
+                if config.transform.dedup_columns
+                else None
+            )
+            df_transformed = remove_duplicates(df_transformed, key_columns=dedup_cols)
+        result["stages"]["transform"] = {"skipped": True}
+
+    # ----- Stage (to ClickHouse) - separate job before analytics -----
+    if run_stage:
+        logger.info("Stage: Stage to ClickHouse")
+        stage_to_clickhouse(spark, df_transformed, config)
+        result["stages"]["stage"] = {
+            "destination": f"{config.clickhouse.database}.{config.transform.staging_table}",
+            "row_count": df_transformed.count(),
+        }
+    else:
+        result["stages"]["stage"] = {"skipped": True}
+
+    # For analytics: use in-memory df (we always have df_transformed from transform/load)
+    df_for_analytics = df_transformed
+
+    # ----- Log stage metrics -----
+    log_run(
+        run_id,
+        "pipeline",
+        result["stages"],
+        output_dir=config.log.log_path,
+        extra={"config_summary": {"staging_format": config.transform.staging_format}},
+    )
+
+    # ----- Analytics (runs after stage; must run extract+transform+stage first, or use --from-stage) -----
+    if run_analytics:
+        modules = modules or DEFAULT_MODULES
+        for mod in modules:
+            if mod not in MODULE_REGISTRY:
+                logger.warning("Unknown module %r, skipping. Valid: %s", mod, VALID_MODULES)
+                continue
+            run_fn, result_key = MODULE_REGISTRY[mod]
+            try:
+                mod_result = run_fn(spark, df_for_analytics, config)
+                result["analytics"][result_key] = mod_result
+            except Exception as e:
+                logger.exception("%s failed: %s", result_key, e)
+                result["analytics"][result_key] = {"error": str(e)}
+
+        task_id = config.output.task_id or run_id
+        write_analytics_output(
+            run_id=run_id,
+            task_id=task_id,
+            artifacts=result["analytics"],
+            output_type=config.output.type,
+            path=config.log.log_path
+            if config.output.type == "local"
+            else config.output.s3_path,
+            database=config.output.clickhouse_database,
+            table=config.output.clickhouse_table,
+            host=config.clickhouse.host,
+            port=config.clickhouse.port,
+            user=config.clickhouse.user,
+            password=config.clickhouse.password,
+        )
+
+    return result
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Batch analytics: Extract, Transform, Log + analytics")
+    parser.add_argument(
+        "--extract",
+        action="store_true",
+        default=True,
+        help="Run extract stage (default: true)",
+    )
+    parser.add_argument(
+        "--no-extract",
+        action="store_false",
+        dest="extract",
+    )
+    parser.add_argument(
+        "--transform",
+        action="store_true",
+        default=True,
+        help="Run transform stage (default: true)",
+    )
+    parser.add_argument(
+        "--no-transform",
+        action="store_false",
+        dest="transform",
+    )
+    parser.add_argument(
+        "--stage",
+        action="store_true",
+        default=True,
+        help="Run stage step: write transformed data to ClickHouse (default: true)",
+    )
+    parser.add_argument(
+        "--no-stage",
+        action="store_false",
+        dest="stage",
+    )
+    parser.add_argument(
+        "--analytics",
+        action="store_true",
+        default=True,
+        help="Run analytics modules (default: true)",
+    )
+    parser.add_argument(
+        "--no-analytics",
+        action="store_false",
+        dest="analytics",
+    )
+    parser.add_argument(
+        "--modules",
+        nargs="+",
+        choices=VALID_MODULES,
+        default=None,
+        help="Analytics modules to run (default: all). Must match catalog module_arg.",
+    )
+    parser.add_argument(
+        "--from-stage",
+        action="store_true",
+        help="Load from staged ClickHouse table, run analytics only (implies --no-extract --no-transform --no-stage)",
+    )
+    args = parser.parse_args()
+
+    if args.from_stage:
+        args.extract = False
+        args.transform = False
+        args.stage = False
+
+    result = run_pipeline(
+        run_extract=args.extract,
+        run_transform=args.transform,
+        run_stage=args.stage,
+        run_analytics=args.analytics,
+        modules=args.modules,
+    )
+
+    logger.info("Run completed: %s", result["run_id"])
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

batch_analytics/log.py

@@ -0,0 +1,101 @@
+"""
+Log stage: Persist run metadata, metrics, and analytics results for audit and debugging.
+"""
+
+import json
+import logging
+from datetime import datetime
+from pathlib import Path
+
+from pyspark.sql import SparkSession
+
+logger = logging.getLogger(__name__)
+
+
+def log_run(
+    run_id: str,
+    stage: str,
+    metrics: dict,
+    output_dir: str | Path | None = None,
+    extra: dict | None = None,
+) -> Path:
+    """
+    Write run log (metadata + metrics) to a JSON file.
+    Returns the path of the written file.
+    """
+    output_dir = Path(output_dir) if output_dir else Path("/tmp/analytics_logs")
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    log_data = {
+        "run_id": run_id,
+        "stage": stage,
+        "timestamp": datetime.utcnow().isoformat() + "Z",
+        "metrics": metrics,
+        **(extra or {}),
+    }
+
+    log_file = output_dir / f"{run_id}_{stage}.json"
+    with open(log_file, "w") as f:
+        json.dump(log_data, f, indent=2)
+
+    logger.info("Logged %s to %s", stage, log_file)
+    return log_file
+
+
+def log_dataframe_summary(
+    spark: SparkSession,
+    df_name: str,
+    df,
+) -> dict:
+    """Produce a summary dict for a DataFrame (row count, schema snippet)."""
+    count = df.count()
+    schema = [f.name for f in df.schema.fields[:20]]  # first 20 columns
+    if len(df.schema.fields) > 20:
+        schema.append("...")
+    return {
+        "name": df_name,
+        "row_count": count,
+        "column_count": len(df.schema.fields),
+        "columns": schema,
+    }
+
+
+def log_analytics_artifacts(
+    run_id: str,
+    artifacts: dict,
+    output_dir: str | Path,
+) -> list[Path]:
+    """
+    Write analytics module outputs (e.g. slopes, correlation matrix, PCA loadings)
+    to JSON files in the log directory.
+    """
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    paths: list[Path] = []
+    for name, data in artifacts.items():
+        path = output_dir / f"{run_id}_analytics_{name}.json"
+        with open(path, "w") as f:
+            json.dump(_serialize_for_json(data), f, indent=2)
+        paths.append(path)
+
+    return paths
+
+
+def _serialize_for_json(obj):
+    """Convert numpy/Python types to JSON-serializable forms."""
+    import numpy as np
+
+    if isinstance(obj, dict):
+        return {k: _serialize_for_json(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [_serialize_for_json(x) for x in obj]
+    if isinstance(obj, np.ndarray):
+        return obj.tolist()
+    if isinstance(obj, (np.floating, np.float32, np.float64)):
+        return float(obj)
+    if isinstance(obj, (np.integer, np.int32, np.int64)):
+        return int(obj)
+    if isinstance(obj, np.bool_):
+        return bool(obj)
+    return obj