batch-analytics 0.3.2__tar.gz → 0.3.4__tar.gz

{batch_analytics-0.3.2 → batch_analytics-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.2
+Version: 0.3.4
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT

{batch_analytics-0.3.2 → batch_analytics-0.3.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "batch-analytics"
-version = "0.3.2"
+version = "0.3.4"
 description = "PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test)."
 readme = "README.md"
 requires-python = ">=3.8"

{batch_analytics-0.3.2 → batch_analytics-0.3.4}/src/batch_analytics/analytics/pca_clustering.py

@@ -84,6 +84,7 @@ def run_pca_clustering(
     n_comp_max = min(len(feature_cols), 20)
     pca = PCA(k=n_comp_max, inputCol="features", outputCol="pca_features")
     pca_model = pca.fit(df_scaled)
+    df_pca = pca_model.transform(df_scaled)
 
     explained = pca_model.explainedVariance.toArray().tolist()
     cumsum = []
@@ -108,9 +109,9 @@ def run_pca_clustering(
 
     k = config.analytics.cluster_k
     kmeans = KMeans(k=k, seed=42, featuresCol="pca_features", predictionCol="cluster")
-    kmeans_model = kmeans.fit(df_scaled)
+    kmeans_model = kmeans.fit(df_pca)
 
-    df_clustered = kmeans_model.transform(df_scaled)
+    df_clustered = kmeans_model.transform(df_pca)
     cluster_sizes = (
         df_clustered.groupBy("cluster")
         .count()

{batch_analytics-0.3.2 → batch_analytics-0.3.4}/src/batch_analytics/job_runner.py

@@ -113,6 +113,20 @@ def create_spark_session(
 
     if jar_list:
         builder = builder.config("spark.jars", ",".join(jar_list))
+    elif os.environ.get("BATCH_ALLOW_INHERITED_SPARK_JARS", "").strip().lower() not in (
+        "1",
+        "true",
+        "yes",
+    ):
+        # spark-submit / image ENV often set SPARK_JARS → --jars; Python never sees it in jar_list
+        # above, but Spark still adds them ("Added JAR /opt/spark/jars/...") and K8s executors fail.
+        # Empty spark.jars overrides inherited submit --jars; $SPARK_HOME/jars stay on JVM classpath.
+        builder = builder.config("spark.jars", "")
+        logger.info(
+            "spark.jars cleared (override spark-submit --jars / SPARK_JARS); "
+            "JARs under $SPARK_HOME/jars remain on the classpath. "
+            "Set BATCH_ALLOW_INHERITED_SPARK_JARS=1 to keep submit-inherited spark.jars."
+        )
     if packages:
         builder = builder.config("spark.jars.packages", ",".join(packages))
 

{batch_analytics-0.3.2 → batch_analytics-0.3.4}/src/batch_analytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: batch-analytics
-Version: 0.3.2
+Version: 0.3.4
 Summary: PySpark batch analytics: Extract, Transform, Stage, and analytical modules (linear regression, correlation, PCA, t-test).
 Author: Litewave Analytics Team
 License: MIT