sws-spark-dissemination-helper 0.0.133__tar.gz → 0.0.136__tar.gz

{sws_spark_dissemination_helper-0.0.133 → sws_spark_dissemination_helper-0.0.136}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.133
+Version: 0.0.136
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.133 → sws_spark_dissemination_helper-0.0.136}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.133"
+version = "0.0.136"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.133 → sws_spark_dissemination_helper-0.0.136}/src/sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -240,21 +240,26 @@ class SWSEasyIcebergSparkHelper:
             u.email,
             o.created_on,
             o.replaced_on,
-            o.version,
-            o.flag_obs_status,
-            o.flag_method"""
+            o.version"""
 
         from_statement = f"""
         from {self.dataset_tables.OBSERVATION.iceberg_id} o
             join {self.dataset_tables.USER.iceberg_id} u ON u.id = o.created_by
             left join {self.dataset_tables.OBSERVATION_COORDINATE.iceberg_id} as oc on oc.id = o.observation_coordinates"""
 
+        logging.info("self.flag_col_to_id_mapping")
+        logging.info(self.flag_col_to_id_mapping)
+        for flag_col in self.flag_columns:
+            select_statement += (
+                f",\no.{self.flag_col_to_id_mapping[flag_col]} as {flag_col}"
+            )
+
         id_to_dim_col_mapping = {v: k for k, v in self.dim_col_to_id_mapping.items()}
-        for i, (dim, cl) in enumerate(
+        for i, (dim_col, cl) in enumerate(
             zip(self.dim_columns_w_time, self.dataset_tables.CODELISTS)
         ):
-            select_statement += f",\nd{i}.code as {dim}"
-            from_statement += f"\nleft join {cl.iceberg_id} d{i} on d{i}.id = oc.{id_to_dim_col_mapping[dim]}"
+            select_statement += f",\nd{i}.code as {dim_col}"
+            from_statement += f"\nleft join {cl.iceberg_id} d{i} on d{i}.id = oc.{id_to_dim_col_mapping[dim_col]}"
 
         final_query = select_statement + from_statement
         if not self.keep_history: