sws-spark-dissemination-helper 0.0.86__py3-none-any.whl → 0.0.88__py3-none-any.whl

sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -443,3 +443,78 @@ class SWSBronzeIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Bronze Dissemination tags successfully written")
+
+    def write_bronze_disseminated_tag_data_to_iceberg_and_csv(
+        self, dimensions: Dict[str, List[str]]
+    ) -> DataFrame:
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.BRONZE.iceberg_id}.`tag_{self.tag_name}` CREATE OR REPLACE BRANCH `diss_tag_{self.tag_name}`"  # AS OF VERSION `{tag_name}`
+        )
+
+        for dimension_name, codes in dimensions.items():
+            if len(codes) != 0:
+                not_in_codes = ",".join([f"'{code}'" for code in codes])
+                self.spark.sql(
+                    f"DELETE FROM {self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}` WHERE {dimension_name} NOT IN ({not_in_codes})"
+                )
+        disseminated_tag_df = self.spark.read.option("branch", self.tag_name).table(
+            self.iceberg_tables.BRONZE.iceberg_id
+        )
+
+        disseminated_tag_df = disseminated_tag_df.withColumn(
+            "metadata", F.to_json(col("metadata"))
+        ).coalesce(1)
+
+        save_cache_csv(
+            df=disseminated_tag_df,
+            bucket=self.bucket,
+            prefix=f"{self.iceberg_tables.BRONZE.csv_prefix}_disseminated_tag",
+            tag_name=self.tag_name,
+        )
+
+        return disseminated_tag_df
+
+    def write_bronze_sws_filtered_disseminated_tag(self, tags: Tags):
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_bronze_disseminated_tag_iceberg",
+            name=f"{self.domain_code} bronze disseminated tag Iceberg",
+            description="Bronze table containing the raw data imported from the SWS, denormalized and filtered per dimension",
+            layer=TableLayer.BRONZE,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.BRONZE_DATABASE,
+            table=self.iceberg_tables.BRONZE.table,
+            path=self.iceberg_tables.BRONZE.path,
+            structure={
+                "columns": self.disseminated_tag_df.schema.jsonValue()["fields"]
+            },
+        )
+        tag = tags.add_dissemination_table(
+            self.dataset_id, self.tag_name, new_iceberg_table
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_csv_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_bronze_disseminated_tag_csv",
+            name=f"{self.domain_code} bronze disseminated tag csv",
+            description="Bronze table containing the raw data imported from the SWS, denormalized and filtered per dimension cached in csv",
+            layer=TableLayer.BRONZE,
+            private=True,
+            type=TableType.CSV,
+            # TODO Correct the path in the origin library
+            path=self.iceberg_tables.BRONZE.csv_path,
+            structure={
+                "columns": self.disseminated_tag_df.schema.jsonValue()["fields"]
+            },
+        )
+        tag = tags.add_dissemination_table(
+            self.dataset_id, self.tag_name, new_csv_table
+        )
+
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        logging.info("Bronze Disseminated tag with selection successfully written")

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -89,20 +89,20 @@ class SWSGoldIcebergSparkHelper:
     def keep_dim_val_attr_columns(self, df: DataFrame):
         return df.select(*self.cols_to_keep_sws)
 
+    def read_silver_data(self) -> DataFrame:
+        return self.spark.read.option("tag", self.tag_name).table(
+            self.iceberg_tables.SILVER.iceberg_id
+        )
+
     def gen_gold_sws_disseminated_data(self) -> DataFrame:
         return (
-            self.spark.read.option("tag", self.tag_name)
-            .table(self.iceberg_tables.SILVER.iceberg_id)
+            self.read_silver_data()
             .transform(self.apply_diss_flag_filter)
             .transform(self.keep_dim_val_attr_columns)
         )
 
     def gen_gold_sws_validated_data(self) -> DataFrame:
-        return (
-            self.spark.read.option("tag", self.tag_name)
-            .table(self.iceberg_tables.BRONZE.iceberg_id)
-            .transform(self.keep_dim_val_attr_columns)
-        )
+        return self.read_silver_data().transform(self.keep_dim_val_attr_columns)
 
     def write_gold_sws_validated_data_to_iceberg_and_csv(
         self, df: DataFrame

sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -110,6 +110,10 @@ class SWSSilverIcebergSparkHelper:
         return self.spark.read.option("tag", self.tag_name).table(
             self.iceberg_tables.BRONZE.iceberg_id
         )
+    def read_bronze_diss_tag_data(self) -> DataFrame:
+        return self.spark.read.option("branch", f"diss_tag_{self.tag_name}").table(
+            self.iceberg_tables.BRONZE.iceberg_id
+        )
 
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""

{sws_spark_dissemination_helper-0.0.86.dist-info → sws_spark_dissemination_helper-0.0.88.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.86
+Version: 0.0.88
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.86.dist-info → sws_spark_dissemination_helper-0.0.88.dist-info}/RECORD

@@ -1,11 +1,11 @@
-sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=tyC3e2LNBes9J2UFR-j7bDlvEffeI0YsiYlMvk0wPxA,16382
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=o8THI9uVKkNdtZVqs4Rsa9666eOjcSxAD0H_tKs9v4w,16059
+sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=5z3uaVgRjtvZFO8C8LG9k3GO2dO21Ht5l7MXPY5Hb5M,19673
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=ZC7hxkppo6qmfCc2z5vm2Y2iH1901F-rx9Er9cxuzP4,16037
 sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=ja7AbOfbmC_EXHCJk7UMDzzbA-LRxzPkaaUmuvcihJ8,17449
-sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=F0g4N95QIApVNvPFWuQfHphGE320LKoimBRisln7Luk,22033
+sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=zEppNq5shiHZH2yt5faWGsb5QEmpAQS0ToIrG6fmv6o,22231
 sws_spark_dissemination_helper/__init__.py,sha256=Efjoe9V4vGXWVp-DY5P6NbRwIUr_zkZJkDmMi-lf5Bc,262
 sws_spark_dissemination_helper/constants.py,sha256=hpHHlbojShMWRfyIelXz6c5BqFzO48Oap1zmztlMMrs,11349
 sws_spark_dissemination_helper/utils.py,sha256=6SzrXX0xhvynRyv-vRFDbc6V4UNe_RzKKETZAtefnhg,21341
-sws_spark_dissemination_helper-0.0.86.dist-info/METADATA,sha256=vACLr-NqneuRqAeZOkE0ZGAhWJozhBprO-zNJzniLgk,2823
-sws_spark_dissemination_helper-0.0.86.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.86.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.86.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.88.dist-info/METADATA,sha256=ta-N8JQzmir7jdw5Sm6k5dHKg1gkHkt-yCu25z0HbUY,2823
+sws_spark_dissemination_helper-0.0.88.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.88.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.88.dist-info/RECORD,,