sws-spark-dissemination-helper 0.0.87__py3-none-any.whl → 0.0.89__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py +4 -4
- {sws_spark_dissemination_helper-0.0.87.dist-info → sws_spark_dissemination_helper-0.0.89.dist-info}/METADATA +1 -1
- {sws_spark_dissemination_helper-0.0.87.dist-info → sws_spark_dissemination_helper-0.0.89.dist-info}/RECORD +5 -5
- {sws_spark_dissemination_helper-0.0.87.dist-info → sws_spark_dissemination_helper-0.0.89.dist-info}/WHEEL +0 -0
- {sws_spark_dissemination_helper-0.0.87.dist-info → sws_spark_dissemination_helper-0.0.89.dist-info}/licenses/LICENSE +0 -0
|
@@ -457,9 +457,9 @@ class SWSBronzeIcebergSparkHelper:
|
|
|
457
457
|
self.spark.sql(
|
|
458
458
|
f"DELETE FROM {self.iceberg_tables.BRONZE.iceberg_id}.`branch_diss_tag_{self.tag_name}` WHERE {dimension_name} NOT IN ({not_in_codes})"
|
|
459
459
|
)
|
|
460
|
-
disseminated_tag_df = self.spark.read.option(
|
|
461
|
-
self.
|
|
462
|
-
)
|
|
460
|
+
self.disseminated_tag_df = self.spark.read.option(
|
|
461
|
+
"branch", self.tag_name
|
|
462
|
+
).table(self.iceberg_tables.BRONZE.iceberg_id)
|
|
463
463
|
|
|
464
464
|
disseminated_tag_df = disseminated_tag_df.withColumn(
|
|
465
465
|
"metadata", F.to_json(col("metadata"))
|
|
@@ -474,7 +474,7 @@ class SWSBronzeIcebergSparkHelper:
|
|
|
474
474
|
|
|
475
475
|
return disseminated_tag_df
|
|
476
476
|
|
|
477
|
-
def
|
|
477
|
+
def write_bronze_sws_filtered_disseminated_tag(self, tags: Tags):
|
|
478
478
|
# Get or create a new tag
|
|
479
479
|
tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
|
|
480
480
|
logging.debug(f"Tag: {tag}")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sws-spark-dissemination-helper
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.89
|
|
4
4
|
Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
|
|
5
5
|
Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
|
|
6
6
|
Author-email: Daniele Mansillo <danielemansillo@gmail.com>
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=
|
|
1
|
+
sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=7q7OZt0EJ7MmoAfVeJGqxvd0737w33-DUjV1mcpkJUY,19678
|
|
2
2
|
sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=ZC7hxkppo6qmfCc2z5vm2Y2iH1901F-rx9Er9cxuzP4,16037
|
|
3
3
|
sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=ja7AbOfbmC_EXHCJk7UMDzzbA-LRxzPkaaUmuvcihJ8,17449
|
|
4
4
|
sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=zEppNq5shiHZH2yt5faWGsb5QEmpAQS0ToIrG6fmv6o,22231
|
|
5
5
|
sws_spark_dissemination_helper/__init__.py,sha256=Efjoe9V4vGXWVp-DY5P6NbRwIUr_zkZJkDmMi-lf5Bc,262
|
|
6
6
|
sws_spark_dissemination_helper/constants.py,sha256=hpHHlbojShMWRfyIelXz6c5BqFzO48Oap1zmztlMMrs,11349
|
|
7
7
|
sws_spark_dissemination_helper/utils.py,sha256=6SzrXX0xhvynRyv-vRFDbc6V4UNe_RzKKETZAtefnhg,21341
|
|
8
|
-
sws_spark_dissemination_helper-0.0.
|
|
9
|
-
sws_spark_dissemination_helper-0.0.
|
|
10
|
-
sws_spark_dissemination_helper-0.0.
|
|
11
|
-
sws_spark_dissemination_helper-0.0.
|
|
8
|
+
sws_spark_dissemination_helper-0.0.89.dist-info/METADATA,sha256=Kmx76PB-4p4QP0AJum15odQukfN2siUGMv40Az1bZS4,2823
|
|
9
|
+
sws_spark_dissemination_helper-0.0.89.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
10
|
+
sws_spark_dissemination_helper-0.0.89.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
|
|
11
|
+
sws_spark_dissemination_helper-0.0.89.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|