sws-spark-dissemination-helper 0.0.144__tar.gz → 0.0.146__tar.gz

{sws_spark_dissemination_helper-0.0.144 → sws_spark_dissemination_helper-0.0.146}/.gitignore

@@ -2,7 +2,6 @@
 # You should customize this list as applicable to your project.
 # Learn more about .gitignore:
 #     https://www.atlassian.com/git/tutorials/saving-changes/gitignore
-.*
 
 # Node artifact files
 node_modules/
@@ -49,3 +48,4 @@ Thumbs.db
 *.mov
 *.wmv
 
+.venv/

{sws_spark_dissemination_helper-0.0.144 → sws_spark_dissemination_helper-0.0.146}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.144
+Version: 0.0.146
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
-Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
+Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>
 License: MIT License
         
@@ -31,8 +31,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Requires-Dist: annotated-types==0.7.0
-Requires-Dist: boto3==1.36.18
-Requires-Dist: botocore==1.36.18
+Requires-Dist: boto3>=1.36.18
+Requires-Dist: botocore>=1.36.18
 Requires-Dist: certifi==2025.1.31
 Requires-Dist: charset-normalizer==3.4.1
 Requires-Dist: idna==3.10
@@ -49,7 +49,7 @@ Requires-Dist: pytz==2025.1
 Requires-Dist: requests==2.32.3
 Requires-Dist: s3transfer==0.11.2
 Requires-Dist: six==1.17.0
-Requires-Dist: sws-api-client>=1.5.1
+Requires-Dist: sws-api-client==1.5.3
 Requires-Dist: typing-extensions==4.12.2
 Requires-Dist: tzdata==2025.1
 Requires-Dist: urllib3==1.26.20

{sws_spark_dissemination_helper-0.0.144 → sws_spark_dissemination_helper-0.0.146}/pyproject.toml

@@ -4,11 +4,11 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.144"
+version = "0.0.146"
 dependencies = [
     "annotated-types==0.7.0",
-    "boto3==1.36.18",
-    "botocore==1.36.18",
+    "boto3>=1.36.18",
+    "botocore>=1.36.18",
     "certifi==2025.1.31",
     "charset-normalizer==3.4.1",
     "idna==3.10",
@@ -25,7 +25,7 @@ dependencies = [
     "requests==2.32.3",
     "s3transfer==0.11.2",
     "six==1.17.0",
-    "sws_api_client>=1.5.1",
+    "sws_api_client==1.5.3",
     "typing_extensions==4.12.2",
     "tzdata==2025.1",
     "urllib3==1.26.20"
@@ -42,4 +42,4 @@ classifiers = [
 ]
 
 [project.urls]
-Repository = "https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper"
+Repository = "https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper"

{sws_spark_dissemination_helper-0.0.144 → sws_spark_dissemination_helper-0.0.146}/src/sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -26,6 +26,7 @@ class SWSEasyIcebergSparkHelper:
         dataset_tables: DatasetTables = None,
         keep_history: bool = False,
         write_csv: bool = True,
+        source_tag: str | None = None,
     ) -> None:
         self.spark: SparkSession = spark
         self.dataset_details: dict = dataset_details
@@ -37,6 +38,7 @@ class SWSEasyIcebergSparkHelper:
         self.iceberg_tables: IcebergTables = iceberg_tables
         self.keep_history: bool = keep_history
         self.write_csv: bool = write_csv
+        self.source_tag: str | None = source_tag
 
         if dataset_details is not None:
             (
@@ -69,6 +71,7 @@ class SWSEasyIcebergSparkHelper:
                 self.df_obs_coord,
                 self.df_metadata,
                 self.df_meta_elem,
+                self.df_tag_observation,
             ) = self.raw_data
 
             (
@@ -78,10 +81,11 @@ class SWSEasyIcebergSparkHelper:
                 self.df_meta_elem_type,
                 self.df_language,
                 self.df_unit_of_measure,
+                self.df_dataset,
                 self.dfs_dimension,
             ) = self.raw_reference_data
 
-            self.df_user = self.raw_operational_data
+            (self.df_user, self.df_tag) = self.raw_operational_data
 
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""
@@ -278,6 +282,69 @@ class SWSEasyIcebergSparkHelper:
 
         return df_obs_denorm
 
+    def _gen_denormalized_observation_sql_from_tag(self) -> DataFrame:
+        # ----------------
+        # Prepare dataframes for the joins
+        # ----------------
+
+        select_statement = """
+            o.id,
+            o.value,
+            u.email,
+            o.created_on,
+            o.replaced_on,
+            o.version"""
+
+        from_statement = f"""
+        FROM {self.dataset_tables.OBSERVATION.iceberg_id} o
+            INNER JOIN {self.dataset_tables.TAG_OBSERVATION.iceberg_id} to ON o.id = to.observation
+            INNER JOIN {self.dataset_tables.TAG.iceberg_id} t ON to.tag = t.id
+            INNER JOIN {self.dataset_tables.DATASET.iceberg_id} d ON t.dataset = d.id
+            LEFT JOIN {self.dataset_tables.USER.iceberg_id} u ON u.id = o.created_by
+            LEFT JOIN {self.dataset_tables.OBSERVATION_COORDINATE.iceberg_id} AS oc ON oc.id = o.observation_coordinates"""
+
+        hint_statement = ""
+
+        id_to_flag_col_mapping = {v: k for k, v in self.flag_col_to_id_mapping.items()}
+        for flag_col in self.flag_columns:
+            select_statement += f",\no.{id_to_flag_col_mapping[flag_col]} AS {flag_col}"
+
+        id_to_dim_col_mapping = {v: k for k, v in self.dim_col_to_id_mapping.items()}
+        for i, (dim_col, cl) in enumerate(
+            zip(self.dim_columns_w_time, self.dataset_tables.CODELISTS)
+        ):
+            select_statement += f",\nd{i}.code AS {dim_col}"
+            from_statement += f"\nLEFT JOIN {cl.iceberg_id} d{i} ON d{i}.id = oc.{id_to_dim_col_mapping[dim_col]}"
+            hint_statement = (
+                hint_statement + f", BROADCAST({cl.iceberg_id})"
+                if hint_statement
+                else f"BROADCAST({cl.iceberg_id})"
+            )
+
+        hint_statement = "/*+ " + hint_statement + " */"
+
+        # TODO Add tag name as a parameter
+        where_statement = (
+            f"\nWHERE t.name = '{self.source_tag}' AND d.xml_name = '{self.dataset_id}'"
+        )
+
+        final_query = (
+            "SELECT "
+            + hint_statement
+            + select_statement
+            + from_statement
+            + where_statement
+        )
+        if not self.keep_history:
+            final_query += "\n AND o.replaced_on IS NULL"
+
+        logging.info("Final query for merging observation and observation_coordinares")
+        logging.info(final_query)
+
+        df_obs_denorm = self.spark.sql(final_query)
+
+        return df_obs_denorm
+
     def _gen_denormalized_metadata(self) -> DataFrame:
         """Original query upon which the below computation is based
 
@@ -431,9 +498,23 @@ class SWSEasyIcebergSparkHelper:
             .drop("m.observation_id")
         )
 
-    def write_data_to_iceberg_and_csv(self, sql=False) -> DataFrame:
+    def _gen_denormalied_data_sql_from_tag(self) -> DataFrame:
+        return (
+            self._gen_denormalized_observation_sql_from_tag()
+            .alias("o")
+            .join(
+                self._gen_grouped_metadata_sql().alias("m"),
+                col("o.id") == col("m.observation_id"),
+                "left",
+            )
+            .drop("m.observation_id")
+        )
+
+    def write_data_to_iceberg_and_csv(self, sql=False, from_tag=False) -> DataFrame:
         if sql:
             self.df_denorm = self._gen_denormalied_data_sql()
+        elif from_tag:
+            self.df_denorm = self._gen_denormalied_data_sql_from_tag()
         else:
             self.df_denorm = self._gen_denormalied_data()
 

{sws_spark_dissemination_helper-0.0.144 → sws_spark_dissemination_helper-0.0.146}/src/sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -195,6 +195,7 @@ class SWSPostgresSparkReader:
             (dataset_tables.OBSERVATION_COORDINATE, "id", 10),
             (dataset_tables.METADATA, "id", 10),
             (dataset_tables.METADATA_ELEMENT, "metadata", 10),
+            (dataset_tables.TAG_OBSERVATION, "tag", 10),
         ]
         return self._import_tables(data_tables)
 
@@ -209,6 +210,7 @@ class SWSPostgresSparkReader:
             dataset_tables.METADATA_ELEMENT_TYPE,
             dataset_tables.LANGUAGE,
             dataset_tables.UNIT_OF_MEASURE,
+            dataset_tables.DATSET,
             *dataset_tables.CODELISTS,
         ]
         return self._import_tables(
@@ -217,17 +219,18 @@ class SWSPostgresSparkReader:
 
     def import_operational_data_tables(
         self, dataset_tables: DatasetTables
-    ) -> DataFrame:
+    ) -> List[DataFrame]:
         # Define and import operational data table without partitioning
         operational_data_tables = [
             (dataset_tables.USER, None, 1),
+            (dataset_tables.TAG, None, 1),
         ]
-        return self._import_tables(operational_data_tables)[0]
+        return self._import_tables(operational_data_tables)
 
     def import_data_reference_data_operational_data(
         self, dataset_tables: DatasetTables
     ) -> Tuple[
-        Tuple[DataFrame, DataFrame, DataFrame, DataFrame],
+        Tuple[DataFrame, DataFrame, DataFrame, DataFrame, DataFrame],
         Tuple[
             DataFrame,
             DataFrame,
@@ -235,14 +238,15 @@ class SWSPostgresSparkReader:
             DataFrame,
             DataFrame,
             DataFrame,
+            DataFrame,
             List[DataFrame],
         ],
-        DataFrame,
+        Tuple[DataFrame, DataFrame],
     ]:
         # Import and organize DataFrames into the desired output structure
         data_dfs = self.import_data_tables(dataset_tables)
         reference_data_dfs = self.import_reference_data_tables(dataset_tables)
-        operational_data_df = self.import_operational_data_tables(dataset_tables)
+        operational_data_dfs = self.import_operational_data_tables(dataset_tables)
 
         return (
             tuple(data_dfs),
@@ -250,7 +254,7 @@ class SWSPostgresSparkReader:
                 *reference_data_dfs[:6],
                 reference_data_dfs[6:],
             ),
-            operational_data_df,
+            tuple(operational_data_dfs),
         )
 
     def get_codelist_type_mapping(

{sws_spark_dissemination_helper-0.0.144 → sws_spark_dissemination_helper-0.0.146}/src/sws_spark_dissemination_helper/constants.py

@@ -172,6 +172,11 @@ class DatasetTables:
             iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_metadata_element",
             schema="id BIGINT, metadata INT, metadata_element_type INT, value STRING",
         )
+        self.TAG_OBSERVATION = self.__SWSTable(
+            postgres_id=f"{self.__dataset_id}.tag_observation",
+            iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_tag_observation",
+            schema="tag BIGINT, observation INT",
+        )
 
         # Reference data
         self.CODELISTS = [
@@ -203,18 +208,21 @@ class DatasetTables:
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.metadata_element_type",
         schema="id INT, metadata_type INT, code STRING, description STRING, mandatory BOOLEAN, repeatable BOOLEAN, private BOOLEAN",
     )
-
     LANGUAGE = __SWSTable(
         postgres_id="reference_data.language",
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.language",
         schema="id INT, country_code STRING, description STRING",
     )
-
     UNIT_OF_MEASURE = __SWSTable(
         postgres_id="reference_data.unit_of_measure",
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.unit_of_measure",
         schema="id INT, code STRING, sdmx_code STRING, metric BOOLEAN, description STRING, symbol STRING, base_unit STRING, multiplier DECIMAL",
     )
+    DATASET = __SWSTable(
+        postgres_id="reference_data.dataset",
+        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.dataset",
+        schema="id INT, xml_name STRING",
+    )
 
     # Operational data
     USER = __SWSTable(
@@ -222,6 +230,11 @@ class DatasetTables:
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.user",
         schema="id INT, username STRING, preferences INT, email STRING, active BOOLEAN, settings STRING",
     )
+    TAG = __SWSTable(
+        postgres_id="operational_data.tag",
+        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.tag",
+        schema="id INT, name STRING, reference_date DATE, dataset INT, type STRING, released_ON DATE, released_by INT, properties STRING",
+    )
 
 
 class IcebergTable:

sws_spark_dissemination_helper-0.0.144/old_requirements.txt

@@ -1,23 +0,0 @@
-annotated-types==0.7.0
-boto3==1.36.18
-botocore==1.36.18
-certifi==2025.1.31
-charset-normalizer==3.4.1
-idna==3.10
-jmespath==1.0.1
-numpy==2.0.2
-pandas==2.2.3
-py4j==0.10.9.7
-pydantic==2.10.6
-pydantic_core==2.27.2
-pyspark==3.5.4
-python-dateutil==2.9.0.post0
-python-dotenv==0.19.2
-pytz==2025.1
-requests==2.32.3
-s3transfer==0.11.2
-six==1.17.0
-sws_api_client==1.4.4
-typing_extensions==4.12.2
-tzdata==2025.1
-urllib3==1.26.20

sws_spark_dissemination_helper-0.0.144/requirements.txt

@@ -1,23 +0,0 @@
-annotated-types==0.7.0
-boto3==1.36.18
-botocore==1.36.18
-certifi==2025.1.31
-charset-normalizer==3.4.1
-idna==3.10
-jmespath==1.0.1
-numpy==2.0.2
-pandas==2.2.3
-py4j==0.10.9.7
-pydantic==2.10.6
-pydantic_core==2.27.2
-pyspark==3.5.4
-python-dateutil==2.9.0.post0
-python-dotenv==0.19.2
-pytz==2025.1
-requests==2.32.3
-s3transfer==0.11.2
-six==1.17.0
-sws_api_client==1.5.0
-typing_extensions==4.12.2
-tzdata==2025.1
-urllib3==1.26.20