logos-sdk 0.0.25.dev27__tar.gz → 0.0.25.dev28__tar.gz

{logos_sdk-0.0.25.dev27 → logos_sdk-0.0.25.dev28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: logos-sdk
-Version: 0.0.25.dev27
+Version: 0.0.25.dev28
 Summary: SDK for Logos platform
 Home-page: https://bitbucket.org/databy/logos-sdk-pip/src/master/
 Author: Databy.io

{logos_sdk-0.0.25.dev27 → logos_sdk-0.0.25.dev28}/logos_sdk/big_query/BigQuery.py

@@ -2,7 +2,7 @@ import os
 from datetime import datetime
 
 import google.auth.exceptions
-from typing import List, Dict, Union, Optional
+from typing import List, Dict, Union, Optional, Sequence
 
 from google.api_core.exceptions import NotFound
 from google.cloud.logging import Client as LoggerClient
@@ -218,3 +218,53 @@ class BigQuery:
             errors = self._service.insert_rows(bq_table, records)
             if errors:
                 raise BigQueryException(errors)
+
+    def upsert_into_table(
+            self,
+            dataset_id: str,
+            target_table: str,
+            records: List[Dict],
+            schema_columns: List[Dict],
+            key_columns: Sequence[str],
+    ) -> None:
+        if not records:
+            return
+        if not key_columns:
+            return
+
+        bq_table = self.check_table_exists(dataset_id, target_table)
+        if bq_table is None:
+            self.create_table(dataset_id, target_table, schema_columns)
+        table_id = f"{self.project_id}.{dataset_id}.{target_table}"
+        cols = [f["name"] for f in schema_columns]
+
+        staging_table_name = f"_staging_{target_table}"
+        staging_table_id = f"{self.project_id}.{dataset_id}.{staging_table_name}"
+        self.create_table(dataset_id, staging_table_name, schema_columns)
+
+        self.insert_into_table(dataset_id, staging_table_name, records)
+
+        q = lambda c: f"{c}"
+
+        on_clause = " AND ".join(f"T.{q(k)} = S.{q(k)}" for k in key_columns)
+        update_cols = [c for c in cols if c not in set(key_columns)]
+        update_set = ",\n    ".join(f"{q(c)} = S.{q(c)}" for c in update_cols)
+
+        insert_cols = ", ".join(q(c) for c in cols)
+        insert_vals = ", ".join(f"S.{q(c)}" for c in cols)
+
+        merge_sql = f"""
+            MERGE `{table_id}` T
+            USING `{staging_table_id}` S
+            ON {on_clause}
+            WHEN MATCHED THEN
+              UPDATE SET
+                {update_set}
+            WHEN NOT MATCHED THEN
+              INSERT ({insert_cols})
+              VALUES ({insert_vals})
+        """
+        job = self._service.query(merge_sql)
+        job.result()
+
+        self.delete_table(dataset_id, staging_table_name)

{logos_sdk-0.0.25.dev27 → logos_sdk-0.0.25.dev28}/logos_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: logos-sdk
-Version: 0.0.25.dev27
+Version: 0.0.25.dev28
 Summary: SDK for Logos platform
 Home-page: https://bitbucket.org/databy/logos-sdk-pip/src/master/
 Author: Databy.io