logos-sdk 0.0.25.dev29__tar.gz → 0.0.25.dev30__tar.gz

{logos_sdk-0.0.25.dev29 → logos_sdk-0.0.25.dev30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: logos-sdk
-Version: 0.0.25.dev29
+Version: 0.0.25.dev30
 Summary: SDK for Logos platform
 Home-page: https://bitbucket.org/databy/logos-sdk-pip/src/master/
 Author: Databy.io

{logos_sdk-0.0.25.dev29 → logos_sdk-0.0.25.dev30}/logos_sdk/big_query/BigQuery.py

@@ -1,4 +1,5 @@
 import os
+import uuid
 from datetime import datetime
 
 import google.auth.exceptions
@@ -96,6 +97,8 @@ class BigQuery:
         schema_columns: List[Dict],
         partitioning_column_name: str,
         partition_type=bigquery.TimePartitioningType.DAY,
+        retention_days: Optional[int] = None,
+        cluster_fields: Optional[List[str]] = None,
     ):
         bq_table = self.check_table_exists(dataset_id, table_id)
         if bq_table is None:
@@ -105,6 +108,8 @@ class BigQuery:
                 schema_columns,
                 partitioning_column_name,
                 partition_type,
+                retention_days,
+                cluster_fields,
             )
 
         self._insert_into_table(bq_table, records)
@@ -127,6 +132,8 @@ class BigQuery:
         schema_columns: List[Dict],
         partitioning_column_name: str,
         partition_type=bigquery.TimePartitioningType.DAY,
+        retention_days: Optional[int] = None,
+        cluster_fields: Optional[List[str]] = None,
     ):
         table_schema = [
             bigquery.schema.SchemaField(
@@ -140,11 +147,20 @@ class BigQuery:
         try:
             sql_format = self._get_table_id_sql_format(dataset_id, table_id)
             table_object = bigquery.Table(sql_format, schema=table_schema)
-            # Set partitioning on the "report_date" column
+
+            time_partitioning_kwargs = {
+                "type_": partition_type,
+                "field": partitioning_column_name,
+            }
+            if retention_days is not None and retention_days > 0:
+                time_partitioning_kwargs["expiration_ms"] = (
+                    retention_days * 24 * 60 * 60 * 1000
+                )
             table_object.time_partitioning = bigquery.TimePartitioning(
-                type_=partition_type,
-                field=partitioning_column_name,  # The column to use for partitioning
+                **time_partitioning_kwargs
             )
+            if cluster_fields:
+                table_object.clustering_fields = cluster_fields
             return self._service.create_table(table_object)
         except google.cloud.exceptions.Conflict:
             return False
@@ -220,25 +236,22 @@ class BigQuery:
                 raise BigQueryException(errors)
 
     def upsert_into_table(
-            self,
-            dataset_id: str,
-            target_table: str,
-            records: List[Dict],
-            schema_columns: List[Dict],
-            key_columns: Sequence[str],
+        self,
+        dataset_id: str,
+        target_table: str,
+        records: List[Dict],
+        schema_columns: List[Dict],
+        key_columns: Sequence[str],
     ) -> None:
         if not records:
             return
         if not key_columns:
             return
 
-        bq_table = self.check_table_exists(dataset_id, target_table)
-        if bq_table is None:
-            self.create_table(dataset_id, target_table, schema_columns)
         table_id = f"{self.project_id}.{dataset_id}.{target_table}"
         cols = [f["name"] for f in schema_columns]
 
-        staging_table_name = f"_staging_{target_table}"
+        staging_table_name = f"_staging_{str(uuid.uuid4())}"
         staging_table_id = f"{self.project_id}.{dataset_id}.{staging_table_name}"
         self.create_table(dataset_id, staging_table_name, schema_columns)
 
@@ -248,7 +261,7 @@ class BigQuery:
 
         on_clause = " AND ".join(f"T.{q(k)} = S.{q(k)}" for k in key_columns)
         update_cols = [c for c in cols if c not in set(key_columns)]
-        update_set = ",\n    ".join(f"{q(c)} = S.{q(c)}" for c in update_cols)
+        update_set = ",\n    ".join(f"T.{q(c)} = S.{q(c)}" for c in update_cols)
 
         insert_cols = ", ".join(q(c) for c in cols)
         insert_vals = ", ".join(f"S.{q(c)}" for c in cols)

{logos_sdk-0.0.25.dev29 → logos_sdk-0.0.25.dev30}/logos_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: logos-sdk
-Version: 0.0.25.dev29
+Version: 0.0.25.dev30
 Summary: SDK for Logos platform
 Home-page: https://bitbucket.org/databy/logos-sdk-pip/src/master/
 Author: Databy.io