maisaedu-poormans-dms 1.2.6__tar.gz → 1.2.8__tar.gz

{maisaedu-poormans-dms-1.2.6 → maisaedu-poormans-dms-1.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.2.6
+Version: 1.2.8
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.2.6 → maisaedu-poormans-dms-1.2.8}/maisaedu_poormans_dms/redshift_dynamo_migration/Writer/WriterCDC.py

@@ -7,6 +7,9 @@ class WriterCDC:
         self.logger = logger
         self.struct = struct
 
+        self.partition_key = self.struct.partition_key
+        self.sort_key = self.struct.sort_key
+
         self.target = f"{self.struct.target_schema}.{self.struct.target_table}"
         self.view_name = self._get_view_name()
 
@@ -31,13 +34,13 @@ class WriterCDC:
             INSERT INTO tmp_cdc
             SELECT
               "payload"."eventName"::varchar AS event_name,
-              "payload"."dynamodb"."Keys"."partitionKey"."S"::varchar AS partition_key,
-              "payload"."dynamodb"."Keys"."sortKey"."S"::varchar AS sort_key,
+              "payload"."dynamodb"."Keys".{self.partition_key}::varchar AS partition_key,
+              "payload"."dynamodb"."Keys".{self.sort_key}::varchar AS sort_key,
               "payload"."dynamodb"."NewImage" AS item,
               TIMESTAMP 'epoch' + ("payload"."dynamodb"."ApproximateCreationDateTime"::bigint / 1e6) * INTERVAL '1 second' AS updated_at
             FROM (
                    SELECT *,
-                          ROW_NUMBER() OVER (PARTITION BY "payload"."dynamodb"."Keys"."partitionKey"."S", "payload"."dynamodb"."Keys"."sortKey"."S" ORDER BY "payload"."dynamodb"."ApproximateCreationDateTime" DESC) as row_num
+                          ROW_NUMBER() OVER (PARTITION BY "payload"."dynamodb"."Keys".{self.partition_key}, "payload"."dynamodb"."Keys".{self.sort_key} ORDER BY "payload"."dynamodb"."ApproximateCreationDateTime" DESC) as row_num
                    FROM {self.view_name}
                    WHERE
                         (("payload"."dynamodb"."ApproximateCreationDateTime"::bigint) / 1e6) > (coalesce(extract('epoch' from (select max(updated_at) from {self.target})), 0) - {self.struct.incremental_interval_delta})

{maisaedu-poormans-dms-1.2.6 → maisaedu-poormans-dms-1.2.8}/maisaedu_poormans_dms/redshift_dynamo_migration/Writer/WriterFull.py

@@ -12,6 +12,8 @@ class WriterFull:
         self.struct = struct
         self.iam_role = get_iam_role(env)
 
+        self.partition_key = self.struct.partition_key
+        self.sort_key = self.struct.sort_key
         self.target = f"{self.struct.target_schema}.{self.struct.target_table}"
 
     def create_temp_table(self):
@@ -41,8 +43,8 @@ class WriterFull:
             f"""
                 INSERT INTO {self.target} (partition_key, sort_key, item, updated_at)
                 SELECT
-                  item."partitionKey"."S"::varchar AS partition_key,
-                  item."sortKey"."S"::varchar AS sort_key,
+                  item.{self.partition_key}::varchar AS partition_key,
+                  item.{self.sort_key}::varchar AS sort_key,
                   item,
                   NULL
                 FROM

{maisaedu-poormans-dms-1.2.6 → maisaedu-poormans-dms-1.2.8}/maisaedu_poormans_dms/redshift_dynamo_migration/models/Struct.py

@@ -1,3 +1,4 @@
+from typing import Optional
 from dataclasses import dataclass
 
 @dataclass
@@ -9,5 +10,8 @@ class Struct:
 
     target_schema: str
     target_table: str
+    
+    partition_key: str
+    sort_key: Optional[str]
 
     incremental_interval_delta: int # Interval in seconds to consider for incremental loads

{maisaedu-poormans-dms-1.2.6 → maisaedu-poormans-dms-1.2.8}/maisaedu_poormans_dms/redshift_dynamo_migration/services/Struct.py

@@ -33,6 +33,8 @@ class Struct:
                       source_table,
                       target_schema,
                       target_table,
+                      source_dynamo_partition_key,
+                      source_dynamo_sort_key,
                       incremental_interval_delta
                   from 
                       dataeng.relations_extraction
@@ -65,7 +67,9 @@ class Struct:
                         source_table=r[2],
                         target_schema=r[3],
                         target_table=r[4],
-                        incremental_interval_delta=r[5] or DEFAULT_INCREMENTAL_INTERVAL_DELTA,
+                        partition_key=r[5],
+                        sort_key=r[6],
+                        incremental_interval_delta=r[7] or DEFAULT_INCREMENTAL_INTERVAL_DELTA,
                         s3_url=f"{base_url}/{s3_url}data"
                     )
                 )

{maisaedu-poormans-dms-1.2.6 → maisaedu-poormans-dms-1.2.8}/maisaedu_poormans_dms/sql_server_migration.py

@@ -357,8 +357,11 @@ def execute_sp(origin_secret, sp, variables):
             database=origin_secret["database"],
         ) as mssql_conn, mssql_conn.cursor(as_dict=True) as mssql_cursor:
             mssql_cursor.execute(query)
-            data = mssql_cursor.fetchall()
-            return data
+            while True:
+                data = mssql_cursor.fetchmany(size=1000)
+                if not data:
+                    break
+                yield data
 
 
 def extract_load_data(origin_secret, info, insert_type="full", delete_type="truncate"):

{maisaedu-poormans-dms-1.2.6 → maisaedu-poormans-dms-1.2.8}/maisaedu_poormans_dms.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.2.6
+Version: 1.2.8
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.2.6 → maisaedu-poormans-dms-1.2.8}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="maisaedu-poormans-dms",
-    version="1.2.6",
+    version="1.2.8",
     description="A library for making database migration tasks, for +A Education",
     license="MIT License",
     author="A+ Educação",