maisaedu-poormans-dms 1.1.80__tar.gz → 1.1.82__tar.gz

{maisaedu-poormans-dms-1.1.80 → maisaedu-poormans-dms-1.1.82}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.80
+Version: 1.1.82
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.80 → maisaedu-poormans-dms-1.1.82}/maisaedu_poormans_dms/redshift_dynamo_migration/Writer/WriterCDC.py

@@ -40,7 +40,7 @@ class WriterCDC:
                           ROW_NUMBER() OVER (PARTITION BY "payload"."dynamodb"."Keys"."partitionKey"."S", "payload"."dynamodb"."Keys"."sortKey"."S" ORDER BY "payload"."dynamodb"."ApproximateCreationDateTime" DESC) as row_num
                    FROM {self.view_name}
                    WHERE
-                        (("payload"."dynamodb"."ApproximateCreationDateTime"::bigint) / 1e6) > (coalesce(extract('epoch' from (select max(updated_at) from {self.target})), 0) - 3600)
+                        (("payload"."dynamodb"."ApproximateCreationDateTime"::bigint) / 1e6) > (coalesce(extract('epoch' from (select max(updated_at) from {self.target})), 0) - {self.struct.incremental_interval_delta})
                  ) subquery
             WHERE row_num = 1;
         """)

{maisaedu-poormans-dms-1.1.80 → maisaedu-poormans-dms-1.1.82}/maisaedu_poormans_dms/redshift_dynamo_migration/Writer/WriterFull.py

@@ -1,3 +1,4 @@
+from maisaedu_utilities_prefect.constants.redshift import get_iam_role
 from ..models.Struct import Struct
 
 TEMP_TABLE_NAME = 'tmp_full'
@@ -9,6 +10,7 @@ class WriterFull:
         self.conn = conn
         self.logger = logger
         self.struct = struct
+        self.iam_role = get_iam_role(env)
 
         self.target = f"{self.struct.target_schema}.{self.struct.target_table}"
 
@@ -25,7 +27,7 @@ class WriterFull:
             f"""
               COPY {TEMP_TABLE_NAME}
                 FROM '{self.struct.s3_url}'
-                IAM_ROLE 'arn:aws:iam::977647303146:role/PrefectAssumeRoleInsideRedshiftDEV'
+                IAM_ROLE '{self.iam_role}'
                 GZIP JSON 'auto ignorecase'
                 MAXERROR {MAX_ERRORS};
             """
@@ -42,7 +44,7 @@ class WriterFull:
                   item."partitionKey"."S"::varchar AS partition_key,
                   item."sortKey"."S"::varchar AS sort_key,
                   item,
-                  GETDATE()
+                  NULL
                 FROM
                   tmp_full;
             """

{maisaedu-poormans-dms-1.1.80 → maisaedu-poormans-dms-1.1.82}/maisaedu_poormans_dms/redshift_dynamo_migration/models/Struct.py

@@ -8,4 +8,6 @@ class Struct:
     source_table: str # DynamoDB table name
 
     target_schema: str
-    target_table: str
+    target_table: str
+
+    incremental_interval_delta: int # Interval in seconds to consider for incremental loads

{maisaedu-poormans-dms-1.1.80 → maisaedu-poormans-dms-1.1.82}/maisaedu_poormans_dms/redshift_dynamo_migration/services/Struct.py

@@ -4,6 +4,7 @@ from ..Connector import Connector
 
 SOURCE_ENGINE = "dynamo"
 AWS_REGION = "us-east-1"
+DEFAULT_INCREMENTAL_INTERVAL_DELTA = 3600
 
 class Struct:
     def __init__(self, env=None, aws_credentials=None):
@@ -31,7 +32,8 @@ class Struct:
                       source_schema,
                       source_table,
                       target_schema,
-                      target_table
+                      target_table,
+                      incremental_interval_delta
                   from 
                       dataeng.relations_extraction
                   where
@@ -63,6 +65,7 @@ class Struct:
                         source_table=r[2],
                         target_schema=r[3],
                         target_table=r[4],
+                        incremental_interval_delta=r[5] or DEFAULT_INCREMENTAL_INTERVAL_DELTA,
                         s3_url=f"{base_url}/{s3_url}data"
                     )
                 )

{maisaedu-poormans-dms-1.1.80 → maisaedu-poormans-dms-1.1.82}/maisaedu_poormans_dms.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.80
+Version: 1.1.82
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.80 → maisaedu-poormans-dms-1.1.82}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="maisaedu-poormans-dms",
-    version="1.1.80",
+    version="1.1.82",
     description="A library for making database migration tasks, for +A Education",
     license="MIT License",
     author="A+ Educação",