maisaedu-poormans-dms 1.1.73__tar.gz → 1.1.74__tar.gz

{maisaedu-poormans-dms-1.1.73 → maisaedu-poormans-dms-1.1.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.73
+Version: 1.1.74
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.73 → maisaedu-poormans-dms-1.1.74}/maisaedu_poormans_dms/redshift_migration/MigratorRedshift.py

@@ -81,6 +81,7 @@ class MigratorRedshift:
             return True
         
     def __run_incremental_cdc_flow(self):
+        perfomed_operations = False
         has_more = True
         extraction_operation_service = ExtractionOperation(
             conn=self.migrator_redshift_connector.target_conn
@@ -100,7 +101,12 @@ class MigratorRedshift:
                     status=SAVED_REDSHIFT,
                 )
 
+                perfomed_operations = True
+        
+        return perfomed_operations
+
     def __run_non_cdc_flow(self, load_option):
+        perfomed_operations = False
         if load_option == FULL or self.target_table_has_data is False:
             RelationExtraction(
                 conn=self.migrator_redshift_connector.target_conn
@@ -112,6 +118,8 @@ class MigratorRedshift:
         operation = self.migrator_redshift_reader.save_data_to_s3(load_option)
         if operation is not None:
             self.migrator_redshift_writer.save_to_redshift([operation])
+            perfomed_operations = True
+
             ExtractionOperation(
                 conn=self.migrator_redshift_connector.target_conn
             ).update(
@@ -134,7 +142,8 @@ class MigratorRedshift:
                 struct=self.struct,
                 is_active=True,
             )
-    
+
+        return perfomed_operations
 
     def extract_to_redshift(self, load_option=None):
         check_if_option_is_valid(load_option)
@@ -150,9 +159,12 @@ class MigratorRedshift:
         )
 
         if update_by_cdc is True:
-            self.__run_incremental_cdc_flow()
+            perfomed_operations = self.__run_incremental_cdc_flow()
         else:
-            self.__run_non_cdc_flow(load_option)
+            perfomed_operations = self.__run_non_cdc_flow(load_option)
+
+        if perfomed_operations is True:
+            self.migrator_redshift_writer.clean_duplicate_inconsistencies()
 
         self.migrator_redshift_connector.close_target()
 

{maisaedu-poormans-dms-1.1.73 → maisaedu-poormans-dms-1.1.74}/maisaedu_poormans_dms/redshift_migration/Writer/GenericWriter.py

@@ -25,6 +25,23 @@ class GenericWriter:
             """
         )
 
+    def create_table_temp_target_relation_to_remove_dups(self):
+        pk_columns = ", ".join(self.struct.columns_upsert)
+
+        self.target_cursor.execute(
+            f"""
+                create temp table {self.temp_target_relation} as (
+                    SELECT distinct a.* FROM {self.target_relation} as a
+                    join (
+                        SELECT {pk_columns}, count(*) FROM {self.target_relation} 
+                        group by {pk_columns} having count(*) > 1
+                    ) as subselect 
+                        on 1=1 
+                        {self.create_statement_comparison_pk_columns('subselect', 'a')}
+                );
+            """
+        )
+
     def copy_data_to_target(self, url, target):
         self.target_cursor.execute(
             f"""
@@ -53,7 +70,7 @@ class GenericWriter:
                     DELETE FROM {self.target_relation}
                     USING {self.temp_target_relation}
                     WHERE 1=1
-                        {self.create_statement_delete(self.target_relation, self.temp_target_relation)}
+                        {self.create_statement_comparison_pk_columns(self.target_relation, self.temp_target_relation)}
                     ;
                 """
         )
@@ -78,7 +95,7 @@ class GenericWriter:
                 return "SERIALIZETOJSON"
         return ""
 
-    def create_statement_delete(self, target_relation, temp_target_relation):
+    def create_statement_comparison_pk_columns(self, target_relation, temp_target_relation):
         statement_upsert = ""
         for c in self.struct.columns_upsert:
             statement_upsert = (
@@ -89,6 +106,23 @@ class GenericWriter:
             )
 
         return statement_upsert
+    
+    def clean_duplicate_inconsistencies(self):
+        cursor = self.migrator_redshift_connector.target_conn.cursor()
+
+        self.target_cursor = cursor
+
+        self.migrator_redshift_connector.target_conn.autocommit = False
+
+        self.set_temp_target_relation()
+        self.set_target_relation()
+        self.create_table_temp_target_relation_to_remove_dups()
+        self.delete_using_temp_target_from_target()
+        self.insert_data_from_temp_to_target()
+        self.drop_table_temp_target_relation()
+
+        self.migrator_redshift_connector.target_conn.commit()
+        self.migrator_redshift_connector.target_conn.autocommit = True
 
     def save_to_redshift(self, operations):
         cursor = self.migrator_redshift_connector.target_conn.cursor()

{maisaedu-poormans-dms-1.1.73 → maisaedu-poormans-dms-1.1.74}/maisaedu_poormans_dms.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.73
+Version: 1.1.74
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.73 → maisaedu-poormans-dms-1.1.74}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="maisaedu-poormans-dms",
-    version="1.1.73",
+    version="1.1.74",
     description="A library for making database migration tasks, for +A Education",
     license="MIT License",
     author="A+ Educação",