maisaedu-poormans-dms 1.1.46__tar.gz → 1.1.48__tar.gz

{maisaedu-poormans-dms-1.1.46 → maisaedu-poormans-dms-1.1.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.46
+Version: 1.1.48
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.46 → maisaedu-poormans-dms-1.1.48}/maisaedu_poormans_dms/redshift_migration/Reader.py

@@ -11,7 +11,6 @@ from .Types import (
     PREFECT,
     S3,
 )
-from maisaedu_utilities_prefect.secrets import get_cipher_key
 from .Services.ExtractionOperation import ExtractionOperation
 from .Services.AdapterSourceTarget import AdapterSourceTarget
 from .Models.ExtractionOperation import ExtractionOperation as ExtractionOperationModel
@@ -22,19 +21,14 @@ class Reader:
         self.struct = struct
         self.s3_credentials = s3_credentials
         self.migrator_redshift_connector = migrator_redshift_connector
-        self.__set_cipher_key()
-
-    def __set_cipher_key(self):
-        if self.migrator_redshift_connector.env == PROD:
-            self.cipher_key = None
-        else:
-            self.cipher_key = get_cipher_key(self.migrator_redshift_connector.env)
 
     def get_incremental_statement(self):
         if (
-            self.struct.source_incremental_column is not None
-            and self.struct.target_incremental_column is not None
-            and (self.load_option is None)
+            (
+                self.struct.source_incremental_column is not None
+                and self.struct.target_incremental_column is not None
+                and (self.load_option is None) 
+            ) or (self.load_option == INCREMENTAL)
         ):
             sql = f"""
                 select max("{self.struct.target_incremental_column}") as max_value
@@ -71,7 +65,8 @@ class Reader:
 
             return sql_return
         else:
-            self.load_option = FULL
+            if (self.load_option is None):
+                self.load_option = FULL
             return ""
 
     def get_columns_source(self):
@@ -134,7 +129,7 @@ class Reader:
                 path_file = f"raw/prefect/{self.migrator_redshift_connector.env}/{self.struct.target_schema}/{self.struct.target_table}/{time}/{idx}.parquet"
                 path_file_tmp = f"raw/tmp/{self.migrator_redshift_connector.env}/{self.struct.target_schema}/{self.struct.target_table}/{time}/{idx}.csv"
 
-                adapter = AdapterSourceTarget(self.struct, self.cipher_key)
+                adapter = AdapterSourceTarget(self.struct)
                 chunk_df_s3 = chunk_df.copy()
 
                 chunk_df_s3 = adapter.transform_data(chunk_df_s3, target_save=S3)

{maisaedu-poormans-dms-1.1.46 → maisaedu-poormans-dms-1.1.48}/maisaedu_poormans_dms/redshift_migration/Services/AdapterSourceTarget.py

@@ -19,9 +19,8 @@ from ..Types import (
 
 
 class AdapterSourceTarget:
-    def __init__(self, struct, cipher_key=None):
+    def __init__(self, struct):
         self.struct = struct
-        self.cipher_key = cipher_key
 
     def convert_types(self, df):
         for c in self.struct.columns:
@@ -65,27 +64,6 @@ class AdapterSourceTarget:
 
         return df
 
-
-    def __transform_sensitive_data(self, df, struct_column):
-        if self.cipher_key is None:
-            return df
-        else:
-
-            def cipher(x):
-                if x is not None and x != "":
-                    last_letters = x[-3:]
-                    text = x + self.cipher_key
-                    hash_obj = hashlib.sha256()
-                    hash_obj.update(text.encode("utf-8"))
-                    result = hash_obj.hexdigest()
-                    return result + last_letters
-
-            df[struct_column["source_name"]] = df[struct_column["source_name"]].apply(
-                cipher
-            )
-
-            return df
-
     def transform_data(self, df, target_save=REDSHIFT):
         if target_save == REDSHIFT:
             for c in self.struct.columns:
@@ -94,8 +72,6 @@ class AdapterSourceTarget:
                 else:
                     if c["target_type"] == SUPER:
                         df = self.__transform_super_redshift(df, c)
-                    elif c["is_sensitive_data"] is True:
-                        df = self.__transform_sensitive_data(df, c)
         elif target_save == S3:
             for c in self.struct.columns:
                 if c["source_type"] == JSON or c["source_type"] == JSONB:

{maisaedu-poormans-dms-1.1.46 → maisaedu-poormans-dms-1.1.48}/maisaedu_poormans_dms/redshift_migration/Writer/GenericWriter.py

@@ -1,4 +1,3 @@
-from maisaedu_utilities_prefect.secrets import get_cipher_key
 from ..Types import PROD
 
 
@@ -8,13 +7,6 @@ class GenericWriter:
         self.struct = struct
         self.migrator_redshift_connector = migrator_redshift_connector
         self.logger = logger
-        self.__set_cipher_key()
-
-    def __set_cipher_key(self):
-        if self.migrator_redshift_connector.env == PROD:
-            self.cipher_key = None
-        else:
-            self.cipher_key = get_cipher_key(self.env)
 
     def set_temp_target_relation(self):
         self.temp_target_relation = (

{maisaedu-poormans-dms-1.1.46 → maisaedu-poormans-dms-1.1.48}/maisaedu_poormans_dms/redshift_migration/Writer/WriterCDC.py

@@ -15,7 +15,7 @@ class WriterCDC(GenericWriter, WriterInterface):
     def save_data_on_tmp_s3(self, bucket, df, path_file, file_name):
         def save_data_on_tmp(op, df, path_file):
             df = self.get_op_rows(df, op=op)
-            adapter = AdapterSourceTarget(self.struct, self.cipher_key)
+            adapter = AdapterSourceTarget(self.struct)
             df = adapter.convert_types(df)
             df = adapter.transform_data(df)
             df = adapter.equalize_number_columns(df)

{maisaedu-poormans-dms-1.1.46 → maisaedu-poormans-dms-1.1.48}/maisaedu_poormans_dms.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.46
+Version: 1.1.48
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.46 → maisaedu-poormans-dms-1.1.48}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="maisaedu-poormans-dms",
-    version="1.1.46",
+    version="1.1.48",
     description="A library for making database migration tasks, for +A Education",
     license="MIT License",
     author="A+ Educação",