maisaedu-poormans-dms 1.1.6__tar.gz → 1.1.8__tar.gz

{maisaedu-poormans-dms-1.1.6 → maisaedu-poormans-dms-1.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.6
+Version: 1.1.8
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.6 → maisaedu-poormans-dms-1.1.8}/maisaedu_poormans_dms/redshift_migration/Models/Struct.py

@@ -29,13 +29,16 @@ class Struct:
         self.columns = []
         self.columns_upsert = []
 
-    def add_column(self, source_name, target_name, source_type, target_type, is_upsert):
+    def add_column(
+        self, source_name, target_name, source_type, target_type, is_upsert, is_active
+    ):
         self.columns.append(
             {
                 "source_name": source_name,
                 "target_name": target_name,
                 "source_type": source_type,
                 "target_type": target_type,
+                "is_active": is_active,
             }
         )
 

{maisaedu-poormans-dms-1.1.6 → maisaedu-poormans-dms-1.1.8}/maisaedu_poormans_dms/redshift_migration/Services/AdapterSourceTarget.py

@@ -58,18 +58,25 @@ class AdapterSourceTarget:
     def transform_data(self, df, target_save=REDSHIFT):
         if target_save == REDSHIFT:
             for c in self.struct.columns:
-                if c["target_type"] == SUPER:
-
-                    def decode_json(x):
-                        try:
-                            return json.loads(x)
-                        except Exception as e:
-                            return {"redshift_error": "Field is too long to be saved"}
-
-                    df[c["source_name"]] = df[c["source_name"]].apply(json.dumps)
-                    df[c["source_name"]] = df[c["source_name"]].str[:MAX_VARCHAR_LENGTH]
-                    df[c["source_name"]] = df[c["source_name"]].apply(decode_json)
-                    df[c["source_name"]] = df[c["source_name"]].apply(json.dumps)
+                if c["is_active"] is False:
+                    df[c["source_name"]] = ""
+                else:
+                    if c["target_type"] == SUPER:
+
+                        def decode_json(x):
+                            try:
+                                return json.loads(x)
+                            except Exception as e:
+                                return {
+                                    "redshift_error": "Field is too long to be saved"
+                                }
+
+                        df[c["source_name"]] = df[c["source_name"]].apply(json.dumps)
+                        df[c["source_name"]] = df[c["source_name"]].str[
+                            :MAX_VARCHAR_LENGTH
+                        ]
+                        df[c["source_name"]] = df[c["source_name"]].apply(decode_json)
+                        df[c["source_name"]] = df[c["source_name"]].apply(json.dumps)
         elif target_save == S3:
             for c in self.struct.columns:
                 if c["source_type"] == JSON or c["source_type"] == JSONB:

{maisaedu-poormans-dms-1.1.6 → maisaedu-poormans-dms-1.1.8}/maisaedu_poormans_dms/redshift_migration/Services/Struct.py

@@ -60,17 +60,19 @@ class Struct:
             )
             cursor.execute(
                 f"""
-                  select 
-                      source_name, 
-                      target_name, 
-                      source_type, 
-                      target_type,
-                      is_upsert
-                  from
-                      dataeng.relations_columns_extraction
-                  where
-                      relation_id = {r[0]}
-                      and is_active is true;
+                    select 
+                        source_name, 
+                        target_name, 
+                        source_type, 
+                        target_type,
+                        is_upsert,
+                        is_active
+                    from
+                        dataeng.relations_columns_extraction
+                    where
+                        relation_id = {r[0]}
+                    order by 
+                        id;
               """
             )
             columns = cursor.fetchall()
@@ -81,6 +83,7 @@ class Struct:
                     source_type=c[2],
                     target_type=c[3],
                     is_upsert=c[4],
+                    is_active=c[5],
                 )
 
             structs.append(struct_model)

{maisaedu-poormans-dms-1.1.6 → maisaedu-poormans-dms-1.1.8}/maisaedu_poormans_dms/redshift_migration/Writer/WriterCDC.py

@@ -107,9 +107,6 @@ class WriterCDC(GenericWriter, WriterInterface):
                 file_content = obj.get()["Body"].read()
 
                 df = pd.read_parquet(io.BytesIO(file_content))
-                print(len(df))
-                print("-----")
-                print(len(main_df))
 
                 main_df = pd.concat([main_df, df], ignore_index=True)
 
@@ -123,4 +120,11 @@ class WriterCDC(GenericWriter, WriterInterface):
 
         except Exception as e:
             self.delete_on_tmp_s3(bucket, path_file)
-            raise e
+            raise Exception(
+                f"""
+                    Error on save data on redshift 
+                    Cause of error: {e}
+                    Operation url: {url}
+                    You can download/check the file on s3, if the error was not direct in redshift
+                """
+            )

{maisaedu-poormans-dms-1.1.6 → maisaedu-poormans-dms-1.1.8}/maisaedu_poormans_dms.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.6
+Version: 1.1.8
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.6 → maisaedu-poormans-dms-1.1.8}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="maisaedu-poormans-dms",
-    version="1.1.6",
+    version="1.1.8",
     description="A library for making database migration tasks, for +A Education",
     license="MIT License",
     author="A+ Educação",