maisaedu-poormans-dms 1.1.76__tar.gz → 1.1.77__tar.gz

{maisaedu-poormans-dms-1.1.76 → maisaedu-poormans-dms-1.1.77}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.76
+Version: 1.1.77
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.76 → maisaedu-poormans-dms-1.1.77}/maisaedu_poormans_dms/redshift_migration/Reader.py

@@ -1,4 +1,5 @@
 import io
+import threading
 import pandas as pd
 from datetime import datetime
 from .Types import (
@@ -109,6 +110,20 @@ class Reader:
 
         buffer.close()
 
+    def __process_chunk(self, chunk_df, path_file, path_file_tmp):
+        adapter = AdapterSourceTarget(self.struct)
+        chunk_df_s3 = chunk_df.copy()
+
+        chunk_df_s3 = adapter.transform_data(chunk_df_s3, target_save=S3)
+
+        self.save_on_bucket(chunk_df_s3, path_file)
+
+        chunk_df = adapter.convert_types(chunk_df)
+        chunk_df = adapter.transform_data(chunk_df)
+        chunk_df = adapter.equalize_number_columns(chunk_df)
+
+        self.save_on_bucket(chunk_df, path_file_tmp, format="csv")
+
     def save_data_to_s3(self, load_option=None):
         self.load_option = load_option
         self.migrator_redshift_connector.connect_s3()
@@ -119,6 +134,7 @@ class Reader:
         time = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
         idx = 1
         path_file = None
+        threads = []
 
         for chunk_df in pd.read_sql(
             sql,
@@ -128,20 +144,15 @@ class Reader:
             if len(chunk_df) != 0:
                 path_file = f"raw/prefect/{self.migrator_redshift_connector.env}/{self.struct.target_schema}/{self.struct.target_table}/{time}/{idx}.parquet"
                 path_file_tmp = f"raw/tmp/{self.migrator_redshift_connector.env}/{self.struct.target_schema}/{self.struct.target_table}/{time}/{idx}.csv"
-
-                adapter = AdapterSourceTarget(self.struct)
-                chunk_df_s3 = chunk_df.copy()
-
-                chunk_df_s3 = adapter.transform_data(chunk_df_s3, target_save=S3)
-
-                self.save_on_bucket(chunk_df_s3, path_file)
-
-                chunk_df = adapter.convert_types(chunk_df)
-                chunk_df = adapter.transform_data(chunk_df)
-                chunk_df = adapter.equalize_number_columns(chunk_df)
-
-                self.save_on_bucket(chunk_df, path_file_tmp, format="csv")
+                
+                thread = threading.Thread(target=self.__process_chunk, args=(chunk_df, path_file, path_file_tmp))
+                thread.start()
+                threads.append(thread)
+                
                 idx = idx + 1
+        
+        for thread in threads:
+            thread.join()
 
         self.migrator_redshift_connector.close_source()
 

{maisaedu-poormans-dms-1.1.76 → maisaedu-poormans-dms-1.1.77}/maisaedu_poormans_dms.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.76
+Version: 1.1.77
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.76 → maisaedu-poormans-dms-1.1.77}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="maisaedu-poormans-dms",
-    version="1.1.76",
+    version="1.1.77",
     description="A library for making database migration tasks, for +A Education",
     license="MIT License",
     author="A+ Educação",