maisaedu-poormans-dms 1.1.77__tar.gz → 1.1.78__tar.gz

{maisaedu-poormans-dms-1.1.77 → maisaedu-poormans-dms-1.1.78}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.77
+Version: 1.1.78
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.77 → maisaedu-poormans-dms-1.1.78}/maisaedu_poormans_dms/redshift_migration/Connector.py

@@ -3,17 +3,28 @@ import psycopg2
 from sqlalchemy import create_engine
 
 from maisaedu_utilities_prefect.dw import get_red_credentials
-from .Types import DEV, LOCAL, get_iam_role
+from .Types import DEV, LOCAL, POSTGRES, SQLSERVER, get_iam_role
 
 
 class Connector:
     def __init__(self, env, s3_credentials, source_credentials, target_credentials):
         self.source_credentials = source_credentials
+        self.__set_source_conn_type()
         self.target_credentials = target_credentials
         self.s3_credentials = s3_credentials
         self.env = env
         self.iam_role = get_iam_role(env)
 
+    def __set_source_conn_type(self):
+        if self.source_credentials is None: 
+            self.source_conn_type = None
+        else:
+            if 'type' not in self.source_credentials:
+                self.source_conn_type = POSTGRES
+            else:
+                self.source_conn_type = self.source_credentials['type']
+
+
     def connect_target(self):
         if self.target_credentials is None:
             if self.env == LOCAL:
@@ -48,10 +59,14 @@ class Connector:
         self.s3_session = session.resource("s3")
 
     def connect_source(self):
-        engine = create_engine(
-            f"postgresql+psycopg2://{self.source_credentials['user']}:{self.source_credentials['password']}@{self.source_credentials['host']}:{self.source_credentials['port']}/{self.source_credentials['database']}"
-        )
-        self.source_conn = engine.connect().execution_options(stream_results=True)
+        if self.source_conn_type == POSTGRES:
+            engine = create_engine(
+                f"postgresql+psycopg2://{self.source_credentials['user']}:{self.source_credentials['password']}@{self.source_credentials['host']}:{self.source_credentials['port']}/{self.source_credentials['database']}"
+            )
+            self.source_conn = engine.connect().execution_options(stream_results=True)
+        elif self.source_conn_type == SQLSERVER:
+            print("SQLSERVER")
+            # TODO
 
     def close_source(self):
         self.source_conn.close()

maisaedu-poormans-dms-1.1.78/maisaedu_poormans_dms/redshift_migration/Contracts/ReaderInterface.py

@@ -0,0 +1,22 @@
+from abc import ABC, abstractmethod
+
+class ReaderInterface(ABC):
+    @abstractmethod
+    def get_incremental_statement(self) -> str:
+        pass
+
+    @abstractmethod
+    def get_columns_source(self) -> str:
+        pass
+
+    @abstractmethod
+    def get_order_by_sql_statement(self) -> str:
+        pass
+
+    @abstractmethod
+    def get_limit_sql_statement(self) -> str:
+        pass
+
+    @abstractmethod
+    def get_sql_statement(self) -> str:
+        pass

{maisaedu-poormans-dms-1.1.77 → maisaedu-poormans-dms-1.1.78}/maisaedu_poormans_dms/redshift_migration/MigratorRedshift.py

@@ -1,7 +1,7 @@
 from .Connector import Connector
-from .Reader import Reader
+from .Reader import factory as reader_factory
 from .Logger import Logger
-from .Writer import factory
+from .Writer import factory as writer_factory
 from .Services.Struct import Struct
 from .Services.ExtractionOperation import ExtractionOperation
 from .Services.RelationExtraction import RelationExtraction
@@ -38,7 +38,7 @@ class MigratorRedshift:
 
         self.migrator_redshift_connector.connect_target()
 
-        self.migrator_redshift_reader = Reader(
+        self.migrator_redshift_reader = reader_factory(
             s3_credentials=s3_credentials,
             struct=struct,
             migrator_redshift_connector=self.migrator_redshift_connector,
@@ -150,7 +150,7 @@ class MigratorRedshift:
         self.__check_target_table_has_data()
         update_by_cdc = self.__check_table_will_be_updated_by_cdc(load_option)
 
-        self.migrator_redshift_writer = factory(
+        self.migrator_redshift_writer = writer_factory(
             env=self.env,
             update_by_cdc=update_by_cdc,
             struct=self.struct,

maisaedu-poormans-dms-1.1.77/maisaedu_poormans_dms/redshift_migration/Reader.py → maisaedu-poormans-dms-1.1.78/maisaedu_poormans_dms/redshift_migration/Reader/GenericReader.py

@@ -2,7 +2,7 @@ import io
 import threading
 import pandas as pd
 from datetime import datetime
-from .Types import (
+from ..Types import (
     target_type_is_numeric,
     LOCAL,
     FULL,
@@ -12,91 +12,18 @@ from .Types import (
     PREFECT,
     S3,
 )
-from .Services.ExtractionOperation import ExtractionOperation
-from .Services.AdapterSourceTarget import AdapterSourceTarget
-from .Models.ExtractionOperation import ExtractionOperation as ExtractionOperationModel
+from ..Services.ExtractionOperation import ExtractionOperation
+from ..Services.AdapterSourceTarget import AdapterSourceTarget
+from ..Models.ExtractionOperation import ExtractionOperation as ExtractionOperationModel
 
 
-class Reader:
+class GenericReader:
     def __init__(self, s3_credentials, struct, migrator_redshift_connector):
         self.struct = struct
         self.s3_credentials = s3_credentials
         self.migrator_redshift_connector = migrator_redshift_connector
 
-    def get_incremental_statement(self):
-        if (
-            (
-                self.struct.source_incremental_column is not None
-                and self.struct.target_incremental_column is not None
-                and (self.load_option is None) 
-            ) or (self.load_option == INCREMENTAL)
-        ):
-            sql = f"""
-                select max("{self.struct.target_incremental_column}") as max_value
-                from "{self.struct.target_schema}"."{self.struct.target_table}"
-            """
-
-            cursor = self.migrator_redshift_connector.target_conn.cursor()
-
-            cursor.execute(sql)
-            result = cursor.fetchall()
-
-            if len(result) == 0 or result[0][0] is None:
-                sql_return = ""
-                self.load_option = FULL
-            else:
-                for c in self.struct.columns:
-                    if c["target_name"] == self.struct.target_incremental_column:
-                        target_type = c["target_type"]
-
-                if target_type_is_numeric(target_type):
-                    sql_return = f'and "{self.struct.source_incremental_column}" > {result[0][0]}'
-                else:
-                    if (
-                        self.struct.incremental_interval_delta is None
-                        or self.struct.incremental_interval_delta == ""
-                    ):
-                        sql_return = f"and \"{self.struct.source_incremental_column}\" > '{result[0][0]}'"
-                    else:
-                        sql_return = f"and \"{self.struct.source_incremental_column}\" >= '{result[0][0]}'::timestamp - interval '{self.struct.incremental_interval_delta}'"
-
-                self.load_option = INCREMENTAL
-
-            cursor.close()
-
-            return sql_return
-        else:
-            if (self.load_option is None):
-                self.load_option = FULL
-            return ""
-
-    def get_columns_source(self):
-        return " * "
-
-    def get_order_by_sql_statement(self):
-        if self.struct.source_incremental_column is not None:
-            return f' order by "{self.struct.source_incremental_column}" asc'
-        else:
-            return ""
-
-    def get_limit_sql_statement(self):
-        if self.migrator_redshift_connector.env == LOCAL:
-            return f" limit 100"
-        else:
-            return f""
-
-    def get_sql_statement(self):
-        sql = f"""
-            select {self.get_columns_source()} 
-            from "{self.struct.source_schema}"."{self.struct.source_table}" 
-            where 1=1
-            {self.get_incremental_statement()} 
-            {self.get_order_by_sql_statement()}
-            {self.get_limit_sql_statement()}
-        """
-        return sql
-
-    def save_on_bucket(self, df, path_file, format="parquet"):
+    def __save_on_bucket(self, df, path_file, format="parquet"):
         buffer = io.BytesIO()
 
         if format == "csv":
@@ -116,13 +43,13 @@ class Reader:
 
         chunk_df_s3 = adapter.transform_data(chunk_df_s3, target_save=S3)
 
-        self.save_on_bucket(chunk_df_s3, path_file)
+        self.__save_on_bucket(chunk_df_s3, path_file)
 
         chunk_df = adapter.convert_types(chunk_df)
         chunk_df = adapter.transform_data(chunk_df)
         chunk_df = adapter.equalize_number_columns(chunk_df)
 
-        self.save_on_bucket(chunk_df, path_file_tmp, format="csv")
+        self.__save_on_bucket(chunk_df, path_file_tmp, format="csv")
 
     def save_data_to_s3(self, load_option=None):
         self.load_option = load_option

maisaedu-poormans-dms-1.1.78/maisaedu_poormans_dms/redshift_migration/Reader/PostgresReader.py

@@ -0,0 +1,82 @@
+from ..Types import (
+    target_type_is_numeric,
+    LOCAL,
+    FULL,
+    INCREMENTAL,
+)
+from .GenericReader import GenericReader
+from ..Contracts.ReaderInterface import ReaderInterface
+
+class PostgresReader(GenericReader, ReaderInterface):
+    def get_incremental_statement(self):
+        if (
+            (
+                self.struct.source_incremental_column is not None
+                and self.struct.target_incremental_column is not None
+                and (self.load_option is None) 
+            ) or (self.load_option == INCREMENTAL)
+        ):
+            sql = f"""
+                select max("{self.struct.target_incremental_column}") as max_value
+                from "{self.struct.target_schema}"."{self.struct.target_table}"
+            """
+
+            cursor = self.migrator_redshift_connector.target_conn.cursor()
+
+            cursor.execute(sql)
+            result = cursor.fetchall()
+
+            if len(result) == 0 or result[0][0] is None:
+                sql_return = ""
+                self.load_option = FULL
+            else:
+                for c in self.struct.columns:
+                    if c["target_name"] == self.struct.target_incremental_column:
+                        target_type = c["target_type"]
+
+                if target_type_is_numeric(target_type):
+                    sql_return = f'and "{self.struct.source_incremental_column}" > {result[0][0]}'
+                else:
+                    if (
+                        self.struct.incremental_interval_delta is None
+                        or self.struct.incremental_interval_delta == ""
+                    ):
+                        sql_return = f"and \"{self.struct.source_incremental_column}\" > '{result[0][0]}'"
+                    else:
+                        sql_return = f"and \"{self.struct.source_incremental_column}\" >= '{result[0][0]}'::timestamp - interval '{self.struct.incremental_interval_delta}'"
+
+                self.load_option = INCREMENTAL
+
+            cursor.close()
+
+            return sql_return
+        else:
+            if (self.load_option is None):
+                self.load_option = FULL
+            return ""
+
+    def get_columns_source(self):
+        return " * "
+
+    def get_order_by_sql_statement(self):
+        if self.struct.source_incremental_column is not None:
+            return f' order by "{self.struct.source_incremental_column}" asc'
+        else:
+            return ""
+
+    def get_limit_sql_statement(self):
+        if self.migrator_redshift_connector.env == LOCAL:
+            return f" limit 100"
+        else:
+            return f""
+
+    def get_sql_statement(self):
+        sql = f"""
+            select {self.get_columns_source()} 
+            from "{self.struct.source_schema}"."{self.struct.source_table}" 
+            where 1=1
+            {self.get_incremental_statement()} 
+            {self.get_order_by_sql_statement()}
+            {self.get_limit_sql_statement()}
+        """
+        return sql

maisaedu-poormans-dms-1.1.78/maisaedu_poormans_dms/redshift_migration/Reader/__init__.py

@@ -0,0 +1,9 @@
+from .PostgresReader import PostgresReader
+from ..Types import POSTGRES, SQLSERVER
+
+
+def factory(s3_credentials, struct, migrator_redshift_connector):
+    if migrator_redshift_connector.source_conn_type == POSTGRES:
+      return PostgresReader(s3_credentials, struct, migrator_redshift_connector)
+    elif migrator_redshift_connector.source_conn_type == SQLSERVER:
+      return None

{maisaedu-poormans-dms-1.1.77 → maisaedu-poormans-dms-1.1.78}/maisaedu_poormans_dms/redshift_migration/Types.py

@@ -16,6 +16,8 @@ SAVED_REDSHIFT = "saved-redshift"
 
 S3 = "s3"
 REDSHIFT = "redshift"
+POSTGRES = "postgres"
+SQLSERVER = "sqlserver"
 
 MAX_VARCHAR_LENGTH = 60000
 

{maisaedu-poormans-dms-1.1.77 → maisaedu-poormans-dms-1.1.78}/maisaedu_poormans_dms.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.77
+Version: 1.1.78
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.77 → maisaedu-poormans-dms-1.1.78}/maisaedu_poormans_dms.egg-info/SOURCES.txt

@@ -23,14 +23,17 @@ maisaedu_poormans_dms/postgres_migration/MigratorRow/__init__.py
 maisaedu_poormans_dms/redshift_migration/Connector.py
 maisaedu_poormans_dms/redshift_migration/Logger.py
 maisaedu_poormans_dms/redshift_migration/MigratorRedshift.py
-maisaedu_poormans_dms/redshift_migration/Reader.py
 maisaedu_poormans_dms/redshift_migration/Types.py
 maisaedu_poormans_dms/redshift_migration/__init__.py
+maisaedu_poormans_dms/redshift_migration/Contracts/ReaderInterface.py
 maisaedu_poormans_dms/redshift_migration/Contracts/WriterInterface.py
 maisaedu_poormans_dms/redshift_migration/Contracts/__init__.py
 maisaedu_poormans_dms/redshift_migration/Models/ExtractionOperation.py
 maisaedu_poormans_dms/redshift_migration/Models/Struct.py
 maisaedu_poormans_dms/redshift_migration/Models/__init__.py
+maisaedu_poormans_dms/redshift_migration/Reader/GenericReader.py
+maisaedu_poormans_dms/redshift_migration/Reader/PostgresReader.py
+maisaedu_poormans_dms/redshift_migration/Reader/__init__.py
 maisaedu_poormans_dms/redshift_migration/Services/AdapterSourceTarget.py
 maisaedu_poormans_dms/redshift_migration/Services/ExtractionOperation.py
 maisaedu_poormans_dms/redshift_migration/Services/RelationExtraction.py

{maisaedu-poormans-dms-1.1.77 → maisaedu-poormans-dms-1.1.78}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="maisaedu-poormans-dms",
-    version="1.1.77",
+    version="1.1.78",
     description="A library for making database migration tasks, for +A Education",
     license="MIT License",
     author="A+ Educação",