maisaedu-poormans-dms 1.1.88__tar.gz → 1.1.90__tar.gz

{maisaedu-poormans-dms-1.1.88 → maisaedu-poormans-dms-1.1.90}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.88
+Version: 1.1.90
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.88 → maisaedu-poormans-dms-1.1.90}/maisaedu_poormans_dms/redshift_migration/Connector.py

@@ -65,8 +65,10 @@ class Connector:
             )
             self.source_conn = engine.connect().execution_options(stream_results=True)
         elif self.source_conn_type == SQLSERVER:
-            print("SQLSERVER")
-            # TODO
+            engine = create_engine(
+                f"mssql+pyodbc://{self.source_credentials['user']}:{self.source_credentials['password']}@{self.source_credentials['host']}:{self.source_credentials['port']}/{self.source_credentials['database']}?driver=ODBC+Driver+17+for+SQL+Server"
+            )
+            self.source_conn = engine.connect().execution_options(stream_results=True)
 
     def close_source(self):
         self.source_conn.close()

maisaedu-poormans-dms-1.1.90/maisaedu_poormans_dms/redshift_migration/Reader/SQLServerReader.py

@@ -0,0 +1,88 @@
+from datetime import datetime
+from ..Types import (
+    target_type_is_numeric,
+    LOCAL,
+    FULL,
+    INCREMENTAL,
+)
+from .GenericReader import GenericReader
+from ..Contracts.ReaderInterface import ReaderInterface
+
+class SQLServerReader(GenericReader, ReaderInterface):
+    def get_incremental_statement(self):
+        if (
+            (
+                self.struct.source_incremental_column is not None
+                and self.struct.target_incremental_column is not None
+                and (self.load_option is None) 
+            ) or (self.load_option == INCREMENTAL)
+        ):
+            sql = f"""
+                select max("{self.struct.target_incremental_column}") as max_value
+                from "{self.struct.target_schema}"."{self.struct.target_table}"
+            """
+
+            cursor = self.migrator_redshift_connector.target_conn.cursor()
+
+            cursor.execute(sql)
+            result = cursor.fetchall()
+
+            if len(result) == 0 or result[0][0] is None:
+                sql_return = ""
+                self.load_option = FULL
+            else:
+                for c in self.struct.columns:
+                    if c["target_name"] == self.struct.target_incremental_column:
+                        target_type = c["target_type"]
+
+                if target_type_is_numeric(target_type):
+                    sql_return = f'and "{self.struct.source_incremental_column}" > {result[0][0]}'
+                else:
+                    if (
+                        self.struct.incremental_interval_delta is None
+                        or self.struct.incremental_interval_delta == ""
+                    ):
+                        sql_return = f"and \"{self.struct.source_incremental_column}\" > '{result[0][0]}'"
+                    else:
+                        split = self.struct.incremental_interval_delta.split(" ")
+                        date_string = result[0][0].strftime("%Y-%m-%d %H:%M:%S.%f")
+                        date_string = date_string[:-3]
+                        sql_return = f"and \"{self.struct.source_incremental_column}\" >= DATEADD({split[1]}, -{split[0]}, '{date_string}')"
+
+                self.load_option = INCREMENTAL
+
+            cursor.close()
+
+            return sql_return
+        else:
+            if (self.load_option is None):
+                self.load_option = FULL
+            return ""
+
+    def get_columns_source(self):
+        return " * "
+
+    def get_order_by_sql_statement(self):
+        if self.struct.source_incremental_column is not None:
+            return f' order by "{self.struct.source_incremental_column}" asc'
+        else:
+            return ""
+
+    def get_limit_sql_statement(self):
+        if self.migrator_redshift_connector.env == LOCAL:
+            return f" TOP 100"
+            # return f""
+        else:
+            return f""
+
+    def get_sql_statement(self):
+        sql = f"""
+            select 
+            {self.get_limit_sql_statement()}
+            {self.get_columns_source()} 
+            from "{self.struct.source_schema}"."{self.struct.source_table}" NOLOCK
+            where 1=1
+            {self.get_incremental_statement()} 
+            {self.get_order_by_sql_statement()}
+        """
+        return sql

{maisaedu-poormans-dms-1.1.88 → maisaedu-poormans-dms-1.1.90}/maisaedu_poormans_dms/redshift_migration/Reader/__init__.py

@@ -1,4 +1,5 @@
 from .PostgresReader import PostgresReader
+from .SQLServerReader import SQLServerReader
 from ..Types import POSTGRES, SQLSERVER
 
 
@@ -6,4 +7,4 @@ def factory(s3_credentials, struct, migrator_redshift_connector):
     if migrator_redshift_connector.source_conn_type == POSTGRES:
       return PostgresReader(s3_credentials, struct, migrator_redshift_connector)
     elif migrator_redshift_connector.source_conn_type == SQLSERVER:
-      return None
+      return SQLServerReader(s3_credentials, struct, migrator_redshift_connector)

{maisaedu-poormans-dms-1.1.88 → maisaedu-poormans-dms-1.1.90}/maisaedu_poormans_dms/redshift_migration/Services/AdapterSourceTarget.py

@@ -22,6 +22,19 @@ class AdapterSourceTarget:
     def __init__(self, struct):
         self.struct = struct
 
+    def order_by_cdc(self, op, df):
+        if op == "U":
+            if "cdc_datetime" not in df.columns:
+                return df
+            else:
+                if df['cdc_datetime'].isnull().any():
+                    return df
+                else:
+                    df = df.sort_values('cdc_datetime', ascending=True)
+                    return df
+        else:
+            return df
+
     def convert_types(self, df):
         for c in self.struct.columns:
             case_target_type = {

{maisaedu-poormans-dms-1.1.88 → maisaedu-poormans-dms-1.1.90}/maisaedu_poormans_dms/redshift_migration/Writer/GenericWriter.py

@@ -33,8 +33,8 @@ class GenericWriter:
                 create temp table {self.temp_target_relation} as (
                     SELECT distinct a.* FROM {self.target_relation} as a
                     join (
-                        SELECT {pk_columns}, count(*) FROM {self.target_relation} 
-                        group by {pk_columns} having count(*) > 1
+                        SELECT "{pk_columns}", count(*) FROM {self.target_relation} 
+                        group by "{pk_columns}" having count(*) > 1
                     ) as subselect 
                         on 1=1 
                         {self.create_statement_comparison_pk_columns('subselect', 'a')}

{maisaedu-poormans-dms-1.1.88 → maisaedu-poormans-dms-1.1.90}/maisaedu_poormans_dms/redshift_migration/Writer/WriterCDC.py

@@ -17,14 +17,13 @@ class WriterCDC(GenericWriter, WriterInterface):
             df = self.get_op_rows(df, op=op)
             adapter = AdapterSourceTarget(self.struct)
             df = adapter.convert_types(df)
+            df = adapter.order_by_cdc(op, df)
             df = adapter.transform_data(df)
             df = adapter.equalize_number_columns(df)
 
             if df.empty is False:
                 if op == "U":
-                    df = df.drop_duplicates(
-                        subset=self.struct.columns_upsert, keep="last"
-                    )
+                    df = df.drop_duplicates(subset=self.struct.columns_upsert, keep="last")
 
                 buffer = io.BytesIO()
                 df.to_csv(buffer, index=False)

{maisaedu-poormans-dms-1.1.88 → maisaedu-poormans-dms-1.1.90}/maisaedu_poormans_dms.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.88
+Version: 1.1.90
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.88 → maisaedu-poormans-dms-1.1.90}/maisaedu_poormans_dms.egg-info/SOURCES.txt

@@ -43,6 +43,7 @@ maisaedu_poormans_dms/redshift_migration/Models/Struct.py
 maisaedu_poormans_dms/redshift_migration/Models/__init__.py
 maisaedu_poormans_dms/redshift_migration/Reader/GenericReader.py
 maisaedu_poormans_dms/redshift_migration/Reader/PostgresReader.py
+maisaedu_poormans_dms/redshift_migration/Reader/SQLServerReader.py
 maisaedu_poormans_dms/redshift_migration/Reader/__init__.py
 maisaedu_poormans_dms/redshift_migration/Services/AdapterSourceTarget.py
 maisaedu_poormans_dms/redshift_migration/Services/ExtractionOperation.py

{maisaedu-poormans-dms-1.1.88 → maisaedu-poormans-dms-1.1.90}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="maisaedu-poormans-dms",
-    version="1.1.88",
+    version="1.1.90",
     description="A library for making database migration tasks, for +A Education",
     license="MIT License",
     author="A+ Educação",