maisaedu-poormans-dms 1.0.57__tar.gz → 1.0.59__tar.gz

{maisaedu-poormans-dms-1.0.57 → maisaedu-poormans-dms-1.0.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.0.57
+Version: 1.0.59
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

maisaedu-poormans-dms-1.0.57/maisaedu_poormans_dms/redshift_migration/MigratorRedshiftConnector.py → maisaedu-poormans-dms-1.0.59/maisaedu_poormans_dms/redshift_migration/Connector.py

@@ -3,9 +3,10 @@ import psycopg2
 from sqlalchemy import create_engine
 
 from maisaedu_utilities_prefect.dw import get_red_credentials
+from .Types import DEV, PROD, LOCAL
 
 
-class MigratorRedshiftConnector:
+class Connector:
     def __init__(
         self,
         env,
@@ -18,7 +19,12 @@ class MigratorRedshiftConnector:
         self.iam_role = "arn:aws:iam::977647303146:role/service-role/AmazonRedshift-CommandsAccessRole-20220714T104138"
 
     def connect_target(self):
-        red_credentials = get_red_credentials(self.env)
+        if self.env == LOCAL:
+            env = DEV
+        else:
+            env = self.env
+
+        red_credentials = get_red_credentials(env)
         self.target_conn = psycopg2.connect(
             host=red_credentials["host"],
             database=red_credentials["database"],

maisaedu-poormans-dms-1.0.59/maisaedu_poormans_dms/redshift_migration/MigratorRedshift.py

@@ -0,0 +1,68 @@
+from .Connector import Connector
+from .Reader import Reader
+from .Writer import Writer
+from .Services.Struct import Struct
+from .Types import check_if_env_is_valid, check_if_option_is_valid
+
+
+class MigratorRedshift:
+    def __init__(
+        self,
+        env=None,
+        s3_credentials=None,
+        struct=None,
+        source_credentials=None,
+    ):
+        check_if_env_is_valid(env)
+
+        self.migrator_redshift_connector = Connector(
+            env=env,
+            s3_credentials=s3_credentials,
+            source_credentials=source_credentials,
+        )
+
+        self.migrator_redshift_reader = Reader(
+            s3_credentials=s3_credentials,
+            struct=struct,
+            migrator_redshift_connector=self.migrator_redshift_connector,
+        )
+
+        self.migrator_redshift_writer = Writer(
+            struct=struct,
+            migrator_redshift_connector=self.migrator_redshift_connector,
+        )
+
+        self.source_credentials = source_credentials
+        self.struct = struct
+        self.s3_credentials = s3_credentials
+        self.env = env
+
+    def check_target_table_has_data(self):
+        self.migrator_redshift_connector.connect_target()
+        sql = f"""
+            select count(*) from "{self.struct["target_schema"]}"."{self.struct["target_table"]}" limit 1
+        """
+        cursor = self.migrator_redshift_connector.target_conn.cursor()
+        cursor.execute(sql)
+        result = cursor.fetchall()
+        if result[0][0] == 0:
+            return False
+        else:
+            return True
+
+    def extract_to_redshift(self, load_option=None):
+        check_if_option_is_valid(load_option)
+
+        url = self.migrator_redshift_reader.save_data_to_s3(load_option)
+        return self.migrator_redshift_writer.save_to_redshift(url)
+
+    def get_structs_source_to_target(self, database, tables="all"):
+        self.migrator_redshift_connector.connect_target()
+        structs = Struct(conn=self.migrator_redshift_connector.target_conn).get(
+            database=database,
+            tables=tables,
+        )
+
+        self.migrator_redshift_connector.target_conn.close()
+
+        return structs

maisaedu-poormans-dms-1.0.57/maisaedu_poormans_dms/redshift_migration/MigratorRedshiftReader.py → maisaedu-poormans-dms-1.0.59/maisaedu_poormans_dms/redshift_migration/Reader.py

@@ -1,10 +1,11 @@
 import io
-import io
 import pandas as pd
 from datetime import datetime
+from .Types import LOCAL, FULL, INCREMENTAL, SAVED_S3
+from .Services.ExtractionOperation import ExtractionOperation
 
 
-class MigratorRedshiftReader:
+class Reader:
     def __init__(self, s3_credentials, struct, migrator_redshift_connector):
         self.struct = struct
         self.s3_credentials = s3_credentials
@@ -17,6 +18,8 @@ class MigratorRedshiftReader:
                 "text": "str",
                 "timestamp": "datetime64[ns]",
                 "super": "str",
+                "int": pd.Int64Dtype(),
+                "bigint": pd.Int64Dtype(),
             }
 
             if c["target_type"] in case_target_type.keys():
@@ -25,7 +28,6 @@ class MigratorRedshiftReader:
                 )
                 if case_target_type[c["target_type"]] == "str":
                     df[c["source_name"]].replace("None", "", inplace=True)
-
         return df
 
     def get_incremental_statement(self):
@@ -34,11 +36,12 @@ class MigratorRedshiftReader:
             and self.struct["source_incremental_column"] is not None
             and "target_incremental_column" in self.struct.keys()
             and self.struct["target_incremental_column"] is not None
+            and (self.load_option is None)
         ):
             self.migrator_redshift_connector.connect_target()
             sql = f"""
                 select max("{self.struct["target_incremental_column"]}") as max_value
-                from {self.struct["target_relation"]}
+                from "{self.struct["target_schema"]}"."{self.struct["target_table"]}"
             """
 
             cursor = self.migrator_redshift_connector.target_conn.cursor()
@@ -48,6 +51,7 @@ class MigratorRedshiftReader:
 
             if len(result) == 0 or result[0][0] is None:
                 sql_return = ""
+                self.load_option = FULL
             else:
                 for c in self.struct["columns"]:
                     if c["target_name"] == self.struct["target_incremental_column"]:
@@ -62,14 +66,22 @@ class MigratorRedshiftReader:
                 ):
                     sql_return = f'and "{self.struct["source_incremental_column"]}" > {result[0][0]}'
                 else:
-                    sql_return = f'and "{self.struct["source_incremental_column"]}" > \'{result[0][0]}\''
+                    if (
+                        self.struct["incremental_interval_delta"] is None
+                        or self.struct["incremental_interval_delta"] == ""
+                    ):
+                        sql_return = f'and "{self.struct["source_incremental_column"]}" > \'{result[0][0]}\''
+                    else:
+                        sql_return = f'and "{self.struct["source_incremental_column"]}" >= \'{result[0][0]}\'::timestamp - interval \'{self.struct["incremental_interval_delta"]}\''
+
+                self.load_option = INCREMENTAL
 
             cursor.close()
             self.migrator_redshift_connector.target_conn.close()
 
             return sql_return
-
         else:
+            self.load_option = FULL
             return ""
 
     def get_columns_source(self):
@@ -96,18 +108,26 @@ class MigratorRedshiftReader:
         else:
             return ""
 
+    def get_limit_sql_statement(self):
+        if self.migrator_redshift_connector.env == LOCAL:
+            return f" limit 100"
+        else:
+            return f""
+
     def get_sql_statement(self):
         sql = f"""
             select {self.get_columns_source()} 
-            from {self.struct["source_relation"]} 
+            from "{self.struct["source_schema"]}"."{self.struct["source_table"]}" 
             where 1=1
             {self.get_incremental_statement()} 
             {self.get_order_by_sql_statement()}
+            {self.get_limit_sql_statement()}
         """
         print(f"SQL Statement: {sql}")
         return sql
 
-    def save_data_to_s3(self):
+    def save_data_to_s3(self, load_option=None):
+        self.load_option = load_option
         self.migrator_redshift_connector.connect_s3()
         self.migrator_redshift_connector.connect_source()
 
@@ -123,7 +143,7 @@ class MigratorRedshiftReader:
             chunksize=self.struct["read_batch_size"],
         ):
             if len(chunk_df) != 0:
-                path_file = f'{self.s3_credentials["path"]}{self.struct["database"]}/{self.struct["source_relation"]}/{time}/{idx}.parquet'
+                path_file = f'raw/prefect/{self.migrator_redshift_connector.env}/{self.struct["database"]}/{self.struct["source_schema"]}/{self.struct["source_table"]}/{time}/{idx}.parquet'
                 print(f"Saving file {path_file}")
 
                 buffer = io.BytesIO()
@@ -143,4 +163,17 @@ class MigratorRedshiftReader:
         if path_file is None:
             return None
         else:
-            return f's3://{self.s3_credentials["bucket"]}/{self.s3_credentials["path"]}{self.struct["database"]}/{self.struct["source_relation"]}/{time}/'
+            url = f's3://{self.s3_credentials["bucket"]}/raw/prefect/{self.migrator_redshift_connector.env}/{self.struct["database"]}/{self.struct["source_schema"]}/{self.struct["source_table"]}/{time}/'
+
+            self.migrator_redshift_connector.connect_target()
+            ExtractionOperation(
+                conn=self.migrator_redshift_connector.target_conn,
+            ).create(
+                relation_id=self.struct["id"],
+                url=url,
+                load_option=self.load_option,
+                status=SAVED_S3,
+            )
+            self.migrator_redshift_connector.close_target()
+
+            return url

maisaedu-poormans-dms-1.0.59/maisaedu_poormans_dms/redshift_migration/Services/ExtractionOperation.py

@@ -0,0 +1,31 @@
+class ExtractionOperation:
+    def __init__(self, conn=None):
+        self.conn = conn
+
+    def create(self, relation_id, url, load_option, status):
+        cursor = self.conn.cursor()
+
+        sql = f"""
+            insert into dataeng.relations_extraction_operations
+            (relation_id, url, option, status, created_at, updated_at)
+            values
+            ({relation_id}, '{url}', '{load_option}', '{status}', 'now()', 'now()')
+        """
+
+        cursor.execute(sql)
+
+        self.conn.commit()
+        cursor.close()
+
+    def update(self, url, status):
+        cursor = self.conn.cursor()
+
+        sql = f"""
+            update dataeng.relations_extraction_operations
+            set status = '{status}', updated_at = 'now()'
+            where url = '{url}'
+        """
+
+        cursor.execute(sql)
+        self.conn.commit()
+        cursor.close()

maisaedu-poormans-dms-1.0.59/maisaedu_poormans_dms/redshift_migration/Services/Struct.py

@@ -0,0 +1,93 @@
+class Struct:
+    def __init__(self, conn=None):
+        self.conn = conn
+
+    def get(self, database, tables="all"):
+        cursor = self.conn.cursor()
+
+        statement = f" and database = '{database}'"
+
+        if tables != "all":
+            statement = f"""
+          {statement} and target_relation in ({tables})
+          """
+        else:
+            statement = f" {statement} and is_active is true"
+
+        cursor.execute(
+            f"""
+              select 
+                  id, 
+                  source_schema,
+                  source_table,
+                  source_engine,
+                  target_schema,
+                  target_table,
+                  source_incremental_column,
+                  target_incremental_column,
+                  read_batch_size,
+                  incremental_interval_delta,
+                  database,
+                  extraction_engine
+              from 
+                  dataeng.relations_extraction
+              where
+                  1=1 
+                  {statement};
+          """
+        )
+
+        structs = []
+
+        relations_extraction = cursor.fetchall()
+        for r in relations_extraction:
+            s = {
+                "id": r[0],
+                "source_schema": r[1],
+                "source_table": r[2],
+                "source_engine": r[3],
+                "target_schema": r[4],
+                "target_table": r[5],
+                "source_incremental_column": r[6],
+                "target_incremental_column": r[7],
+                "read_batch_size": r[8],
+                "incremental_interval_delta": r[9],
+                "database": r[10],
+                "extraction_engine": r[11],
+                "columns": [],
+                "columns_upsert": [],
+            }
+            cursor.execute(
+                f"""
+                  select 
+                      source_name, 
+                      target_name, 
+                      source_type, 
+                      target_type,
+                      is_upsert
+                  from
+                      dataeng.relations_colums_extraction
+                  where
+                      relation_id = {r[0]}
+                      and is_active is true
+                  order by source_order asc;
+              """
+            )
+            columns = cursor.fetchall()
+            for c in columns:
+                s["columns"].append(
+                    {
+                        "source_name": c[0],
+                        "target_name": c[1],
+                        "source_type": c[2],
+                        "target_type": c[3],
+                    }
+                )
+                if c[4] is True:
+                    s["columns_upsert"].append(c[1])
+
+            structs.append(s)
+
+        cursor.close()
+
+        return structs

maisaedu-poormans-dms-1.0.59/maisaedu_poormans_dms/redshift_migration/Types.py

@@ -0,0 +1,22 @@
+LOCAL = "local"
+DEV = "dev"
+PROD = "prod"
+
+FULL = "full"
+INCREMENTAL = "incremental"
+
+PREFECT_DMS = "prefect-dms"
+PREFECT = "prefect"
+
+SAVED_S3 = "saved-s3"
+SAVED_REDSHIFT = "saved-redshift"
+
+
+def check_if_env_is_valid(env):
+    if env not in [LOCAL, DEV, PROD]:
+        raise ValueError("env must be 'local', 'dev' or 'prod'")
+
+
+def check_if_option_is_valid(option):
+    if option not in [FULL, INCREMENTAL, None]:
+        raise ValueError("option must be 'full' or 'incremental'")

maisaedu-poormans-dms-1.0.57/maisaedu_poormans_dms/redshift_migration/MigratorRedshiftWriter.py → maisaedu-poormans-dms-1.0.59/maisaedu_poormans_dms/redshift_migration/Writer.py

@@ -1,4 +1,8 @@
-class MigratorRedshiftWriter:
+from .Types import SAVED_REDSHIFT
+from .Services.ExtractionOperation import ExtractionOperation
+
+
+class Writer:
     def __init__(self, struct, migrator_redshift_connector):
         self.struct = struct
         self.migrator_redshift_connector = migrator_redshift_connector
@@ -9,23 +13,30 @@ class MigratorRedshiftWriter:
                 return "SERIALIZETOJSON"
         return ""
 
-    def create_statement_upsert(self, temp_target_relation):
+    def create_statement_upsert(self, target_relation, temp_target_relation):
         statement_upsert = ""
         for c in self.struct["columns_upsert"]:
             statement_upsert = (
                 statement_upsert
-                + f'and {self.struct["target_relation"]}.{c} = {temp_target_relation}.{c} '
+                + f"""
+                    and {target_relation}."{c}" = {temp_target_relation}."{c}" 
+                """
             )
 
         return statement_upsert
 
     def save_data(self, is_upsert, target_cursor, path_files_to_insert):
-        temp_target_relation = f'"temp_{self.struct["target_relation"]}"'
+        temp_target_relation = (
+            f'"temp_{self.struct["target_schema"]}_{self.struct["target_table"]}"'
+        )
+        target_relation = (
+            f'"{self.struct["target_schema"]}"."{self.struct["target_table"]}"'
+        )
 
         target_cursor.execute(
             f"""
-                    CREATE TEMP TABLE {temp_target_relation} (LIKE {self.struct["target_relation"]});
-                """
+                    CREATE TEMP TABLE {temp_target_relation} (LIKE {target_relation});
+            """
         )
 
         target_cursor.execute(
@@ -42,34 +53,38 @@ class MigratorRedshiftWriter:
         if is_upsert is True:
             target_cursor.execute(
                 f"""
-                    DELETE FROM {self.struct["target_relation"]} 
+                    DELETE FROM {target_relation} 
                     USING {temp_target_relation} 
                     WHERE 1=1 
-                        {self.create_statement_upsert(temp_target_relation)}    
+                        {self.create_statement_upsert(target_relation, temp_target_relation)}    
                     ;
                 """
             )
         else:
             target_cursor.execute(
                 f"""
-                    DELETE FROM {self.struct["target_relation"]};
+                    DELETE FROM {target_relation};
                 """
             )
 
         target_cursor.execute(
             f"""
-                INSERT INTO {self.struct["target_relation"]}
+                INSERT INTO {target_relation}
                 SELECT * FROM {temp_target_relation};
             """
         )
 
         self.migrator_redshift_connector.target_conn.commit()
 
-        target_cursor.execute(f"""DROP TABLE {temp_target_relation};""")
+        target_cursor.execute(
+            f"""
+                DROP TABLE {temp_target_relation};
+            """
+        )
 
         self.migrator_redshift_connector.target_conn.commit()
 
-    def save_to_redshift(self, path_files_to_insert):
+    def save_to_redshift(self, url):
         self.migrator_redshift_connector.connect_target()
         cursor = self.migrator_redshift_connector.target_conn.cursor()
 
@@ -84,9 +99,17 @@ class MigratorRedshiftWriter:
 
         self.save_data(
             target_cursor=cursor,
-            path_files_to_insert=path_files_to_insert,
+            path_files_to_insert=url,
             is_upsert=is_upsert,
         )
 
         cursor.close()
+
+        ExtractionOperation(
+            conn=self.migrator_redshift_connector.target_conn,
+        ).update(
+            url=url,
+            status=SAVED_REDSHIFT,
+        )
+
         self.migrator_redshift_connector.target_conn.close()

{maisaedu-poormans-dms-1.0.57 → maisaedu-poormans-dms-1.0.59}/maisaedu_poormans_dms.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.0.57
+Version: 1.0.59
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.0.57 → maisaedu-poormans-dms-1.0.59}/maisaedu_poormans_dms.egg-info/SOURCES.txt

@@ -20,8 +20,12 @@ maisaedu_poormans_dms/postgres_migration/MigratorRow/MigratorRow.py
 maisaedu_poormans_dms/postgres_migration/MigratorRow/Reader.py
 maisaedu_poormans_dms/postgres_migration/MigratorRow/Writer.py
 maisaedu_poormans_dms/postgres_migration/MigratorRow/__init__.py
+maisaedu_poormans_dms/redshift_migration/Connector.py
 maisaedu_poormans_dms/redshift_migration/MigratorRedshift.py
-maisaedu_poormans_dms/redshift_migration/MigratorRedshiftConnector.py
-maisaedu_poormans_dms/redshift_migration/MigratorRedshiftReader.py
-maisaedu_poormans_dms/redshift_migration/MigratorRedshiftWriter.py
-maisaedu_poormans_dms/redshift_migration/__init__.py
+maisaedu_poormans_dms/redshift_migration/Reader.py
+maisaedu_poormans_dms/redshift_migration/Types.py
+maisaedu_poormans_dms/redshift_migration/Writer.py
+maisaedu_poormans_dms/redshift_migration/__init__.py
+maisaedu_poormans_dms/redshift_migration/Services/ExtractionOperation.py
+maisaedu_poormans_dms/redshift_migration/Services/Struct.py
+maisaedu_poormans_dms/redshift_migration/Services/__init__.py

{maisaedu-poormans-dms-1.0.57 → maisaedu-poormans-dms-1.0.59}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="maisaedu-poormans-dms",
-    version="1.0.57",
+    version="1.0.59",
     description="A library for making database migration tasks, for +A Education",
     license="MIT License",
     author="A+ Educação",

maisaedu-poormans-dms-1.0.57/maisaedu_poormans_dms/redshift_migration/MigratorRedshift.py

@@ -1,123 +0,0 @@
-from .MigratorRedshiftConnector import MigratorRedshiftConnector
-from .MigratorRedshiftReader import MigratorRedshiftReader
-from .MigratorRedshiftWriter import MigratorRedshiftWriter
-
-
-class MigratorRedshift:
-    def __init__(
-        self,
-        env=None,
-        s3_credentials=None,
-        struct=None,
-        source_credentials=None,
-    ):
-        self.migrator_redshift_connector = MigratorRedshiftConnector(
-            env=env,
-            s3_credentials=s3_credentials,
-            source_credentials=source_credentials,
-        )
-
-        self.migrator_redshift_reader = MigratorRedshiftReader(
-            s3_credentials=s3_credentials,
-            struct=struct,
-            migrator_redshift_connector=self.migrator_redshift_connector,
-        )
-
-        self.migrator_redshift_writer = MigratorRedshiftWriter(
-            struct=struct,
-            migrator_redshift_connector=self.migrator_redshift_connector,
-        )
-
-        self.source_credentials = source_credentials
-        self.struct = struct
-        self.s3_credentials = s3_credentials
-        self.env = env
-
-    def save_data_to_s3(self):
-        return self.migrator_redshift_reader.save_data_to_s3()
-
-    def save_to_redshift(self, path_files_to_insert):
-        return self.migrator_redshift_writer.save_to_redshift(path_files_to_insert)
-
-    def get_structs_source_to_target(self, database, tables="all"):
-        self.migrator_redshift_connector.connect_target()
-
-        statement = f" and database = '{database}'"
-
-        if tables != "all":
-            statement = f"""
-            {statement} and id in ({tables})
-            """
-        else:
-            statement = f" {statement} and is_active is true"
-
-        cur = self.migrator_redshift_connector.target_conn.cursor()
-        cur.execute(
-            f"""
-                select 
-                    id, 
-                    source_relation,
-                    source_engine,
-                    target_relation,
-                    source_incremental_column,
-                    target_incremental_column,
-                    read_batch_size,
-                    incremental_interval_delta,
-                    database
-                from 
-                    dataeng.relations_extraction
-                where
-                    1=1 
-                    {statement};
-            """
-        )
-
-        structs = []
-
-        relations_extraction = cur.fetchall()
-        for r in relations_extraction:
-            s = {
-                "source_relation": r[1],
-                "source_engine": r[2],
-                "target_relation": r[3],
-                "source_incremental_column": r[4],
-                "target_incremental_column": r[5],
-                "read_batch_size": r[6],
-                "incremental_interval_delta": r[7],
-                "database": r[8],
-                "columns": [],
-                "columns_upsert": [],
-            }
-            cur.execute(
-                f"""
-                    select 
-                        source_name, 
-                        target_name, 
-                        source_type, 
-                        target_type,
-                        is_upsert
-                    from
-                        dataeng.relations_colums_extraction
-                    where
-                        relation_id = {r[0]}
-                    order by source_order asc;
-                """
-            )
-            columns = cur.fetchall()
-            for c in columns:
-                s["columns"].append(
-                    {
-                        "source_name": c[0],
-                        "target_name": c[1],
-                        "source_type": c[2],
-                        "target_type": c[3],
-                    }
-                )
-                if c[4] is True:
-                    s["columns_upsert"].append(c[1])
-
-            structs.append(s)
-
-        self.migrator_redshift_connector.close_target()
-
-        return structs