maisaedu-poormans-dms 1.1.79__tar.gz → 1.1.81__tar.gz

{maisaedu-poormans-dms-1.1.79 → maisaedu-poormans-dms-1.1.81}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.79
+Version: 1.1.81
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

maisaedu-poormans-dms-1.1.81/maisaedu_poormans_dms/redshift_dynamo_migration/Connector.py

@@ -0,0 +1,30 @@
+import psycopg2
+
+from maisaedu_utilities_prefect.dw import get_red_credentials
+from maisaedu_utilities_prefect.constants.redshift import get_iam_role
+
+class Connector:
+    def __init__(self, env):
+        self.env = env
+        self.iam_role = get_iam_role(env)
+        self.redshift = None
+
+        self.has_connected = False
+
+    def connect(self):
+        red_credentials = get_red_credentials(self.env)
+        self.redshift = psycopg2.connect(
+            host=red_credentials["host"],
+            database=red_credentials["database"],
+            user=red_credentials["user"],
+            password=red_credentials["password"],
+            port=red_credentials["port"],
+        )
+
+        self.has_connected = True
+
+    def close(self):
+        if not self.has_connected:
+            return
+        
+        self.redshift.close()

maisaedu-poormans-dms-1.1.81/maisaedu_poormans_dms/redshift_dynamo_migration/DynamoReplicator.py

@@ -0,0 +1,63 @@
+from maisaedu_poormans_dms.redshift_migration.Types import (
+    FULL,
+    check_if_env_is_valid,
+    check_if_option_is_valid
+)
+
+from .Writer.WriterFull import WriterFull
+from .Writer.WriterCDC import WriterCDC
+from .Connector import Connector
+
+class DynamoReplicator:
+    def __init__(
+        self,
+        env=None,
+        struct=None,
+        logger=None
+    ):
+        check_if_env_is_valid(env)
+
+        self.conn = Connector(env)
+        self.env = env
+        self.struct = struct
+        self.logger = logger
+
+    def __connect(self):
+        if self.conn.redshift is None:
+            self.conn.connect()
+    
+    def __run_full_flow(self, struct):
+        writer = WriterFull(
+            env=self.env,
+            conn=self.conn,
+            logger=self.logger,
+            struct=struct
+        )
+        writer.save()
+
+    def __run_incremental_flow(self, struct):
+        writer = WriterCDC(
+            env=self.env,
+            conn=self.conn,
+            logger=self.logger,
+            struct=struct
+        )
+        writer.save()
+
+    def process_records(self, load_option, struct):
+        if struct is None:
+            raise ValueError("struct must be provided")
+        
+        self.__connect()
+        check_if_option_is_valid(load_option)
+
+        self.logger.info(f"Processing records for {struct}")
+        if load_option == FULL:
+            self.__run_full_flow(struct)
+        else:
+            self.__run_incremental_flow(struct)
+
+        self.conn.redshift.commit()
+
+    def __del__(self):
+        self.conn.close()

maisaedu-poormans-dms-1.1.81/maisaedu_poormans_dms/redshift_dynamo_migration/Writer/WriterCDC.py

@@ -0,0 +1,102 @@
+from ..models.Struct import Struct
+
+class WriterCDC:
+    def __init__(self, env, conn, logger, struct: Struct):
+        self.env = env
+        self.conn = conn
+        self.logger = logger
+        self.struct = struct
+
+        self.target = f"{self.struct.target_schema}.{self.struct.target_table}"
+        self.view_name = self._get_view_name()
+
+    def _get_view_name(self):
+        table_name = self.struct.target_table.replace("dynamo_", "")
+        return f"{self.struct.source_schema}.{self.struct.database}_{table_name}_stream"
+
+    def create_temp_table(self):
+        self.cursor.execute(f"""
+            CREATE TEMP TABLE tmp_cdc (
+              event_name varchar(255),
+              partition_key varchar(2048),
+              sort_key varchar(2048),
+              item super,
+              updated_at timestamp
+            );
+            REFRESH MATERIALIZED VIEW {self.view_name};
+        """)
+
+    def insert_data_to_temp(self):
+        self.cursor.execute(f"""
+            INSERT INTO tmp_cdc
+            SELECT
+              "payload"."eventName"::varchar AS event_name,
+              "payload"."dynamodb"."Keys"."partitionKey"."S"::varchar AS partition_key,
+              "payload"."dynamodb"."Keys"."sortKey"."S"::varchar AS sort_key,
+              "payload"."dynamodb"."NewImage" AS item,
+              TIMESTAMP 'epoch' + ("payload"."dynamodb"."ApproximateCreationDateTime"::bigint / 1e6) * INTERVAL '1 second' AS updated_at
+            FROM (
+                   SELECT *,
+                          ROW_NUMBER() OVER (PARTITION BY "payload"."dynamodb"."Keys"."partitionKey"."S", "payload"."dynamodb"."Keys"."sortKey"."S" ORDER BY "payload"."dynamodb"."ApproximateCreationDateTime" DESC) as row_num
+                   FROM {self.view_name}
+                   WHERE
+                        (("payload"."dynamodb"."ApproximateCreationDateTime"::bigint) / 1e6) > (coalesce(extract('epoch' from (select max(updated_at) from {self.target})), 0) - {self.struct.incremental_interval_delta})
+                 ) subquery
+            WHERE row_num = 1;
+        """)
+
+    def delete_records(self):
+        self.cursor.execute(f"""
+            CREATE TEMP TABLE tmp_delete_ids (partition_key varchar(2048), sort_key varchar(2048));
+            INSERT INTO tmp_delete_ids
+            SELECT partition_key, sort_key
+            FROM tmp_cdc
+            WHERE event_name = 'REMOVE';
+
+            DELETE FROM {self.target}
+            USING tmp_delete_ids tmp
+            WHERE
+                {self.target}.partition_key = tmp.partition_key
+                AND {self.target}.sort_key = tmp.sort_key
+        """)
+
+    def insert_records(self):
+        self.cursor.execute(f"""
+            INSERT INTO {self.target} (partition_key, sort_key, item, updated_at)
+            SELECT
+              tmp.partition_key,
+              tmp.sort_key,
+              tmp.item,
+              tmp.updated_at
+            FROM tmp_cdc tmp
+            LEFT JOIN {self.target} t ON tmp.partition_key = t.partition_key AND tmp.sort_key = t.sort_key
+            WHERE tmp.event_name IN ('INSERT', 'MODIFY') AND t.id IS NULL
+        """)
+
+    def update_records(self):
+        self.cursor.execute(f"""
+            UPDATE {self.target} t
+            SET
+                item = tmp.item,
+                updated_at = tmp.updated_at
+            FROM tmp_cdc tmp
+            WHERE t.partition_key = tmp.partition_key AND t.sort_key = tmp.sort_key
+                AND tmp.event_name = 'MODIFY';
+        """)
+
+    def cleanup(self):
+        self.cursor.execute(f"""
+            DROP TABLE tmp_delete_ids;
+            DROP TABLE tmp_cdc;
+        """)
+
+    def save(self):
+        with self.conn.redshift.cursor() as cursor:
+            self.cursor = cursor
+
+            self.create_temp_table()
+            self.insert_data_to_temp()
+            self.delete_records()
+            self.insert_records()
+            self.update_records()
+            self.cleanup()

maisaedu-poormans-dms-1.1.81/maisaedu_poormans_dms/redshift_dynamo_migration/Writer/WriterFull.py

@@ -0,0 +1,62 @@
+from ..models.Struct import Struct
+
+TEMP_TABLE_NAME = 'tmp_full'
+MAX_ERRORS = 100
+
+class WriterFull:
+    def __init__(self, env, conn, logger, struct: Struct):
+        self.env = env
+        self.conn = conn
+        self.logger = logger
+        self.struct = struct
+
+        self.target = f"{self.struct.target_schema}.{self.struct.target_table}"
+
+    def create_temp_table(self):
+        self.cursor.execute(f"""
+          CREATE TEMP TABLE {TEMP_TABLE_NAME} (
+            id bigint identity(1,1) primary key,
+            item super
+          );
+        """)
+
+    def copy_data_to_target(self):
+        self.cursor.execute(
+            f"""
+              COPY {TEMP_TABLE_NAME}
+                FROM '{self.struct.s3_url}'
+                IAM_ROLE 'arn:aws:iam::977647303146:role/PrefectAssumeRoleInsideRedshiftDEV'
+                GZIP JSON 'auto ignorecase'
+                MAXERROR {MAX_ERRORS};
+            """
+        )
+
+    def truncate_target_table(self):
+        self.cursor.execute(f"TRUNCATE TABLE {self.target}")
+
+    def insert_data_from_temp_to_target(self):
+        self.cursor.execute(
+            f"""
+                INSERT INTO {self.target} (partition_key, sort_key, item, updated_at)
+                SELECT
+                  item."partitionKey"."S"::varchar AS partition_key,
+                  item."sortKey"."S"::varchar AS sort_key,
+                  item,
+                  NULL
+                FROM
+                  tmp_full;
+            """
+        )
+
+    def cleanup(self):
+        self.cursor.execute(f"DROP TABLE {TEMP_TABLE_NAME}")
+
+    def save(self):
+        with self.conn.redshift.cursor() as cursor:
+            self.cursor = cursor
+
+            self.create_temp_table()
+            self.copy_data_to_target()
+            self.truncate_target_table()
+            self.insert_data_from_temp_to_target()
+            self.cleanup()

maisaedu-poormans-dms-1.1.81/maisaedu_poormans_dms/redshift_dynamo_migration/models/Struct.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass
+
+@dataclass
+class Struct:
+    s3_url: str # S3 URL for the full load
+    database: str # Database name, used to build the table names in the queries
+    source_schema: str # Schema where the MVW is
+    source_table: str # DynamoDB table name
+
+    target_schema: str
+    target_table: str
+
+    incremental_interval_delta: int # Interval in seconds to consider for incremental loads

maisaedu-poormans-dms-1.1.81/maisaedu_poormans_dms/redshift_dynamo_migration/services/Struct.py

@@ -0,0 +1,76 @@
+import boto3
+from ..models.Struct import Struct as StructModel
+from ..Connector import Connector
+
+SOURCE_ENGINE = "dynamo"
+AWS_REGION = "us-east-1"
+DEFAULT_INCREMENTAL_INTERVAL_DELTA = 3600
+
+class Struct:
+    def __init__(self, env=None, aws_credentials=None):
+        self.conn = Connector(env)
+        self.conn.connect()
+
+        self.s3_client = boto3.client(
+            's3',
+            aws_access_key_id=aws_credentials["aws_edtech_prefect_deploy_access_key_id"],
+            aws_secret_access_key=aws_credentials["aws_edtech_prefect_deploy_secret_access_key"],
+            region_name=AWS_REGION
+        )
+
+    def get(self, database, bucket, tables="all"):
+        with self.conn.redshift.cursor() as cursor:
+            clause = ""
+            if tables != "all":
+                joined_tables = ",".join(map(lambda t: f"'{t}'", tables))
+                clause = f"and target_table in ({joined_tables})"
+
+            cursor.execute(
+                f"""
+                  select 
+                      database,
+                      source_schema,
+                      source_table,
+                      target_schema,
+                      target_table,
+                      incremental_interval_delta
+                  from 
+                      dataeng.relations_extraction
+                  where
+                      source_engine = '{SOURCE_ENGINE}'
+                      and database = '{database}'
+                      and is_active is true
+                      {clause}
+              """
+            )
+
+            relations_extraction = cursor.fetchall()
+            structs = []
+
+            for r in relations_extraction:
+                base_url = f"s3://{bucket}"
+                prefix = f"raw/dynamo/full/{r[2]}/AWSDynamoDB/"
+                s3_objects = self.s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter="/")
+
+                if "CommonPrefixes" not in s3_objects:
+                    raise ValueError(f"No data found in {prefix}")
+                
+                latest_folder = max(s3_objects["CommonPrefixes"], key=lambda x: x["Prefix"])
+                s3_url = latest_folder["Prefix"]
+
+                structs.append(
+                    StructModel(
+                        database=r[0],
+                        source_schema=r[1],
+                        source_table=r[2],
+                        target_schema=r[3],
+                        target_table=r[4],
+                        incremental_interval_delta=r[5] or DEFAULT_INCREMENTAL_INTERVAL_DELTA,
+                        s3_url=f"{base_url}/{s3_url}data"
+                    )
+                )
+
+        return structs
+    
+    def __del__(self):
+        self.conn.close()

{maisaedu-poormans-dms-1.1.79 → maisaedu-poormans-dms-1.1.81}/maisaedu_poormans_dms.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maisaedu-poormans-dms
-Version: 1.1.79
+Version: 1.1.81
 Summary: A library for making database migration tasks, for +A Education
 Home-page: UNKNOWN
 Author: A+ Educação

{maisaedu-poormans-dms-1.1.79 → maisaedu-poormans-dms-1.1.81}/maisaedu_poormans_dms.egg-info/SOURCES.txt

@@ -20,6 +20,16 @@ maisaedu_poormans_dms/postgres_migration/MigratorRow/MigratorRow.py
 maisaedu_poormans_dms/postgres_migration/MigratorRow/Reader.py
 maisaedu_poormans_dms/postgres_migration/MigratorRow/Writer.py
 maisaedu_poormans_dms/postgres_migration/MigratorRow/__init__.py
+maisaedu_poormans_dms/redshift_dynamo_migration/Connector.py
+maisaedu_poormans_dms/redshift_dynamo_migration/DynamoReplicator.py
+maisaedu_poormans_dms/redshift_dynamo_migration/__init__.py
+maisaedu_poormans_dms/redshift_dynamo_migration/Writer/WriterCDC.py
+maisaedu_poormans_dms/redshift_dynamo_migration/Writer/WriterFull.py
+maisaedu_poormans_dms/redshift_dynamo_migration/Writer/__init__.py
+maisaedu_poormans_dms/redshift_dynamo_migration/models/Struct.py
+maisaedu_poormans_dms/redshift_dynamo_migration/models/__init__.py
+maisaedu_poormans_dms/redshift_dynamo_migration/services/Struct.py
+maisaedu_poormans_dms/redshift_dynamo_migration/services/__init__.py
 maisaedu_poormans_dms/redshift_migration/Connector.py
 maisaedu_poormans_dms/redshift_migration/Logger.py
 maisaedu_poormans_dms/redshift_migration/MigratorRedshift.py

{maisaedu-poormans-dms-1.1.79 → maisaedu-poormans-dms-1.1.81}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="maisaedu-poormans-dms",
-    version="1.1.79",
+    version="1.1.81",
     description="A library for making database migration tasks, for +A Education",
     license="MIT License",
     author="A+ Educação",