maisaedu-poormans-dms 1.2.61__tar.gz → 1.2.63__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/PKG-INFO +1 -1
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/sql_server_migration.py +5 -4
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms.egg-info/PKG-INFO +1 -1
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/setup.py +1 -1
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/LICENSE +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/Contracts/MigratorInterface.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/Contracts/MigratorRowInterface.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/Contracts/MigratorRowReaderInterface.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/Contracts/MigratorRowWriterInterface.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/Contracts/MigratorTableInterface.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/Contracts/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/Migrator.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/MigratorRow/MigratorRow.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/MigratorRow/Reader.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/MigratorRow/Writer.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/MigratorRow/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/MigratorTable.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/postgres_migration/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_dynamo_migration/Connector.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_dynamo_migration/DynamoReplicator.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_dynamo_migration/Writer/WriterCDC.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_dynamo_migration/Writer/WriterFull.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_dynamo_migration/Writer/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_dynamo_migration/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_dynamo_migration/models/Struct.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_dynamo_migration/models/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_dynamo_migration/services/Struct.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_dynamo_migration/services/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Connector.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Contracts/ReaderInterface.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Contracts/WriterInterface.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Contracts/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Logger.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/MigratorRedshift.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Models/ExtractionOperation.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Models/Struct.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Models/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Reader/GenericReader.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Reader/PostgresReader.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Reader/SQLServerReader.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Reader/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Services/AdapterSourceTarget.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Services/ExtractionOperation.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Services/RelationExtraction.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Services/Struct.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Services/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Types.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Writer/GenericWriter.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Writer/WriterCDC.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Writer/WriterNonCDC.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/Writer/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/redshift_migration/__init__.py +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms.egg-info/SOURCES.txt +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms.egg-info/dependency_links.txt +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms.egg-info/requires.txt +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms.egg-info/top_level.txt +0 -0
- {maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/setup.cfg +0 -0
|
@@ -184,7 +184,7 @@ def iterate_rows(
|
|
|
184
184
|
batch,
|
|
185
185
|
)
|
|
186
186
|
if show_incremental_logs:
|
|
187
|
-
|
|
187
|
+
print(total_inserted_rows)
|
|
188
188
|
return total_inserted_rows
|
|
189
189
|
|
|
190
190
|
|
|
@@ -223,7 +223,7 @@ def iterate_batches(
|
|
|
223
223
|
on_conflict,
|
|
224
224
|
show_incremental_logs,
|
|
225
225
|
)
|
|
226
|
-
|
|
226
|
+
print(f"Total inserted rows: {total_inserted_rows}")
|
|
227
227
|
|
|
228
228
|
|
|
229
229
|
def get_iter_batches(
|
|
@@ -274,6 +274,7 @@ def get_iter_batches(
|
|
|
274
274
|
while len(rows) > 0:
|
|
275
275
|
yield rows
|
|
276
276
|
rows = mssql_cursor.fetchmany(batch_size)
|
|
277
|
+
print(f"Fetched {len(rows)} rows from source")
|
|
277
278
|
|
|
278
279
|
|
|
279
280
|
def check_if_table_already_exist(dw_cursor, table):
|
|
@@ -472,7 +473,7 @@ def extract_load_data(origin_secret, info, insert_type="full", delete_type="trun
|
|
|
472
473
|
build_prefect_logger().info(f"Deleting data from {dest_table}")
|
|
473
474
|
dw_cursor.execute(f"delete from {dest_table} where 1=1;")
|
|
474
475
|
|
|
475
|
-
|
|
476
|
+
print(f"Extracting in batches of {batch_size} rows")
|
|
476
477
|
batches = get_iter_batches(
|
|
477
478
|
mssql_cursor,
|
|
478
479
|
dest_table,
|
|
@@ -499,7 +500,7 @@ def extract_load_data(origin_secret, info, insert_type="full", delete_type="trun
|
|
|
499
500
|
show_incremental_logs,
|
|
500
501
|
)
|
|
501
502
|
|
|
502
|
-
|
|
503
|
+
print(f"Commiting changes to DW")
|
|
503
504
|
dw_conn.commit()
|
|
504
505
|
|
|
505
506
|
if (is_vacuum_required):
|
|
File without changes
|
{maisaedu_poormans_dms-1.2.61 → maisaedu_poormans_dms-1.2.63}/maisaedu_poormans_dms/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|