acryl-datahub 1.2.0.8rc4__py3-none-any.whl → 1.2.0.9rc1__py3-none-any.whl

{acryl_datahub-1.2.0.8rc4.dist-info → acryl_datahub-1.2.0.9rc1.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.2.0.8rc4.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.2.0.9rc1.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=uu8Uaqh56VQOGHwbtlo282g9v9lvEV1QwtbdKz41OI0,323
+datahub/_version.py,sha256=_H3eOOLcAfSj64l30V-SpArSKBd8L7socBTn4sC-zB0,323
 datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -150,7 +150,7 @@ datahub/ingestion/api/pipeline_run_listener.py,sha256=5uBP__LbMQxJ2utlf07cIzQINq
 datahub/ingestion/api/registry.py,sha256=LbdZr89465Lj7ptQRVB4vI1JR1igWABvQFj9-WX63bI,7454
 datahub/ingestion/api/report.py,sha256=1w63Y2yN49IaDLZaIvXEjRU3yVb_9t3wzymSI-fumZM,18959
 datahub/ingestion/api/report_helpers.py,sha256=WbUC1kQeaKqIagGV3XzfPmPs7slAT1mfNY4og2BH2A8,994
-datahub/ingestion/api/sink.py,sha256=nfal7nsYY1AT2WQRjqO48uAHitpjax7TsRVzYXnqbeM,4918
+datahub/ingestion/api/sink.py,sha256=bureB3_sFXNISCM4yZSqhxMHW-ctDkAQqA0lJgQhJQ4,6047
 datahub/ingestion/api/source.py,sha256=JASs7WygVB6g-tcwtchaftzv3lNtlVM31lEa242pn44,21853
 datahub/ingestion/api/source_helpers.py,sha256=XT9y5HgfVeF52jrX39vlLn1SdXpLVyT2Su8oGNsddYo,21148
 datahub/ingestion/api/transform.py,sha256=X0GpjMJzYkLuZx8MTWxH50cWGm9rGsnn3k188mmC8J8,582
@@ -188,21 +188,21 @@ datahub/ingestion/graph/entity_versioning.py,sha256=nrcNz0Qm6kpE6oTu_mrYUQDx14KP
 datahub/ingestion/graph/filters.py,sha256=OfjKhuNRHHLvhHk6Tfwd2IbMLPbbIq4VUyHaSpcDvKk,8664
 datahub/ingestion/graph/links.py,sha256=UwWSdx-j0dPttfJOjfTf4ZmlO7iIsRz5p3nIsqGVHUA,2169
 datahub/ingestion/reporting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py,sha256=6P8pFLHF6R8d5iU_L_9wWKPj9gITxrqAemJlQGd-ZCs,10025
+datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py,sha256=w-OvKG4Xm6s91czFxkhtawskMOReHyso7WjPlGGY1Rw,10073
 datahub/ingestion/reporting/file_reporter.py,sha256=tiWukmMxHrTQI3rOAumsq6lRlw8T6spqpS6XBDYnrZU,1640
 datahub/ingestion/reporting/reporting_provider_registry.py,sha256=jTYSh3T4sensjnHQfPLiIcbA2dG8w0px9ghChAJjGdU,310
 datahub/ingestion/run/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/run/connection.py,sha256=mngNzr5aRLUDa5Izqxa0xkdDEqEqcDuacWSKIlkdvPc,1483
-datahub/ingestion/run/pipeline.py,sha256=WO9K4yUihmoL6crtilO_cpRAeE6yVa1jd2tzf6spb-A,32541
+datahub/ingestion/run/pipeline.py,sha256=bQf-aeuj41FepLAgCZikCgfso6PIcU1PT7KatEgyOnc,32602
 datahub/ingestion/run/pipeline_config.py,sha256=joG1j9OlwJhb8zqv4TY6_FSzOaKOx6xsBu255A5lP8g,4101
 datahub/ingestion/run/sink_callback.py,sha256=xZAzaKkR0dcQP838pMJWsA52oaQXV5BiqXkpxEpJ_9U,2856
 datahub/ingestion/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/sink/blackhole.py,sha256=-jYcWo4i8q7312bCIoHrGr7nT9JdPvA7c4jvSc7YOgY,557
 datahub/ingestion/sink/console.py,sha256=TZfhA0Ec2eNCrMH7RRy2JOdUE-U-hkoIQrPm1CmKLQs,591
-datahub/ingestion/sink/datahub_kafka.py,sha256=_cjuXu5I6G0zJ2UK7hMbaKjMPZXeIwRMgm7CVeTiNtc,2578
+datahub/ingestion/sink/datahub_kafka.py,sha256=bRBTmvXK8mqNwnI08q846rJCNfwq33xmkm5LwTCbz58,2602
 datahub/ingestion/sink/datahub_lite.py,sha256=7u2aWm7ENLshKHl-PkjJg6Mrw4bWs8sTfKIBz4mm8Ak,1879
-datahub/ingestion/sink/datahub_rest.py,sha256=xw4JIaXOyFTLN3_wYOd3-dKv2ZwrcSxY_oXbeWpsXhs,13189
-datahub/ingestion/sink/file.py,sha256=SxXJPJpkIGoaqRjCcSmj2ZE3xE4rLlBABBGwpTj5LWI,3271
+datahub/ingestion/sink/datahub_rest.py,sha256=Esh5bzvXVpfSlYsrmB90fuFByyYmwst-ZigolqTQtnk,13333
+datahub/ingestion/sink/file.py,sha256=YA6fdnvMLiLlWlPM4LJ4m6BK-BRXVtBeqhGInvD9lzw,3295
 datahub/ingestion/sink/sink_registry.py,sha256=JRBWx8qEYg0ubSTyhqwgSWctgxwyp6fva9GoN2LwBao,490
 datahub/ingestion/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/confluent_schema_registry.py,sha256=WednrFENtANY7bWvrmMKoxEfFK9lnrMDLB0C-hXdJDQ,18808
@@ -333,7 +333,7 @@ datahub/ingestion/source/excel/report.py,sha256=oEkeI8J6is7zB9iz4RqASu_-Q5xl36lA
 datahub/ingestion/source/excel/source.py,sha256=w_vOz4UD7BcXBBDKoo81_6-QFeOPITuXqkfjIMHCQj4,23827
 datahub/ingestion/source/excel/util.py,sha256=YYmadYuCiT-4_MfQM0YSE7wuDcE0k8o2KrlOKM9Z6eI,406
 datahub/ingestion/source/fivetran/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/fivetran/config.py,sha256=A_ZFZ1PAfBeSY2k7wU-ixCFg1FOO7rgPGTd2wxKqbEg,9032
+datahub/ingestion/source/fivetran/config.py,sha256=MkeYBdxDJWH3QRhMIZZ_sfKLC2vgswasi-qlniQ6P8g,9075
 datahub/ingestion/source/fivetran/data_classes.py,sha256=ecdUJH5BEze0yv-uFpKWPNaNmV1gORDA2XMFk0zhcBw,595
 datahub/ingestion/source/fivetran/fivetran.py,sha256=s8wcECtmuugUoZ0Zdthq0SIPpTLvziZXuhhUX9bJ5N4,14492
 datahub/ingestion/source/fivetran/fivetran_log_api.py,sha256=PNzuykiiFTU8FhBIfUbW6udURZpz_35aq7rfffbpIfA,13010
@@ -411,7 +411,7 @@ datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=3i3SdBp267cZRszhm
 datahub/ingestion/source/mock_data/datahub_mock_data_report.py,sha256=sV_H7JgcuVbrpIBqtGse_BBigMdqP32ZXuanpeXmwVI,331
 datahub/ingestion/source/mock_data/table_naming_helper.py,sha256=zJtEBSJGDvVr-kiKjK7LbHAifK3sfE786M3yO--Bn2o,3493
 datahub/ingestion/source/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/neo4j/neo4j_source.py,sha256=JqKCwxBJfOrC8SF7CmDG0cseWxHk_7E2v4Diw3Q0-WM,14181
+datahub/ingestion/source/neo4j/neo4j_source.py,sha256=lxPVGL6bIZVbPtMRVcktZfuCosSnhNZH_z9GKbdQbfo,11914
 datahub/ingestion/source/powerbi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/powerbi/config.py,sha256=KJmg8f0wNuhLAGFwVhVCGVqaOrXuLTbHGQNOW8LnlSM,26467
 datahub/ingestion/source/powerbi/dataplatform_instance_resolver.py,sha256=-njW1kJOy-LY5JFwJLhVQ0bMBj9NQz5TZhQqsSi_KsM,2285
@@ -523,7 +523,7 @@ datahub/ingestion/source/sql/sql_config.py,sha256=u3nGZYYl1WtaxfNsDU5bglgZ5Jq3Fx
 datahub/ingestion/source/sql/sql_generic.py,sha256=9AERvkK8kdJUeDOzCYJDb93xdv6Z4DGho0NfeHj5Uyg,2740
 datahub/ingestion/source/sql/sql_generic_profiler.py,sha256=Zr39j4SI1fPTx1JdopVJyBslFnyp3lZCeb1th9eEB5c,11723
 datahub/ingestion/source/sql/sql_report.py,sha256=gw-OPHSExp_b6DRjvwqE1U6BpkwekxGrsvNMGYSGDio,2671
-datahub/ingestion/source/sql/sql_types.py,sha256=TZSuK29Y5TPFIjzEXbwTbbkSQAS6Dnzuq8mCbJa5fK4,15119
+datahub/ingestion/source/sql/sql_types.py,sha256=AVeBBXw8aKB1_jw6Wtg58miu-YUfN_-7ZcXwSF-ESgA,16021
 datahub/ingestion/source/sql/sql_utils.py,sha256=q-Bsk6WxlsRtrw9RXBxvqI3zuaMTC_F25T2VrCziR9I,8418
 datahub/ingestion/source/sql/sqlalchemy_data_reader.py,sha256=FvHZ4JEK3aR2DYOBZiT_ZsAy12RjTu4t_KIR_92B11k,2644
 datahub/ingestion/source/sql/sqlalchemy_uri.py,sha256=u0ZvgdJjXZdo_vl7YIQfYuuWbGwpnH6OSozI2e8ZV4I,858
@@ -635,8 +635,8 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
 datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
 datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
 datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
-datahub/metadata/_internal_schema_classes.py,sha256=AExBM83VHb4vsnAWF2nCR2x6hGg8NIsGXtAIAmYMs04,1061410
-datahub/metadata/schema.avsc,sha256=HluHCVmYg7RpOaw9xUMigEJBxlHF5WLdNcqVBKPeoOU,704514
+datahub/metadata/_internal_schema_classes.py,sha256=C_RqtHu-dWkZUJWo85PNu9gs4evsVb4vMIbdAyRUM2o,1062642
+datahub/metadata/schema.avsc,sha256=Z5bziVmjWxly-mPLjCWic2eXMCkVIYTbWqRFErSK5uk,705606
 datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
 datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
 datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
@@ -703,7 +703,7 @@ datahub/metadata/com/linkedin/pegasus2avro/policy/__init__.py,sha256=NOszM_xottE
 datahub/metadata/com/linkedin/pegasus2avro/post/__init__.py,sha256=FFiWH2BSo057ZwsnFKjpCoAMo9szvC7Vn_RzGTSLnY0,518
 datahub/metadata/com/linkedin/pegasus2avro/query/__init__.py,sha256=fRO1cOfPrSmM24FBO15eyXnh6DZn41w3kO8E5V3W8Dc,827
 datahub/metadata/com/linkedin/pegasus2avro/retention/__init__.py,sha256=Yc7z_RnA-RtPuoDgHxqL_ZT0nJ3AccCTqrXiIXwqtfo,602
-datahub/metadata/com/linkedin/pegasus2avro/role/__init__.py,sha256=4jXJV-In9gAtL05W-30uwZeQ62BuQDCwNiQGc_BFpb4,429
+datahub/metadata/com/linkedin/pegasus2avro/role/__init__.py,sha256=Vm4oUNq8l_51ywOi3vgIzqnGipgc5Ls4jUppF2DPiVk,503
 datahub/metadata/com/linkedin/pegasus2avro/schema/__init__.py,sha256=7JpzLs6S_EySr4orc6b3AvwcvELQHSahpU0DGVxnoHc,2955
 datahub/metadata/com/linkedin/pegasus2avro/schemafield/__init__.py,sha256=HTWeznycKnHBfPEGcCHXPEz83Iq9ypjNaoSfeQeDU9g,397
 datahub/metadata/com/linkedin/pegasus2avro/secret/__init__.py,sha256=qk61EqqVZF6k1Ct6t4Uo-pLb0WtM1EwJKn1XjVy9LHE,305
@@ -722,7 +722,7 @@ datahub/metadata/com/linkedin/pegasus2avro/usage/__init__.py,sha256=vhCBrCM6hTXc
 datahub/metadata/com/linkedin/pegasus2avro/versionset/__init__.py,sha256=TJ9PXc6rctPWNTY1yYW5lwIh91jD5EDoKex2WYxOLXM,406
 datahub/metadata/com/linkedin/pegasus2avro/view/__init__.py,sha256=-Le-jOqUJKv3ppwMhJHFFV3WwKTKjj1ETTkl9r7fY0o,498
 datahub/metadata/schemas/Access.avsc,sha256=gdEfWJLkvjIz-jzlceK4Dl5pBDdCHG423Ba_EYGQgUk,1562
-datahub/metadata/schemas/Actors.avsc,sha256=M76L2_Dlp7VyhVtu9__jhnh8rBNvNobtNJUfvl7bcPE,1188
+datahub/metadata/schemas/Actors.avsc,sha256=OUFUYD2YnX_75awkX63e58UDBrPLfkda-SN3as8rvBQ,2206
 datahub/metadata/schemas/ApplicationKey.avsc,sha256=WUKo-n1XiCy-09a2Q76md5VhzChj3nK7TENJJN2hW78,638
 datahub/metadata/schemas/ApplicationProperties.avsc,sha256=ZVjgnPEfi4SEvlyyIruK2sDj8xEjJeSeU3pbPINGum4,1560
 datahub/metadata/schemas/Applications.avsc,sha256=zL6yIzsS3w2463cZvTfdsTqBT7Kii8d58qc28QR_OOs,854
@@ -957,14 +957,14 @@ datahub/pydantic/compat.py,sha256=TUEo4kSEeOWVAhV6LQtst1phrpVgGtK4uif4OI5vQ2M,19
 datahub/sdk/__init__.py,sha256=66OOcFi7qlnL6q72c_yUX2mWU2HudbOdRsC5CIoDxow,1922
 datahub/sdk/_all_entities.py,sha256=eQAmD_fcEHlTShe1_nHpdvHxLDN9njk9bdLnuTrYg8M,905
 datahub/sdk/_attribution.py,sha256=0Trh8steVd27GOr9MKCZeawbuDD2_q3GIsZlCtHqEUg,1321
-datahub/sdk/_shared.py,sha256=DAfClwa-hX8VlZUdNy7TvSgudqKPobf-yFza4VYP0NU,28776
+datahub/sdk/_shared.py,sha256=uSLPjXfUl_0SPt-kWirkhE6u1CKOC67q5c4hJzxS2uo,28812
 datahub/sdk/_utils.py,sha256=oXE2BzsXE5zmSkCP3R1tObD4RHnPeH_ps83D_Dw9JaQ,1169
-datahub/sdk/chart.py,sha256=rFbEyRqgEO5HL7DyaVpR2q9zPR5y2fl52_iayhvJHQc,11756
+datahub/sdk/chart.py,sha256=_gixCcKp6kCMizWMXwNH1Ip1ZqJ05_Iu2t94dmONQFM,11774
 datahub/sdk/container.py,sha256=IjnFVGDpSFDvgHuuMb7C3VdBxhJuIMq0q6crOs5PupE,7899
-datahub/sdk/dashboard.py,sha256=WzZKUrAqsnie_rzoEhWvuv36pf60eYW4FsD_SskYLFo,15139
+datahub/sdk/dashboard.py,sha256=BtOslP85IfthdnfWzDTSXJKXsTjXP17nIHlYvK7Zqhg,15173
 datahub/sdk/dataflow.py,sha256=gdAPVVkyKvsKtsa1AwhN_LpzidG_XzV3nhtd1cjnzDA,11128
 datahub/sdk/datajob.py,sha256=5kU0txTDcn2ce3AhNry83TazPVhoYZ2rAPPNWM1_FP8,13677
-datahub/sdk/dataset.py,sha256=eABxeDJm1bxoi3kht-Ix8tbVaslFR1RInBG8AIUMk6k,31076
+datahub/sdk/dataset.py,sha256=-C4TCJAs1PFkLAgkUZEU1JOg3orm7AAIkqjw7oo_4PQ,31400
 datahub/sdk/entity.py,sha256=Q29AbpS58L4gD8ETwoNIwG-ouytz4c0MSSFi6-jLl_4,6742
 datahub/sdk/entity_client.py,sha256=NGVA2CwLqK16EgOPrPiIFodjPD6sM7eQ5E3w5Yl89cM,9428
 datahub/sdk/lineage_client.py,sha256=DRwUCyi-dDCrH5r_ktqvrryCJNPZ5Tr91y6G7dGaKBk,33754
@@ -1114,8 +1114,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.2.0.8rc4.dist-info/METADATA,sha256=w5nzJ9UysAV5hpiguzwq09DBTlLJwLyRbSj3sEEWa8Q,186651
-acryl_datahub-1.2.0.8rc4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.2.0.8rc4.dist-info/entry_points.txt,sha256=qopCAD6qrsijaZ9mTw3UlPCKsE00C3t9MbkkWow7pi4,9943
-acryl_datahub-1.2.0.8rc4.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.2.0.8rc4.dist-info/RECORD,,
+acryl_datahub-1.2.0.9rc1.dist-info/METADATA,sha256=b0i8DTQvhz0wjm4LlnQ4cAEB3Fzj4VWzmznolB87FoU,186907
+acryl_datahub-1.2.0.9rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.2.0.9rc1.dist-info/entry_points.txt,sha256=qopCAD6qrsijaZ9mTw3UlPCKsE00C3t9MbkkWow7pi4,9943
+acryl_datahub-1.2.0.9rc1.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.2.0.9rc1.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.2.0.8rc4"
+__version__ = "1.2.0.9rc1"
 
 
 def is_dev_mode() -> bool:

datahub/ingestion/api/sink.py

@@ -1,7 +1,8 @@
 import datetime
+import logging
 from abc import ABCMeta, abstractmethod
 from dataclasses import dataclass, field
-from typing import Any, Generic, Optional, Type, TypeVar, cast
+from typing import Any, Callable, Generic, List, Optional, Type, TypeVar, cast
 
 from typing_extensions import Self
 
@@ -12,6 +13,8 @@ from datahub.ingestion.api.report import Report
 from datahub.utilities.lossy_collections import LossyList
 from datahub.utilities.type_annotations import get_class_from_annotation
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class SinkReport(Report):
@@ -89,6 +92,7 @@ class Sink(Generic[SinkConfig, SinkReportType], Closeable, metaclass=ABCMeta):
     ctx: PipelineContext
     config: SinkConfig
     report: SinkReportType
+    _pre_shutdown_callbacks: List[Callable[[], None]]
 
     @classmethod
     def get_config_class(cls) -> Type[SinkConfig]:
@@ -106,6 +110,7 @@ class Sink(Generic[SinkConfig, SinkReportType], Closeable, metaclass=ABCMeta):
         self.ctx = ctx
         self.config = config
         self.report = self.get_report_class()()
+        self._pre_shutdown_callbacks = []
 
         self.__post_init__()
 
@@ -144,8 +149,28 @@ class Sink(Generic[SinkConfig, SinkReportType], Closeable, metaclass=ABCMeta):
     def get_report(self) -> SinkReportType:
         return self.report
 
+    def register_pre_shutdown_callback(self, callback: Callable[[], None]) -> None:
+        """Register a callback to be executed before the sink shuts down.
+
+        This is useful for components that need to send final reports or cleanup
+        operations before the sink's resources are released.
+        """
+        self._pre_shutdown_callbacks.append(callback)
+
     def close(self) -> None:
-        pass
+        """Close the sink and clean up resources.
+
+        This method executes any registered pre-shutdown callbacks before
+        performing the actual shutdown. Subclasses should override this method
+        to provide sink-specific cleanup logic while calling super().close()
+        to ensure callbacks are executed.
+        """
+        # Execute pre-shutdown callbacks before shutdown
+        for callback in self._pre_shutdown_callbacks:
+            try:
+                callback()
+            except Exception as e:
+                logger.warning(f"Pre-shutdown callback failed: {e}", exc_info=True)
 
     def configured(self) -> str:
         """Override this method to output a human-readable and scrubbed version of the configured sink"""

datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py

@@ -268,4 +268,4 @@ class DatahubIngestionRunSummaryProvider(PipelineRunListener):
             entity_urn=self.execution_request_input_urn,
             aspect_value=execution_result_aspect,
         )
-        self.sink.close()
+        # Note: sink.close() is handled by the pipeline's context manager

datahub/ingestion/run/pipeline.py

@@ -265,6 +265,11 @@ class Pipeline:
                 with _add_init_error_context("configure transformers"):
                     self._configure_transforms()
 
+                # Register completion callback with sink to handle final reporting
+                self.sink.register_pre_shutdown_callback(
+                    self._notify_reporters_on_ingestion_completion
+                )
+
             # If all of the initialization succeeds, we can preserve the exit stack until the pipeline run.
             # We need to use an exit stack so that if we have an exception during initialization,
             # things that were already initialized are still cleaned up.
@@ -344,8 +349,8 @@ class Pipeline:
         for reporter in self.reporters:
             try:
                 reporter.on_start(ctx=self.ctx)
-            except Exception as e:
-                logger.warning("Reporting failed on start", exc_info=e)
+            except Exception:
+                logger.warning("Reporting failed on start", exc_info=True)
 
     def _warn_old_cli_version(self) -> None:
         """
@@ -407,8 +412,8 @@ class Pipeline:
                     report=self._get_structured_report(),
                     ctx=self.ctx,
                 )
-            except Exception as e:
-                logger.warning("Reporting failed on completion", exc_info=e)
+            except Exception:
+                logger.warning("Reporting failed on completion", exc_info=True)
 
     @classmethod
     def create(
@@ -521,10 +526,10 @@ class Pipeline:
 
                     except (RuntimeError, SystemExit):
                         raise
-                    except Exception as e:
+                    except Exception:
                         logger.error(
                             "Failed to process some records. Continuing.",
-                            exc_info=e,
+                            exc_info=True,
                         )
                         # TODO: Transformer errors should be reported more loudly / as part of the pipeline report.
 
@@ -553,9 +558,9 @@ class Pipeline:
 
                 self.process_commits()
                 self.final_status = PipelineStatus.COMPLETED
-            except (SystemExit, KeyboardInterrupt) as e:
+            except (SystemExit, KeyboardInterrupt):
                 self.final_status = PipelineStatus.CANCELLED
-                logger.error("Caught error", exc_info=e)
+                logger.error("Caught error", exc_info=True)
                 raise
             except Exception as exc:
                 self.final_status = PipelineStatus.ERROR
@@ -563,9 +568,6 @@ class Pipeline:
             finally:
                 clear_global_warnings()
 
-        # This can't be in the finally part because this should happen after context manager exists
-        self._notify_reporters_on_ingestion_completion()
-
     def transform(self, records: Iterable[RecordEnvelope]) -> Iterable[RecordEnvelope]:
         """
         Transforms the given sequence of records by passing the records through the transformers

datahub/ingestion/sink/datahub_kafka.py

@@ -74,4 +74,5 @@ class DatahubKafkaSink(Sink[KafkaSinkConfig, SinkReport]):
             callback(err, f"Failed to write record: {err}")
 
     def close(self) -> None:
+        super().close()
         self.emitter.flush()

datahub/ingestion/sink/datahub_rest.py

@@ -349,6 +349,10 @@ class DatahubRestSink(Sink[DatahubRestSinkConfig, DataHubRestSinkReport]):
         )
 
     def close(self):
+        # Execute pre-shutdown callbacks first (handled by parent class)
+        super().close()
+
+        # Then perform sink-specific shutdown
         with self.report.main_thread_blocking_timer:
             self.executor.shutdown()
 

datahub/ingestion/sink/file.py

@@ -79,6 +79,7 @@ class FileSink(Sink[FileSinkConfig, SinkReport]):
             write_callback.on_success(record_envelope, {})
 
     def close(self):
+        super().close()
         self.file.write("\n]")
         self.file.close()
 

datahub/ingestion/source/fivetran/config.py

@@ -70,6 +70,7 @@ class Constant:
 
 
 KNOWN_DATA_PLATFORM_MAPPING = {
+    "google_cloud_postgresql": "postgres",
     "postgres": "postgres",
     "snowflake": "snowflake",
 }