acryl-datahub 1.0.0.2rc3__py3-none-any.whl → 1.0.0.2rc5__py3-none-any.whl

{acryl_datahub-1.0.0.2rc3.dist-info → acryl_datahub-1.0.0.2rc5.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.0.0.2rc3.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.0.0.2rc5.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=zEgu-5D0-B51vKVharilRcGBBRiiNaqIfj9tNzWK38c,323
+datahub/_version.py,sha256=TEFaI0SngUMeKPXQwQz9bnZDzmSywu7Y6e6m6k--k00,323
 datahub/entrypoints.py,sha256=2TYgHhs3sCxJlojIHjqfxzt3_ImPwPzq4vBtsUuMqu4,8885
 datahub/errors.py,sha256=BzKdcmYseHOt36zfjJXc17WNutFhp9Y23cU_L6cIkxc,612
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -41,21 +41,21 @@ datahub/api/entities/datacontract/datacontract.py,sha256=Vi6u9snSiacDlq2iofHiUGl
 datahub/api/entities/datacontract/freshness_assertion.py,sha256=TpKqNeUzeGH9wrYqelYVuZpolEm7iYXj1nymwWu8-_c,2700
 datahub/api/entities/datacontract/schema_assertion.py,sha256=24VQ0lm4LxWbpIrho2R6Atc80yqo76Vu7AN2LZiMOzc,2418
 datahub/api/entities/datajob/__init__.py,sha256=suzCs4cLphQ64oDG7meXsJVp6ya9_Glzqamoq_ti5GA,222
-datahub/api/entities/datajob/dataflow.py,sha256=Ajs6XP7-WMK4OwSm0JdFuGwggZhx-l5S-fwor68BBYk,6954
+datahub/api/entities/datajob/dataflow.py,sha256=VBaBoO9it2cqoJarXqGQhSGJrecHVqGhdCPbpzqeSz0,7505
 datahub/api/entities/datajob/datajob.py,sha256=tAxdxy8djDKD2THKIYEAsNi1Ly_eRpmBjT6fASCnUqg,7944
 datahub/api/entities/dataprocess/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/api/entities/dataprocess/dataprocess_instance.py,sha256=PIiGXTQLPWl_1rY43N1H-ygDXyzQrSQl0FOLhZ9-YX4,18853
+datahub/api/entities/dataprocess/dataprocess_instance.py,sha256=IhY-rcXs-r8EatwW1_sJA79GxQyg9lhILBR66IrnLkY,19120
 datahub/api/entities/dataproduct/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/api/entities/dataproduct/dataproduct.py,sha256=148TmItxDDyGNzfZdL8aDreSEtyAw79IN8N8oSmNOPE,21461
 datahub/api/entities/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/api/entities/dataset/dataset.py,sha256=6o6Giv7dHxiH0_xQQKv68eQyDaRcJK9e17FO568ULpk,49652
+datahub/api/entities/dataset/dataset.py,sha256=UX9UW5z9realL7lPxAhaKKigpVAqtxAfhqS6vY2eDnU,49638
 datahub/api/entities/forms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/api/entities/forms/forms.py,sha256=17GLVVrunUj6hWS7CADhNPrT4gV6il905Ny_Y_5S5Qc,15889
 datahub/api/entities/forms/forms_graphql_constants.py,sha256=DKpnKlMKTjmnyrCTvp63V4LX4THGTAMq3ep8THrSGP4,537
 datahub/api/entities/platformresource/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/api/entities/platformresource/platform_resource.py,sha256=pVAjv6NoH746Mfvdak7ji0eqlEcEeV-Ji7M5gyNXmds,10603
 datahub/api/entities/structuredproperties/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/api/entities/structuredproperties/structuredproperties.py,sha256=ViRgAsD7aFT_-vIylFNW4D9PSl46UWw0MrxXq3bEh0k,8542
+datahub/api/entities/structuredproperties/structuredproperties.py,sha256=FU50bB1bgGDbitdzK9gHxWMj83KUuxK80mhpK1KmGZQ,8535
 datahub/api/graphql/__init__.py,sha256=5yl0dJxO-2d_QuykdJrDIbWq4ja9bo0t2dAEh89JOog,142
 datahub/api/graphql/assertion.py,sha256=ponITypRQ8vE8kiqRNpvdoniNJzi4aeBK97UvkF0VhA,2818
 datahub/api/graphql/base.py,sha256=9q637r6v-RGOd8Mk8HW2g0vt9zpqFexsQ5R6TPEHVbs,1614
@@ -75,7 +75,7 @@ datahub/cli/iceberg_cli.py,sha256=-XT3wpkr8b-HFMafYk7lSon3Lys6XjTQA8U1b698ByM,23
 datahub/cli/ingest_cli.py,sha256=Welutg0LOjuEiBOnQdNTnEZFDyKZiiWxqyQK-Go8dL8,20540
 datahub/cli/json_file.py,sha256=nWo-VVthaaW4Do1eUqgrzk0fShb29MjiKXvZVOTq76c,943
 datahub/cli/lite_cli.py,sha256=XKMejSuYUToKBvgN3YmmnxjRcaG5WPw23gJuQK8pgRc,13099
-datahub/cli/migrate.py,sha256=3orGfLNsdh1Q7gkPaCaf2bBWM5b3Ih4fGFw3poe0wiA,17937
+datahub/cli/migrate.py,sha256=mVOIayhfQRGZeOofoabgC1hjgD-QzlpSKX8PaCoshCY,17895
 datahub/cli/migration_utils.py,sha256=0qHo_9eSR4buyV_K_tdcHSLBufKphBWwwwT1iK_I4S8,9382
 datahub/cli/put_cli.py,sha256=4ol9aLdidX1VXjVxMG2tkfEMPyjLpgOk2pfl0Gvb8iU,3841
 datahub/cli/quickstart_versioning.py,sha256=MyWvw92s4b84wIEizjSUZjoMClwLbhpgMdHeDav-x2o,5713
@@ -94,7 +94,7 @@ datahub/cli/specific/structuredproperties_cli.py,sha256=Rb06XJKxqda5RSUo188B90Wn
 datahub/cli/specific/user_cli.py,sha256=jGAokb1NRu8obs6P2g4OL2NQdFgpUBa9De55TBBtun0,1897
 datahub/configuration/__init__.py,sha256=5TN3a7CWNsLRHpdj-sv2bxKWF2IslvJwE6EpNMFrIS4,123
 datahub/configuration/_config_enum.py,sha256=ul2hr5gMmdLvBINicFkMNMi1ApmnmZSwNdUYYted5nk,1447
-datahub/configuration/common.py,sha256=GKRNgRCBsFUmZb6huD6DPNYUKNRxUbcHfFazHLte7wk,10626
+datahub/configuration/common.py,sha256=2AUcZD8c4W8gdopt07UcQYPLl0DR-O7Spb-LUqguJXc,10619
 datahub/configuration/config_loader.py,sha256=hRzPFxkz-w9IqkpSa5vwCzSra1p49DyfeJNeyqGa8-4,6827
 datahub/configuration/connection_resolver.py,sha256=n4-6MwMiOEDgTouxO0SMjTILKVhJPo6-naE6FuR5qMs,1516
 datahub/configuration/datetimes.py,sha256=nayNc0mmlVKH6oVv9ud6C1dDUiZPGabW-YZxvrkosPg,2870
@@ -134,7 +134,7 @@ datahub/ingestion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 datahub/ingestion/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/api/closeable.py,sha256=k12AT--s4GDtZ-po_rVm5QKgvGIDteeRPByZPIOfecA,599
 datahub/ingestion/api/committable.py,sha256=4S6GuBzvX2vb1A8P506NbspOKfZ1621sBG8t0lvRb8o,886
-datahub/ingestion/api/common.py,sha256=nJVL8YdvokYFajOjmVpSNlLbZJ5iVOFS4KJDlGtJ_jc,2735
+datahub/ingestion/api/common.py,sha256=26lXJiM4YfdnVH1xfe2bpZNp2VKCdJcJ8ynK7rhh0FY,3029
 datahub/ingestion/api/decorators.py,sha256=-0Kl9G_8DhmSP9Q0ryrWB3a-Odce8WTpvGETJOqRdgw,4015
 datahub/ingestion/api/global_context.py,sha256=OdSJg4a_RKE52nu8MSiEkK2UqRRDhDTyOleHEAzPKho,575
 datahub/ingestion/api/incremental_lineage_helper.py,sha256=7a6FTJ_uz4EEJS1vPtbYB2KvNlcZB3py28_FKxmRiSk,5993
@@ -145,8 +145,8 @@ datahub/ingestion/api/registry.py,sha256=LbdZr89465Lj7ptQRVB4vI1JR1igWABvQFj9-WX
 datahub/ingestion/api/report.py,sha256=eM_TWWz6iJNd-c_S2_4eg2qKLGYP8vSROb_TMiCwBhY,4644
 datahub/ingestion/api/report_helpers.py,sha256=WbUC1kQeaKqIagGV3XzfPmPs7slAT1mfNY4og2BH2A8,994
 datahub/ingestion/api/sink.py,sha256=nfal7nsYY1AT2WQRjqO48uAHitpjax7TsRVzYXnqbeM,4918
-datahub/ingestion/api/source.py,sha256=HrQahSEBeapMDnW8S6wSEyNLLE9RCs2R6eUrVaibuuc,19349
-datahub/ingestion/api/source_helpers.py,sha256=OhgBLdpUIuqF_gl4uV8Y2csp-z97zzXeFj2I5aUypCI,20158
+datahub/ingestion/api/source.py,sha256=2h7Tx2As0gu5-6d7PiRuJ8myr_y3MRx2YYgH735Jj18,19494
+datahub/ingestion/api/source_helpers.py,sha256=tzFl2sP-sFOZWX0O0eEQK-0tJcJ5vMs2bccpKRKBIFY,21081
 datahub/ingestion/api/transform.py,sha256=X0GpjMJzYkLuZx8MTWxH50cWGm9rGsnn3k188mmC8J8,582
 datahub/ingestion/api/workunit.py,sha256=e8n8RfSjHZZm2R4ShNH0UuMtUkMjyqqM2j2t7oL74lo,6327
 datahub/ingestion/api/auto_work_units/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -182,7 +182,7 @@ datahub/ingestion/reporting/file_reporter.py,sha256=tiWukmMxHrTQI3rOAumsq6lRlw8T
 datahub/ingestion/reporting/reporting_provider_registry.py,sha256=jTYSh3T4sensjnHQfPLiIcbA2dG8w0px9ghChAJjGdU,310
 datahub/ingestion/run/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/run/connection.py,sha256=mngNzr5aRLUDa5Izqxa0xkdDEqEqcDuacWSKIlkdvPc,1483
-datahub/ingestion/run/pipeline.py,sha256=pzd6LV1weecVgjMpWyM4gRzJ7FU7fhn0E5Vui8bw4fE,29938
+datahub/ingestion/run/pipeline.py,sha256=-74g3vIuecTuXsNBgspOSEfnpO-NqT-yg-ZqkL6OgP0,29692
 datahub/ingestion/run/pipeline_config.py,sha256=EDwqlid4h_qyqyeTRCEqb1RiFA4py_T-Poz1eIKmzT4,4101
 datahub/ingestion/run/sink_callback.py,sha256=xZAzaKkR0dcQP838pMJWsA52oaQXV5BiqXkpxEpJ_9U,2856
 datahub/ingestion/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -205,12 +205,12 @@ datahub/ingestion/source/ge_profiling_config.py,sha256=FlWfXoVoayabVXNMB9qETEU0G
 datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suabri1yV5geaAPjzg2eORDc,2559
 datahub/ingestion/source/ldap.py,sha256=CNr3foofIpoCXu_GGqfcajlQE2qkHr5isYwVcDutdkk,18695
 datahub/ingestion/source/metabase.py,sha256=j8DRV2GvisezidL1JZ5HJLF_hdFdtvaoyDoEdEyh0Ks,32603
-datahub/ingestion/source/mlflow.py,sha256=6uN1fjyubs9rjAsdtkSRMKf7h3_89UvFfWDqkgdvPdY,32422
-datahub/ingestion/source/mode.py,sha256=20vWflnRIMWqK8q2Mt2PorMBLnzRAA4bMjcWEyqhTio,64506
+datahub/ingestion/source/mlflow.py,sha256=2K5D95HLUhnx8jP54hK8aKNB0nPaCgXaUgO2PdL6Gto,32539
+datahub/ingestion/source/mode.py,sha256=_FKZutF-59w0pYhko6HSVL3yjjYNd329-2DJmyfDqF8,64492
 datahub/ingestion/source/mongodb.py,sha256=2C2Cxn8DXL53IbNiywIuKt8UT_EMcPg9f8su-OPSNGU,21237
-datahub/ingestion/source/nifi.py,sha256=w5TPnqPmpotvzSsJROi6nUiHWPUVC6u1g0CzXIE6FNs,56903
-datahub/ingestion/source/openapi.py,sha256=rCy7fOgBWVoao4i7n-a8BWwBLUalZojLuKI_r7f8hF4,18690
-datahub/ingestion/source/openapi_parser.py,sha256=K3Z4aLXBQX8eR7tvk9iZakOjghjezxIeHCCBhrXfyro,14868
+datahub/ingestion/source/nifi.py,sha256=D1gBXxdpLuUQ0eurwofIR_SGg1rHGhwk3qxsWI1PT9c,56882
+datahub/ingestion/source/openapi.py,sha256=zx976zstg6M2KoTz_iKKgU9VETDeX2rnw6BofiHXbDc,18669
+datahub/ingestion/source/openapi_parser.py,sha256=HsG5W9eBTvgoPC78QZ4jSlX-UnQagQ4uXXdOm_relmM,14812
 datahub/ingestion/source/preset.py,sha256=bbh0ZWiAZMy2zuJDmaRY07_OuGJ9tdtKjwvIxqbY5II,3964
 datahub/ingestion/source/pulsar.py,sha256=u5F8QnCLJsht5-7XCiUTsnfhCPIpKVB_l32CgMCU-As,20187
 datahub/ingestion/source/redash.py,sha256=YxjSad-X_wPmxYH8dJmFz_VCFhiLTCTSlK99WdvcYiA,30653
@@ -238,7 +238,7 @@ datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py,sha256=3eYub
 datahub/ingestion/source/aws/sagemaker_processors/job_classes.py,sha256=CfJkzjZU2uvZvw7qvmxfNgeWI1EvgHFY-7bn5Ih71no,9154
 datahub/ingestion/source/aws/sagemaker_processors/jobs.py,sha256=aHgQ4QMufdWAA62TNBoEPT3YSQKXg39IJ2-6MZXs8sw,32915
 datahub/ingestion/source/aws/sagemaker_processors/lineage.py,sha256=TcT8xmVuQDQdlRKYaCRXbFjMcW5brfSWFBNpoRdPx1o,9789
-datahub/ingestion/source/aws/sagemaker_processors/models.py,sha256=e0H3Z_YW7O-e34Mv_PVtUgCZAtkyGl08h4Q7l3atbUc,19845
+datahub/ingestion/source/aws/sagemaker_processors/models.py,sha256=N9FtnOznwwToc9J8FMehxgIO9hQRsY5LwJqi2s6tTRc,19817
 datahub/ingestion/source/azure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/azure/abs_folder_utils.py,sha256=7skXus-4fSIoKpqCeU-GG0ch1oF2SJSYDZ1JMB_Onso,7605
 datahub/ingestion/source/azure/abs_utils.py,sha256=KdAlCK-PMrn35kFHxz5vrsjajyx2PD5GRgoBKdoRvcg,2075
@@ -257,7 +257,7 @@ datahub/ingestion/source/bigquery_v2/bigquery_schema.py,sha256=DHw5Z_rxj_fR09p7S
 datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py,sha256=Fq9tAVSvYepwweiZuJB5mbT0Y6EzFOEZWtdL4Zafd4E,50999
 datahub/ingestion/source/bigquery_v2/bigquery_test_connection.py,sha256=cATxwi5IPzj3BldRRAVcLqzSFmmYEPvqa7U0RFJbaAc,7645
 datahub/ingestion/source/bigquery_v2/common.py,sha256=Cxjf1a8ibkL_YRQeS0BqsjlyMgFJpaZ3iq_d7e8T8MQ,4030
-datahub/ingestion/source/bigquery_v2/lineage.py,sha256=Dkig1SEfPxw6zZDeSulUYnqsu4WGCVPXypGPEUVriyU,44907
+datahub/ingestion/source/bigquery_v2/lineage.py,sha256=jju14mJbAUMA_K3j2yq-TdZV202cjd5rBAsDPJGEVno,44900
 datahub/ingestion/source/bigquery_v2/profiler.py,sha256=8-yAoq8sX0E6VIwr75YbM8wITRNhGfxgte9BCeGNkMM,10681
 datahub/ingestion/source/bigquery_v2/queries.py,sha256=c1BpeQP8p8y-FOhmiQkkY2IqGrEqrXtARDCYQ2xhXvo,20145
 datahub/ingestion/source/bigquery_v2/queries_extractor.py,sha256=_5cAXVU8b8T_nAPDsvN2JRd2dmM1t1J1mRylfKiPen4,19530
@@ -307,12 +307,12 @@ datahub/ingestion/source/dremio/dremio_source.py,sha256=_52Z0ifntbhYNwlrMs6jZ59C
 datahub/ingestion/source/dremio/dremio_sql_queries.py,sha256=W0rcXawlwJOHNYr5o73rilMijtFOO3cVkn6pY-JLc6o,8186
 datahub/ingestion/source/dynamodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/dynamodb/data_reader.py,sha256=vC77KpcP8LJN0g8wsPRDVw4sebv0ZWIP3tJkEIHaomA,3120
-datahub/ingestion/source/dynamodb/dynamodb.py,sha256=vPDp0Au4qpkKJVDXdOb6lwUtNKDTeDKP0yHIcQxXK5k,22632
+datahub/ingestion/source/dynamodb/dynamodb.py,sha256=Z1cwomrAgTfBfSdVtG-Tfbn_--2STY6wTsm6VoACVA4,22625
 datahub/ingestion/source/fivetran/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/fivetran/config.py,sha256=BP3KRfAQ6H5qyEeJNu9vNfZNwLoyj4Tl2kXiLVR5DNM,9027
 datahub/ingestion/source/fivetran/data_classes.py,sha256=ecdUJH5BEze0yv-uFpKWPNaNmV1gORDA2XMFk0zhcBw,595
-datahub/ingestion/source/fivetran/fivetran.py,sha256=CVJhW7_os5BTRlzaUX2KOK6CkAVJ0mWQtgTnE6F3fhE,13760
-datahub/ingestion/source/fivetran/fivetran_log_api.py,sha256=EAak3hJpe75WZSgz6wP_CyAT5Cian2N4a-lb8x1NKHk,12776
+datahub/ingestion/source/fivetran/fivetran.py,sha256=avP54ePLFVpkKVv8tr6mzC7dniTmZbKn13LP0-Ohj9k,13821
+datahub/ingestion/source/fivetran/fivetran_log_api.py,sha256=DBS5qDRsXvJzR-x6euaR5Ss3yrV76lswcweI1XwmABw,12769
 datahub/ingestion/source/fivetran/fivetran_query.py,sha256=vLrTj7e-0NxZ2U4bWTB57pih42WirqPlUvwtIRfStlQ,5275
 datahub/ingestion/source/gc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/gc/datahub_gc.py,sha256=EXO-Stj6gGMLTSTbSBC-C3_zpjpQtFN9pAMWR95ma0I,12830
@@ -332,22 +332,22 @@ datahub/ingestion/source/hex/constants.py,sha256=8hUTMWyG5keTNfXoLu_Dh413Hw_mGGJ
 datahub/ingestion/source/hex/hex.py,sha256=PIRl8fPkKtlHV7cqR4H8RKVYdTLgEFXHFzc3QAqJLhE,12733
 datahub/ingestion/source/hex/mapper.py,sha256=N3mTlEcrOmhv9ia1dnHGFgFJD2ddyTtU3H5IUbb-UxU,13344
 datahub/ingestion/source/hex/model.py,sha256=S9bUhfFcjzuio2dBS6HzSyRVPiSJvRvMQ0qyVrjV5-E,1766
-datahub/ingestion/source/hex/query_fetcher.py,sha256=5r065vL7XohcgZ_fj-1h6o8cxrPin37IeYsC99GU6LA,12287
+datahub/ingestion/source/hex/query_fetcher.py,sha256=ZaRrja05mK0VlIvpsFi-8-EBoJr0GSLbUxBUjycibIU,12505
 datahub/ingestion/source/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/iceberg/iceberg.py,sha256=PhLLXWgBdfZ3hL7LgLvDr6aTK-QKmiZCFNz5jD-mxZM,30773
+datahub/ingestion/source/iceberg/iceberg.py,sha256=s69XzCGD5oV_hqTyvzCt5eLKZVEzVIJo_DiAEDk3p6A,34759
 datahub/ingestion/source/iceberg/iceberg_common.py,sha256=VGosqYPmn_j6GETSnDHZ8Ay1BVOedmx2x5LHxw16I3A,12278
 datahub/ingestion/source/iceberg/iceberg_profiler.py,sha256=9iwp2vpQTi4OMbIKoDZV5lAdvjMR0ls6Llpck9grJIE,9875
 datahub/ingestion/source/identity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/identity/azure_ad.py,sha256=9Hrvm4CSfc02yjnPUsCYSY4Qw9fXPnDFWLexab0mcpc,28559
 datahub/ingestion/source/identity/okta.py,sha256=jC21myJuMRTaPgj0OD9heaC-mz8ECjqpy2hSJwlUSwM,31943
 datahub/ingestion/source/kafka/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/kafka/kafka.py,sha256=mboUWQmlumEwcXwY2POeK1L8tdk5-CABakZ-MWbvdNQ,26579
+datahub/ingestion/source/kafka/kafka.py,sha256=Bo9l8KbCle8ZUOyJMOo2HsJcT63EM_R7bCrPS-FhWT4,26512
 datahub/ingestion/source/kafka/kafka_schema_registry_base.py,sha256=13XjSwqyVhH1CJUFHAbWdmmv_Rw0Ju_9HQdBmIzPNNA,566
 datahub/ingestion/source/kafka_connect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/kafka_connect/common.py,sha256=lH64n1v_rJamWGfidBeuQJj8W1_IvOBpXQLR2YZaEvQ,7057
 datahub/ingestion/source/kafka_connect/kafka_connect.py,sha256=AVAgBvgH7kM9I2ke3mwr8CfIL1J2SdVHH_86rnCFwrM,17727
-datahub/ingestion/source/kafka_connect/sink_connectors.py,sha256=rNxolagqwQWQmVp4mDr1C-1TB6Drxc2b1dM9JSjNnuA,12905
-datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=UKcKGEtQGtVcaAuGQiNXQ0REtlBYMpZpMr1juJ-N1QM,21087
+datahub/ingestion/source/kafka_connect/sink_connectors.py,sha256=kEnxOXTik5HSDLjRDQQ2LnK944w_wfa8-apEHk_-v7Q,12898
+datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=OQ0vjz9xF0T30pRln_gDvelmaOE5jTAxwsCtm1K4SWM,21080
 datahub/ingestion/source/looker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/looker/lkml_patched.py,sha256=XShEU7Wbz0DubDhYMjKf9wjKZrBJa2XPg9MIjp8rPhk,733
 datahub/ingestion/source/looker/looker_common.py,sha256=hR4iufcjwh_iBzIJs9tYR4lwSn6vXk2WF5VFAYepKGM,62094
@@ -359,7 +359,7 @@ datahub/ingestion/source/looker/looker_file_loader.py,sha256=gb2Z97_w28MsybYe01J
 datahub/ingestion/source/looker/looker_lib_wrapper.py,sha256=2aEorQ3WjBzYVQIm-5QR2qDGAhpKflstwO5X9oboXS8,11553
 datahub/ingestion/source/looker/looker_liquid_tag.py,sha256=mO4G4MNA4YZFvZaDBpdiJ2vP3irC82kY34RdaK4Pbfs,3100
 datahub/ingestion/source/looker/looker_query_model.py,sha256=N0jBbFruiCIIGT6sJn6tNeppeQ78KGTkOwTLirhxFNc,2144
-datahub/ingestion/source/looker/looker_source.py,sha256=25Ha1lUdmGbLZl4vuARyEi5i7MedHaKu4HEwBQgbvAU,66413
+datahub/ingestion/source/looker/looker_source.py,sha256=rMuq0awCCmKJLbxPDCzl6VlrRe7vgRZ9wJ68b6iCNt4,66383
 datahub/ingestion/source/looker/looker_template_language.py,sha256=5fZFPKFP3IYbJg3jLifjaji4wWg8wRy-1XDvc8Qucus,17949
 datahub/ingestion/source/looker/looker_usage.py,sha256=qFBX7OHtIcarYIqFe0jQMrDV8MMPV_nN4PZrZRUznTw,23029
 datahub/ingestion/source/looker/looker_view_id_cache.py,sha256=92gDy6NONhJYBp92z_IBzDVZvezmUIkaBCZY1bdk6mE,4392
@@ -376,10 +376,10 @@ datahub/ingestion/source/metadata/lineage.py,sha256=2iK-hsORWm7NSvMZcG4D5hb8_PH5
 datahub/ingestion/source/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/neo4j/neo4j_source.py,sha256=O3jjdnsx7IyYPBLbxowL85Qo4zs4H-maMOH4-6ZNCk4,13063
 datahub/ingestion/source/powerbi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/powerbi/config.py,sha256=5rG62dspGF9jIo8l6HLpB6ECv5n-t1un2ZyGiisD784,24219
+datahub/ingestion/source/powerbi/config.py,sha256=bflLIq6rpZeJ7ULvN2gaAVcSHO5jTJ6vdNPvwo1LH7M,24212
 datahub/ingestion/source/powerbi/dataplatform_instance_resolver.py,sha256=-njW1kJOy-LY5JFwJLhVQ0bMBj9NQz5TZhQqsSi_KsM,2285
 datahub/ingestion/source/powerbi/powerbi-lexical-grammar.rule,sha256=5df3qvalCS9hZ46DPXs6XDcw9-IofGf8Eol_rUC7LHI,20329
-datahub/ingestion/source/powerbi/powerbi.py,sha256=f0vwzVdKAU7Qp7gRHgTOPq45ThUmsbFXCwcIDaS1S34,56464
+datahub/ingestion/source/powerbi/powerbi.py,sha256=b9zNeT9aS7v2GWUL1SROnIMwQwAFX0YTO2UNQMLWItc,56450
 datahub/ingestion/source/powerbi/m_query/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/powerbi/m_query/data_classes.py,sha256=yDi0C13ko2dVxdLJBYvUuGbT4Q2hxQRse3sL7Ul1ZU0,2050
 datahub/ingestion/source/powerbi/m_query/native_sql_parser.py,sha256=zzKVDGeUM3Yv3-zNah4D6mSnr6jXsstNuLmzczcPQEE,3683
@@ -412,7 +412,7 @@ datahub/ingestion/source/redshift/datashares.py,sha256=kH3YkoenOa59XZU12XeUf283l
 datahub/ingestion/source/redshift/exception.py,sha256=dxzYUIv5B_FAWhOuzG2u5We7FX-ar4jhOXPXAlEIvgM,2055
 datahub/ingestion/source/redshift/lineage.py,sha256=IPF8vHy2MFyhK-hu2-lxV2-kcnNAEzltPLnnIvwIBMY,44100
 datahub/ingestion/source/redshift/lineage_v2.py,sha256=dbTvuaJBV5yvCWM_oEAqZIA1JOlGxLJOexbEB47A_xE,17962
-datahub/ingestion/source/redshift/profile.py,sha256=dq7m9YG3TvEMbplwVIutUpzbXLPH8KIj9SuWNo7PWWE,4323
+datahub/ingestion/source/redshift/profile.py,sha256=H1Xtc2rXScUv4w0b2BbM7POjYEwqIql_rpWvlumY_EM,4309
 datahub/ingestion/source/redshift/query.py,sha256=vVIuNUaU4a7AfMFJZlgLuqi0cGVl0gVz8xZUSnPhWvs,47845
 datahub/ingestion/source/redshift/redshift.py,sha256=whMujnJxwNT2ZXnOVRrZQiy317hlsvbARzabKmI3oN8,43536
 datahub/ingestion/source/redshift/redshift_data_reader.py,sha256=zc69jwXHdF-w8J4Hq-ZQ6BjHQ75Ij2iNDMpoRJlcmlU,1724
@@ -464,7 +464,7 @@ datahub/ingestion/source/snowflake/snowflake_shares.py,sha256=maZyFkfrbVogEFM0tT
 datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=kTmuCtRnvHqM8WBYhWeK4XafJq3ssFL9kcS03jEeWT4,5506
 datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=1eLYTcgmfzDs9xktMTTE74L5SeNP48Qg3uLr9y-Ez3Y,8733
 datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=ySFm7WDk8FW9KjCnX4HQfTqObIrlUS-V8WIHl3j0CTI,24848
-datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=22JhQx22kP__ux7mKsXJTFvIYrj9uRym8vKBDsVSKSs,14088
+datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=2lmvAeZELTjAzg4Y5E0oY41r1IzVEvg6OHAvVJftSFk,14081
 datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=nAbudDVh9A0kqao3jnIdgBlFNhNk1WIxoU1cofeXkFQ,33905
 datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/sql/athena.py,sha256=X4SC1kwKJ3a3T0wGQIlOQxzJKE-LEr-U55QKpRUH4j8,23861
@@ -496,7 +496,7 @@ datahub/ingestion/source/sql/mssql/__init__.py,sha256=1agpl8S_uDW40olkhCX_W19dbr
 datahub/ingestion/source/sql/mssql/job_models.py,sha256=nAo3rciu-w2-dXCz6_ekDEbGMEjCMEfh8WvSfXoF2l0,9359
 datahub/ingestion/source/sql/mssql/source.py,sha256=Er0uTAsEOzLg2KVddf22pOdzEAwkOjNdqYCPxeCiHdA,32755
 datahub/ingestion/source/sql/stored_procedures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/sql/stored_procedures/base.py,sha256=X4RXYknK8ZPsMgXQFzo2_CsMcXYhDWLm-fLFmZNkZfc,7888
+datahub/ingestion/source/sql/stored_procedures/base.py,sha256=n0l5OaTuW-m3TRvkxs3TqvgMeWF6BagzW3tjyWUcC1A,8631
 datahub/ingestion/source/sql/stored_procedures/lineage.py,sha256=9kBoq4JLa4-I3TMAYwChXp76Kag8YBUEIAkaQMplIkY,1844
 datahub/ingestion/source/state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/state/checkpoint.py,sha256=-fTUZKkY4nHTFqSWZ0jJkkdIu_tWlOjRNhm4FTr4ul4,8860
@@ -514,8 +514,8 @@ datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider
 datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py,sha256=DziD57PbHn2Tcy51tYXCG-GQgyTGMUxnkuzVS_xihFY,4079
 datahub/ingestion/source/state_provider/state_provider_registry.py,sha256=SVq4mIyGNmLXE9OZx1taOiNPqDoQp03-Ot9rYnB5F3k,401
 datahub/ingestion/source/tableau/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/tableau/tableau.py,sha256=AtQYzVWBLwrEjwgfBlBIv7aJJwZVloQSgJqt9ML6IrU,154137
-datahub/ingestion/source/tableau/tableau_common.py,sha256=OhBJDdX-cT93BIDVYPHAxNJYr0dvpT_udqY_GR9kk_E,26945
+datahub/ingestion/source/tableau/tableau.py,sha256=tBDyDAAFnFS1pag_cK1hjE73qpFRcZK-BVphBJ5r_Gs,154109
+datahub/ingestion/source/tableau/tableau_common.py,sha256=MIkHKZg_v2IVCRk-YdPlLZl3m0LcWZm5Indwb3IV2ZQ,26931
 datahub/ingestion/source/tableau/tableau_constant.py,sha256=ZcAeHsQUXVVL26ORly0ByZk_GJAFbxaKuJAlX_sYMac,2686
 datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=nSyx9RzC6TCQDm-cTVJ657qT8iDwzk_8JMKpohhmOc4,1046
 datahub/ingestion/source/tableau/tableau_validation.py,sha256=pd--LcTLTfrFsouhCOvGC_2IjeMfKbJV81EEo3ibMwE,1820
@@ -529,7 +529,7 @@ datahub/ingestion/source/unity/proxy.py,sha256=qYgjw0COscvUk8TvgWwZKgYvkYyA3j4yc
 datahub/ingestion/source/unity/proxy_profiling.py,sha256=WLqvYP6MziaisA4LYL4T_GA-kPt6Xdde7bfaYsjYw40,9663
 datahub/ingestion/source/unity/proxy_types.py,sha256=qrvHiwPzl5cPX-KRvcIGGeJVdr0I8XUQmoAI6ErZ-v8,9371
 datahub/ingestion/source/unity/report.py,sha256=XFT9oQfvEB4RkTvWGgFOoQuLPUN_AIoPXZ79xeDhGHQ,2831
-datahub/ingestion/source/unity/source.py,sha256=5w24IC4oDhsycdt3TG7rtXOkoQpxE_-dHlLGYui4K8I,42368
+datahub/ingestion/source/unity/source.py,sha256=eOEWH_1a7IrKPrkhOc5bKl_5PiqfMHBXVoPy-e-Ij9c,42361
 datahub/ingestion/source/unity/usage.py,sha256=0wETBAaZvHI_EGgBlxX3bKsVHEAdnUV8_bKI_lbyWjY,11500
 datahub/ingestion/source/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/usage/clickhouse_usage.py,sha256=jJ-EUJdS7t4d9RVjLWQQ2e36wmYzs8xtpD632z6pLiw,9974
@@ -549,8 +549,8 @@ datahub/ingestion/source_report/pulsar.py,sha256=f6CMNw8TyPp3tuSGsLLPEhSvoQLXwxt
 datahub/ingestion/source_report/time_window.py,sha256=9yI5l2S1DcF7ClvUHLeN8m62I5vlhV9k-aQqSZh2l7w,229
 datahub/ingestion/transformer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/transformer/add_dataset_browse_path.py,sha256=7ngPAAAsdV8nsqFHZyHfO5j1vkCcf5zNqOkG2Cpx1Tw,3420
-datahub/ingestion/transformer/add_dataset_dataproduct.py,sha256=O1tiwCIF_FfsroX9aRYtJiJjbSUDoI2w8cPXr0pyOUI,6431
-datahub/ingestion/transformer/add_dataset_ownership.py,sha256=xuv6qymZceHYOtFIdo5H-rSRk_eSSMnePTz4qjf3YPU,8671
+datahub/ingestion/transformer/add_dataset_dataproduct.py,sha256=uOMnbJoZxvNnJSx_IOU7M83or4Bl9d4pfsnvkn49yE0,6424
+datahub/ingestion/transformer/add_dataset_ownership.py,sha256=D3uNRoiHNiqg7PS2wP_IWcanUap2l3PyAuWeGQwd4QI,8664
 datahub/ingestion/transformer/add_dataset_properties.py,sha256=9p96YkNzoTCmdsuWvygvsYgSklhzLMhA9fKO8GiA2gk,5607
 datahub/ingestion/transformer/add_dataset_schema_tags.py,sha256=yRWT9nEXbdcH50qL954N90BREgF8DTXdx2twmuu1p4g,5666
 datahub/ingestion/transformer/add_dataset_schema_terms.py,sha256=ftoZ4DJE0lJTKnzATyrK-9JtjiI1Etz7yWSqta9KowM,6550
@@ -558,7 +558,7 @@ datahub/ingestion/transformer/add_dataset_tags.py,sha256=eyvQpb0hntSmaP9HSHcIiHL
 datahub/ingestion/transformer/add_dataset_terms.py,sha256=F3DgVFJfm_Ofh4NOt7caKbVx4FQE-tOns17E0qDn43o,5799
 datahub/ingestion/transformer/auto_helper_transformer.py,sha256=MuxoHr0_SPi8LpHbYYOYaEO8d-7XBu2zRn-PDnqrIHU,2843
 datahub/ingestion/transformer/base_transformer.py,sha256=j5HmnplhY1K_7oa9DNHhCegs6eMKndc-VFoT8GGxgpU,12383
-datahub/ingestion/transformer/dataset_domain.py,sha256=rJNUcDEailHHmjmY4ZDpqZaeB5VC2QJ55DsDHF2k2Ao,9039
+datahub/ingestion/transformer/dataset_domain.py,sha256=WWvATUs9Z59_m5F6ZUA5fktJ6OBWzrJuJ2VAjdmHrCI,9032
 datahub/ingestion/transformer/dataset_domain_based_on_tags.py,sha256=HN3QapXNg2CTj54F4mY3f-XkAIB-rMu0mG7ThOCE5fI,3092
 datahub/ingestion/transformer/dataset_transformer.py,sha256=dOK0oO6R6dbuxk5i5Za6hkzy8xCEpQxG8iKjXeIZKM4,5305
 datahub/ingestion/transformer/extract_dataset_tags.py,sha256=uCxf7L9AdMjVs7gvq1k1geuxcWDxv0LXEwXj6EQlWSE,2499
@@ -569,7 +569,6 @@ datahub/ingestion/transformer/pattern_cleanup_dataset_usage_user.py,sha256=jTURu
 datahub/ingestion/transformer/pattern_cleanup_ownership.py,sha256=YJH4lv1ztKAYwsdRz5RiUu6SX08h2GBr1S-9QlcmB18,3755
 datahub/ingestion/transformer/remove_dataset_ownership.py,sha256=kHiIcT19BDKNIuCQqAj827E1ZEvME0eGGrILEACALRc,1195
 datahub/ingestion/transformer/replace_external_url.py,sha256=Nw2V1m86fD1vMGLYGPlaoIJEORV0O9qAqWydL-1n2Ng,4058
-datahub/ingestion/transformer/system_metadata_transformer.py,sha256=KbYWBlg5mk_nz8_AVoHS7w0nIPe8wimeI69HVyhAsJQ,1755
 datahub/ingestion/transformer/tags_to_terms.py,sha256=VDcd7cM5hGCxo6QP1x4RNEw5Q9v4WDxjRhQMpAl95-A,5558
 datahub/ingestion/transformer/transform_registry.py,sha256=bartmA1zEaULNy5W1Q7gRF8h5Y57BFC6XNOGfCzh1Zw,251
 datahub/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -591,14 +590,14 @@ datahub/lite/duckdb_lite_config.py,sha256=PGY5Hab_xbbqoA1hf7OKySBJ2JQJaLNKl-4CO3
 datahub/lite/lite_local.py,sha256=jsAwvnMJz_aR_a7y1ju2_ER3J3PS8wtEhutA9GmSihA,2858
 datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw,286
 datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
-datahub/lite/lite_util.py,sha256=Cm6trMTeo0X1fv4nSsW9lC0jqce7Jt-05GhOtIGzsVc,4559
+datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
 datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
-datahub/metadata/_schema_classes.py,sha256=HvQKAu3eTDt3lkQcoLpBqYlPo0jfBONWCd6T9a1Meck,1011170
-datahub/metadata/schema.avsc,sha256=rX4HzpW5NPL28A2vfu6ESo-hSyQDnK7LUnvasa7dQCE,752628
+datahub/metadata/_schema_classes.py,sha256=n8NwTeBKsnQdtNCMG85LDc6mPgDT5JGUbsgg9KcNj18,1012635
+datahub/metadata/schema.avsc,sha256=ckv1TFKtvz0eGTSqL2qijL4eqfTuQYMIj91pO0vbc2M,753114
 datahub/metadata/schema_classes.py,sha256=X5Jl5EaSxyHdXOQv14pJ5WkQALun4MRpJ4q12wVFE18,1299
 datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
 datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
-datahub/metadata/_urns/urn_defs.py,sha256=mQ52ozRUt19MyBLNZh1f1ETlafCzCYmEbcKxAjR_8o4,133983
+datahub/metadata/_urns/urn_defs.py,sha256=SCNCcJBVZjQd2LE98oMyAA8Lg30Umc6mExPdzAKBhgs,136344
 datahub/metadata/com/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/events/__init__.py,sha256=s_dR0plZF-rOxxIbE8ojekJqwiHzl2WYR-Z3kW6kKS0,298
@@ -640,7 +639,7 @@ datahub/metadata/com/linkedin/pegasus2avro/identity/__init__.py,sha256=1U583fdMT
 datahub/metadata/com/linkedin/pegasus2avro/incident/__init__.py,sha256=LfB7ytT1uuGl5Y8oeU6ILCpSGsGmUKgCjsHphn7AThg,916
 datahub/metadata/com/linkedin/pegasus2avro/ingestion/__init__.py,sha256=1bfG2naq4iS_pwU4J-BVer_gfL0hDbJbnH0gh1MPNgA,871
 datahub/metadata/com/linkedin/pegasus2avro/metadata/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
-datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py,sha256=kooj_lICFtq7GjWmYFtQOGuXYy9u4QomKrhTTVKFnDg,4812
+datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py,sha256=bvCD3zLtdgAtLT6ZpjeBQ8-QglBL9fGBPW89whWGNwM,4929
 datahub/metadata/com/linkedin/pegasus2avro/metadata/query/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/pegasus2avro/metadata/query/filter/__init__.py,sha256=DBP_QtxkFmC5q_kuk4dGjb4uOKbB4xKgqTWXGxmNbBQ,532
 datahub/metadata/com/linkedin/pegasus2avro/metadata/snapshot/__init__.py,sha256=OPboF8SV11wGnjvWQB-rxtB0otMdCsE7Tcy7xkOUgz8,2358
@@ -718,6 +717,7 @@ datahub/metadata/schemas/DataHubConnectionDetails.avsc,sha256=IvZj6OA7HRvy-ZIIn0
 datahub/metadata/schemas/DataHubConnectionKey.avsc,sha256=VwbamVFoEdp6epz1lJm_UShBl6ksBxoA7jAYuPI5u3M,522
 datahub/metadata/schemas/DataHubIngestionSourceInfo.avsc,sha256=4wac7sluRIq-0ZjODE5SmuVKuQeW8ajLJNRpqEBRyio,4601
 datahub/metadata/schemas/DataHubIngestionSourceKey.avsc,sha256=CSmoOx_Eqa1-he5dRaVOUQWIv1l2e2lraEPIixKK-lo,526
+datahub/metadata/schemas/DataHubOpenAPISchemaKey.avsc,sha256=q6ZyMoxInwmrkrXkUgMe-i-WZzAxbjcvJ-EI99SnEp8,599
 datahub/metadata/schemas/DataHubPersonaInfo.avsc,sha256=OUvbTgPQsBtzkDDb9pxHXpQ6A7dkL77ZnCXZ-MLEG14,227
 datahub/metadata/schemas/DataHubPersonaKey.avsc,sha256=ddj-DhXa0_YMdLaGkKLLSklfIeDRvSwPXu8o__YEXUE,448
 datahub/metadata/schemas/DataHubPolicyInfo.avsc,sha256=mK8tvgX4hDYuck2R6EihKSYzo6s8hNvvA_hgTOgMrFc,9397
@@ -753,7 +753,7 @@ datahub/metadata/schemas/DataProcessInstanceRunEvent.avsc,sha256=zwTYULEnpMbqwkL
 datahub/metadata/schemas/DataProcessKey.avsc,sha256=sQzsBZAAIJwwZcDrEU3d4G50qlGwrISC1Tqgb9BLq2Y,2258
 datahub/metadata/schemas/DataProductKey.avsc,sha256=5az_f7g0F0oMwWTI4OyEJn47PKQWNVC6OnMZPs9P4Oo,639
 datahub/metadata/schemas/DataProductProperties.avsc,sha256=nYEK6JgpTprU0iZaqWLZsBGYJLkh6HCi1qCu-wbYhvM,6925
-datahub/metadata/schemas/DataTransformLogic.avsc,sha256=wDng1GK9znVoK0INHGiSCSa-AH5MrDkVdMzz4wOWmrY,2011
+datahub/metadata/schemas/DataTransformLogic.avsc,sha256=nHTH6UzJ2Zz88N2aWa96hawLUR20HP7eSynfPtI1kzg,2111
 datahub/metadata/schemas/DataTypeInfo.avsc,sha256=MCjzal71P8uIXZg161LrU8rZTJocZeizK-YxYA0Det0,704
 datahub/metadata/schemas/DataTypeKey.avsc,sha256=Gs5uc_azwg10e36ZbwDTFQMevr0IfiFvJoEGHRzEilw,546
 datahub/metadata/schemas/DatahubIngestionCheckpoint.avsc,sha256=m2Zyrx3ZWDc5gHuwbmBSRJ3JN4NFkpUhDEKM2Yeuqrw,5681
@@ -798,7 +798,7 @@ datahub/metadata/schemas/ExecutionRequestKey.avsc,sha256=SvjnlTAGYsSnvVE0rZ9-7UP
 datahub/metadata/schemas/ExecutionRequestResult.avsc,sha256=kg3xMNr9kYLPnFsV-iqcGm1sh1muQVGJvxUt15L1yKo,2333
 datahub/metadata/schemas/ExecutionRequestSignal.avsc,sha256=dsIUa6tfVSXqYOgh4cW6_Hzi8RjHuJJoO-mBAuZukpA,2515
 datahub/metadata/schemas/Filter.avsc,sha256=PU-aGkc2-sI3ZXY7ci-Y0A7zp1jux3VW_6c8MJRAokg,5933
-datahub/metadata/schemas/FormInfo.avsc,sha256=FbN34htiCgm3LqKDL3sVsJhMUHIyc5jYpGJtYm7Ysd4,6270
+datahub/metadata/schemas/FormInfo.avsc,sha256=tlNI-m6uWJ46yF6Ls6Q_DOEZM2bQgXTd4bSeH30N9MA,6438
 datahub/metadata/schemas/FormKey.avsc,sha256=1-wE28B8T3WJ3JtexreNtFvP3To3n7U-jvYudCuSM9o,437
 datahub/metadata/schemas/Forms.avsc,sha256=shmkhRoHN2gTaTsqGrGDRoNwe_z-nrFbbLjH9MtVDCs,10955
 datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=OVMM6FwhHhufHkezYcVePK0zI2llzFYLVFJhmAiHoiI,10102
@@ -829,7 +829,7 @@ datahub/metadata/schemas/MLFeatureTableProperties.avsc,sha256=BtrqcsxoQXObPZXSGR
 datahub/metadata/schemas/MLHyperParam.avsc,sha256=dE6i5r6LTYMNrQe9yy-jKoP09GOJUf__1bO69ldpydc,833
 datahub/metadata/schemas/MLMetric.avsc,sha256=y8WPVVwjhu3YGtqpFFJYNYK8w778RRL_d2sHG1Dc7uM,804
 datahub/metadata/schemas/MLModelDeploymentKey.avsc,sha256=vt04jFF_ZHSvWhqLoxC8C_KspiRLkvNNIXJI0aKPF1Q,2425
-datahub/metadata/schemas/MLModelDeploymentProperties.avsc,sha256=I3v-uNOeYxO4hooPHOjafWWHuVyeGvG90oma0tzpNFg,5409
+datahub/metadata/schemas/MLModelDeploymentProperties.avsc,sha256=7IlGrMmX8nfgezvaZyrXskCTCRlwvRzGOYUOpFV3r6Y,5480
 datahub/metadata/schemas/MLModelFactorPrompts.avsc,sha256=8kX-P4F4mVLFT980z3MwIautt1_6uA-c_Z87nYNDK-k,2712
 datahub/metadata/schemas/MLModelGroupKey.avsc,sha256=3LoMWejMfCwdoqz3PFinRbY1_Yy4Kypw7pwg3tL42Jg,2497
 datahub/metadata/schemas/MLModelGroupProperties.avsc,sha256=AZ5Pohk3_pCctQ4hcE1UOURQFYHQne0dw_lRUpOu5WY,6924
@@ -838,9 +838,9 @@ datahub/metadata/schemas/MLModelProperties.avsc,sha256=hDCBHxGe-cmCBeU1k0ANuQlKj
 datahub/metadata/schemas/MLPrimaryKeyKey.avsc,sha256=Kq2Q9WxZ6nQ8wR4P6wpPCI-J7FwXQyoa10s6BvXtkm8,1110
 datahub/metadata/schemas/MLPrimaryKeyProperties.avsc,sha256=URIuOpS93RVk8MZVcbZ-dmTwu_cN3KSOKxSR8fm-eTo,6744
 datahub/metadata/schemas/MLTrainingRunProperties.avsc,sha256=WGgj0MuQrGD4UgvyHCJHzTnHja2LlJTOr1gLu8SySj0,4269
-datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=rM6klxSttK-cS083tR0BaVgWVJskjMmhwFbHuV8VKI8,375579
-datahub/metadata/schemas/MetadataChangeLog.avsc,sha256=mpdodpx25E6M1Gq_7slEcPAm-1Es5xPsoqV60HgO7zg,12167
-datahub/metadata/schemas/MetadataChangeProposal.avsc,sha256=EMfQrYsuHf1p6UvBjoLtfdTHGe-vGNJaCFEHz8hdKU0,9698
+datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=OiEGLnlNxypkw2pyupuRj88k5Bkl0hLoiBQ3bODXQjc,375776
+datahub/metadata/schemas/MetadataChangeLog.avsc,sha256=Cf5eECeShCA_XHFr2MRhRQpPE61F6Xv-z1jjoBLJLgc,12239
+datahub/metadata/schemas/MetadataChangeProposal.avsc,sha256=tvO5cGIqZAIvUbMon1RAKgSY4E0jvBqT5VmLWAuNGkY,9770
 datahub/metadata/schemas/Metrics.avsc,sha256=O7DJGjOwmHbb1x_Zj7AuM_HaHKjBvkfJKfUsX8icXD4,690
 datahub/metadata/schemas/NativeGroupMembership.avsc,sha256=9mh9tzyj3ErmTIhX7ERRUm78j1QtGwXUl9UuIXPndBg,588
 datahub/metadata/schemas/NotebookContent.avsc,sha256=ck3yDt0qK5Hn3-mWTNLlYnohXCs6kMUUWPXm7o1JEWE,12496
@@ -859,7 +859,7 @@ datahub/metadata/schemas/PostInfo.avsc,sha256=CxL1Z-6jWO-xQ7mei5dwAFe_U7sfkQrkSB
 datahub/metadata/schemas/PostKey.avsc,sha256=MEA-S-RonGrsrpwYEejR6CXRka2DcqP7S7qvO5JgEFU,475
 datahub/metadata/schemas/QuantitativeAnalyses.avsc,sha256=XrGYVoGUh7ZRjAbMORSiqF5Zl4a0IbYDPiQuSJhABZM,960
 datahub/metadata/schemas/QueryKey.avsc,sha256=VI4oIHvAO7f0lN_7V3QVuBfHcPz31c57XtW6IrlDfxc,518
-datahub/metadata/schemas/QueryProperties.avsc,sha256=fiLYT539EM0jVEAngWXC7P9QnnIMoEsAaUp7nClwu0s,5509
+datahub/metadata/schemas/QueryProperties.avsc,sha256=26Q3zzuzJbUCUG7IJ3q_OEdcNbyzloZzDJWPbv3GQAk,5589
 datahub/metadata/schemas/QuerySubjects.avsc,sha256=WbnUGiOBdc1Ypn4P_R5gCfPH-wNHJUIk7YeyJ2NiOZQ,1747
 datahub/metadata/schemas/QueryUsageStatistics.avsc,sha256=z1gfAnXdBoPEeERi5RESjrdBuS6AcIdqdN5JqWOSuNo,6192
 datahub/metadata/schemas/RoleKey.avsc,sha256=Uas5jFViSHXhFqq8D4P6-UXqywOth3coztjQ5wA7wL0,449
@@ -878,6 +878,7 @@ datahub/metadata/schemas/StructuredPropertyDefinition.avsc,sha256=a-6TaOQ4A7LDFL
 datahub/metadata/schemas/StructuredPropertyKey.avsc,sha256=lp7tQBgeriEU1YMQ6a4-6aUGSWDqNl00lLDym97j1yI,618
 datahub/metadata/schemas/StructuredPropertySettings.avsc,sha256=EDNlXfT1TqogfulCanIc-nuYO9ZxRFOGzD9tl3ZJdB8,3732
 datahub/metadata/schemas/SubTypes.avsc,sha256=bhXbzK020zDyQno97Xp05vmoMeZ82IGu2jz7pWDo3RQ,655
+datahub/metadata/schemas/SystemMetadata.avsc,sha256=wDVdpa9LSAlMzHIiWw-fMLHTCrxcJdnDOY_n5CDNTN8,2068
 datahub/metadata/schemas/TagKey.avsc,sha256=BfckMlx-wg_LV1_PFVgItfNBPtCQ8_erGeQM4LzOXmY,640
 datahub/metadata/schemas/TagProperties.avsc,sha256=Qzttxd7BB38JUwwl7tZzIV1Warnh-uQO-Ahw9Sd-vH4,883
 datahub/metadata/schemas/TelemetryClientId.avsc,sha256=GScej0kXFZxoBUcRVrVynzArFSYQpO_dnhY5Po5dlx0,408
@@ -951,7 +952,7 @@ datahub/testing/check_str_enum.py,sha256=yqk0XXHOGteN-IGqCp5JHy0Kca13BnI09ZqKc4N
 datahub/testing/compare_metadata_json.py,sha256=mTU5evu7KLS3cx8OLOC1fFxj0eY1J1CGV2PEQZmapos,5361
 datahub/testing/docker_utils.py,sha256=g169iy_jNR_mg0p8X31cChZqjOryutAIHUYLq3xqueY,2415
 datahub/testing/doctest.py,sha256=1_8WEhHZ2eRQtw8vsXKzr9L5zzvs0Tcr6q4mnkyyvtw,295
-datahub/testing/mcp_diff.py,sha256=-4Q1GYvv1zXMEBXGCIxAwCwrOttNV5QNZvxFFz0WNuE,10205
+datahub/testing/mcp_diff.py,sha256=Jk1NluXkKWEMLOE11mHE98lfYE_Gn2GeFuu5TNB3YSs,10198
 datahub/testing/pytest_hooks.py,sha256=eifmj0M68AIfjTn_-0vtaBkKl75vNKMjsbYX-pJqmGY,1417
 datahub/upgrade/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/upgrade/upgrade.py,sha256=lf60_dCu51twObAL5E8NqdrW3_2lsnUJUaB9MSEVXwI,16638
@@ -967,10 +968,10 @@ datahub/utilities/dedup_list.py,sha256=dUSpe1AajfuwlHVJKNv-CzDXSCkaw0HgSMOsxqUkQ
 datahub/utilities/delayed_iter.py,sha256=XlsI0DCXkVVejFKOW_uMT0E8DTqqOHQN3Ooak4EcULE,645
 datahub/utilities/delta.py,sha256=hkpF8W7Lvg2gUJBQR3mmIzOxsRQ6i5cchRPFlAVoV10,1128
 datahub/utilities/docs_build.py,sha256=uFMK3z1d4BExpsrvguHunidbEDAzQ8hoOP7iQ0A_IVw,211
-datahub/utilities/file_backed_collections.py,sha256=zW-Xy6zO1E6R-FRJKjlimMKr-emWrgcXvFFY6QQ23aA,21715
+datahub/utilities/file_backed_collections.py,sha256=4_11YQUaIdjr7SUV4AHczArBi8OrrlzuX15ldR1GhKA,21673
 datahub/utilities/global_warning_util.py,sha256=adrEl3WhetQ-bymrPINjd976ZFndhbvk3QosUYGsos8,261
 datahub/utilities/groupby.py,sha256=pe6rP4ZCttYB98yjbs0Aey8C32aLb7rq-NJ_BFky0H4,524
-datahub/utilities/hive_schema_to_avro.py,sha256=1MP0a6FFVEYxLg_4lKF7hPxbHJJy0uRQYkML5zRwV3Q,11622
+datahub/utilities/hive_schema_to_avro.py,sha256=YCdq3jNUTij8ehWgX9v6CiOrf5aTCXr4DERcp_-wBbo,11608
 datahub/utilities/ingest_utils.py,sha256=OgETzX_9g6DcYlxGH0_xgAmAlWLSROr25ydDL-mBhKM,3137
 datahub/utilities/is_pytest.py,sha256=2m9T4S9IIKhI5RfTqrB2ZmumzHocdxBHpM1HroWj2XQ,138
 datahub/utilities/logging_manager.py,sha256=bc-x5VZGvFUHT0HD-TF3Uz_nzw3dpKdJSbz6kjpAqAQ,10073
@@ -1044,8 +1045,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.0.0.2rc3.dist-info/METADATA,sha256=QM27NG1azmKgGpnS7CNZpX7fANAZeoQQe8etXepFIaQ,176849
-acryl_datahub-1.0.0.2rc3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-acryl_datahub-1.0.0.2rc3.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
-acryl_datahub-1.0.0.2rc3.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.0.0.2rc3.dist-info/RECORD,,
+acryl_datahub-1.0.0.2rc5.dist-info/METADATA,sha256=urp7GO85YeQHY_-wuzs6YWZ6xzfGkunfiD-r-e7CvfY,176853
+acryl_datahub-1.0.0.2rc5.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+acryl_datahub-1.0.0.2rc5.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
+acryl_datahub-1.0.0.2rc5.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.0.0.2rc5.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.0.0.2rc3"
+__version__ = "1.0.0.2rc5"
 
 
 def is_dev_mode() -> bool:

datahub/api/entities/datajob/dataflow.py

@@ -9,6 +9,7 @@ from datahub.metadata.schema_classes import (
     AuditStampClass,
     DataFlowInfoClass,
     DataFlowSnapshotClass,
+    DataPlatformInstanceClass,
     GlobalTagsClass,
     MetadataChangeEventClass,
     OwnerClass,
@@ -164,6 +165,20 @@ class DataFlow:
         )
         yield mcp
 
+        if self.platform_instance:
+            instance = builder.make_dataplatform_instance_urn(
+                platform=self.orchestrator,
+                instance=self.platform_instance,
+            )
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=str(self.urn),
+                aspect=DataPlatformInstanceClass(
+                    platform=builder.make_data_platform_urn(self.orchestrator),
+                    instance=instance,
+                ),
+            )
+            yield mcp
+
         for owner in self.generate_ownership_aspect():
             mcp = MetadataChangeProposalWrapper(
                 entityUrn=str(self.urn),

datahub/api/entities/dataprocess/dataprocess_instance.py

@@ -159,6 +159,7 @@ class DataProcessInstance:
                         env=self.template_urn.get_env(),
                         orchestrator=self.template_urn.get_orchestrator_name(),
                         id=self.template_urn.get_flow_id(),
+                        platform_instance=self.data_platform_instance,
                     )
                     for mcp in template_object.generate_mcp():
                         self._emit_mcp(mcp, emitter, callback)
@@ -168,6 +169,7 @@ class DataProcessInstance:
                         id=self.template_urn.get_job_id(),
                         upstream_urns=input_datajob_urns,
                         flow_urn=self.template_urn.get_data_flow_urn(),
+                        platform_instance=self.data_platform_instance,
                     )
                     for mcp in template_object.generate_mcp():
                         self._emit_mcp(mcp, emitter, callback)
@@ -382,6 +384,7 @@ class DataProcessInstance:
             cluster=datajob.flow_urn.cluster,
             template_urn=datajob.urn,
             id=id,
+            data_platform_instance=datajob.platform_instance,
         )
         dpi._template_object = datajob
 
@@ -438,6 +441,7 @@ class DataProcessInstance:
             orchestrator=dataflow.orchestrator,
             cluster=cast(str, dataflow.env),
             template_urn=dataflow.urn,
+            data_platform_instance=dataflow.platform_instance,
         )
         dpi._template_object = dataflow
         return dpi

datahub/api/entities/dataset/dataset.py

@@ -483,7 +483,7 @@ class Dataset(StrictModel):
                                 f"{urn_prefix}:{prop_key}"
                                 if not prop_key.startswith(urn_prefix)
                                 else prop_key
-                                for prop_key in field.structured_properties.keys()
+                                for prop_key in field.structured_properties
                             ]
                         )
                     if field.glossaryTerms:
@@ -497,7 +497,7 @@ class Dataset(StrictModel):
                     f"{urn_prefix}:{prop_key}"
                     if not prop_key.startswith(urn_prefix)
                     else prop_key
-                    for prop_key in self.structured_properties.keys()
+                    for prop_key in self.structured_properties
                 ]
             )
         if self.glossary_terms:

datahub/api/entities/structuredproperties/structuredproperties.py

@@ -43,7 +43,7 @@ class AllowedValue(ConfigModel):
 
 
 VALID_ENTITY_TYPE_URNS = [
-    Urn.make_entity_type_urn(entity_type) for entity_type in URN_TYPES.keys()
+    Urn.make_entity_type_urn(entity_type) for entity_type in URN_TYPES
 ]
 _VALID_ENTITY_TYPES_STRING = f"Valid entity type urns are {', '.join(VALID_ENTITY_TYPE_URNS)}, etc... Ensure that the entity type is valid."
 

datahub/cli/migrate.py

@@ -76,13 +76,13 @@ class MigrationReport:
     def __repr__(self) -> str:
         repr = f"{self._get_prefix()}Migration Report:\n--------------\n"
         repr += f"{self._get_prefix()}Migration Run Id: {self.run_id}\n"
-        repr += f"{self._get_prefix()}Num entities created = {len(set([x[0] for x in self.entities_created.keys()]))}\n"
-        repr += f"{self._get_prefix()}Num entities affected = {len(set([x[0] for x in self.entities_affected.keys()]))}\n"
-        repr += f"{self._get_prefix()}Num entities {'kept' if self.keep else 'migrated'} = {len(set([x[0] for x in self.entities_migrated.keys()]))}\n"
+        repr += f"{self._get_prefix()}Num entities created = {len(set([x[0] for x in self.entities_created]))}\n"
+        repr += f"{self._get_prefix()}Num entities affected = {len(set([x[0] for x in self.entities_affected]))}\n"
+        repr += f"{self._get_prefix()}Num entities {'kept' if self.keep else 'migrated'} = {len(set([x[0] for x in self.entities_migrated]))}\n"
         repr += f"{self._get_prefix()}Details:\n"
-        repr += f"{self._get_prefix()}New Entities Created: {set([x[0] for x in self.entities_created.keys()]) or 'None'}\n"
-        repr += f"{self._get_prefix()}External Entities Affected: {set([x[0] for x in self.entities_affected.keys()]) or 'None'}\n"
-        repr += f"{self._get_prefix()}Old Entities {'Kept' if self.keep else 'Migrated'} = {set([x[0] for x in self.entities_migrated.keys()]) or 'None'}\n"
+        repr += f"{self._get_prefix()}New Entities Created: {set([x[0] for x in self.entities_created]) or 'None'}\n"
+        repr += f"{self._get_prefix()}External Entities Affected: {set([x[0] for x in self.entities_affected]) or 'None'}\n"
+        repr += f"{self._get_prefix()}Old Entities {'Kept' if self.keep else 'Migrated'} = {set([x[0] for x in self.entities_migrated]) or 'None'}\n"
         return repr
 
 

datahub/configuration/common.py

@@ -317,7 +317,7 @@ class KeyValuePattern(ConfigModel):
         return KeyValuePattern()
 
     def value(self, string: str) -> List[str]:
-        matching_keys = [key for key in self.rules.keys() if re.match(key, string)]
+        matching_keys = [key for key in self.rules if re.match(key, string)]
         if not matching_keys:
             return []
         elif self.first_match_only:

datahub/ingestion/api/common.py

@@ -12,6 +12,9 @@ if TYPE_CHECKING:
 
 T = TypeVar("T")
 
+if TYPE_CHECKING:
+    from datahub.ingestion.run.pipeline_config import FlagsConfig
+
 
 @dataclass
 class RecordEnvelope(Generic[T]):
@@ -60,6 +63,12 @@ class PipelineContext:
 
         self._set_dataset_urn_to_lower_if_needed()
 
+    @property
+    def flags(self) -> "FlagsConfig":
+        from datahub.ingestion.run.pipeline_config import FlagsConfig
+
+        return self.pipeline_config.flags if self.pipeline_config else FlagsConfig()
+
     def _set_dataset_urn_to_lower_if_needed(self) -> None:
         # TODO: Get rid of this function once lower-casing is the standard.
         if self.graph:

datahub/ingestion/api/source.py

@@ -39,6 +39,7 @@ from datahub.ingestion.api.closeable import Closeable
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope, WorkUnit
 from datahub.ingestion.api.report import Report
 from datahub.ingestion.api.source_helpers import (
+    AutoSystemMetadata,
     auto_browse_path_v2,
     auto_fix_duplicate_schema_field_paths,
     auto_fix_empty_field_paths,
@@ -475,8 +476,10 @@ class Source(Closeable, metaclass=ABCMeta):
         return stream
 
     def get_workunits(self) -> Iterable[MetadataWorkUnit]:
+        workunit_processors = self.get_workunit_processors()
+        workunit_processors.append(AutoSystemMetadata(self.ctx).stamp)
         return self._apply_workunit_processors(
-            self.get_workunit_processors(), auto_workunit(self.get_workunits_internal())
+            workunit_processors, auto_workunit(self.get_workunits_internal())
         )
 
     def get_workunits_internal(

datahub/ingestion/api/source_helpers.py

@@ -13,9 +13,14 @@ from typing import (
 )
 
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
-from datahub.emitter.mce_builder import make_dataplatform_instance_urn, parse_ts_millis
+from datahub.emitter.mce_builder import (
+    get_sys_time,
+    make_dataplatform_instance_urn,
+    parse_ts_millis,
+)
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import entity_supports_aspect
+from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.metadata.schema_classes import (
     BrowsePathEntryClass,
@@ -544,3 +549,23 @@ def _prepend_platform_instance(
         return [BrowsePathEntryClass(id=urn, urn=urn)] + entries
 
     return entries
+
+
+class AutoSystemMetadata:
+    def __init__(self, ctx: PipelineContext):
+        self.ctx = ctx
+
+    def stamp(self, stream: Iterable[MetadataWorkUnit]) -> Iterable[MetadataWorkUnit]:
+        for wu in stream:
+            yield self.stamp_wu(wu)
+
+    def stamp_wu(self, wu: MetadataWorkUnit) -> MetadataWorkUnit:
+        if self.ctx.flags.set_system_metadata:
+            if not wu.metadata.systemMetadata:
+                wu.metadata.systemMetadata = SystemMetadataClass()
+            wu.metadata.systemMetadata.runId = self.ctx.run_id
+            if not wu.metadata.systemMetadata.lastObserved:
+                wu.metadata.systemMetadata.lastObserved = get_sys_time()
+            if self.ctx.flags.set_system_metadata_pipeline_name:
+                wu.metadata.systemMetadata.pipelineName = self.ctx.pipeline_name
+        return wu