acryl-datahub 1.1.1rc3__py3-none-any.whl → 1.1.1rc4__py3-none-any.whl

{acryl_datahub-1.1.1rc3.dist-info → acryl_datahub-1.1.1rc4.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.1.1rc3.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.1.1rc4.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=YKv9NztkHrTZ4_8dkC02c7V9Al8mWiRqMfERfcuZjRA,321
+datahub/_version.py,sha256=eN5Hk_vGfWfTdra8X3fmbgWvHtgAObi4UtnL4Fd3io8,321
 datahub/entrypoints.py,sha256=H-YFTvxTJOgpWsFBVlxyb1opjkq-hjTzNmjy5Fq3RHg,8992
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -48,7 +48,7 @@ datahub/api/entities/dataprocess/dataprocess_instance.py,sha256=IhY-rcXs-r8EatwW
 datahub/api/entities/dataproduct/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/api/entities/dataproduct/dataproduct.py,sha256=148TmItxDDyGNzfZdL8aDreSEtyAw79IN8N8oSmNOPE,21461
 datahub/api/entities/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/api/entities/dataset/dataset.py,sha256=Aa89GZA1R3kY6o5YahFC2YF3pq9B0pipWcKThrhY5l4,49481
+datahub/api/entities/dataset/dataset.py,sha256=BLOn7o-A3OLD4D2JTq8vF_dwTTW9HHDnDwd6lhI1TMI,49512
 datahub/api/entities/forms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/api/entities/forms/forms.py,sha256=B1KnoMmaXwznWdbjltoLq3sH9qj-BpzyC7z7FcwwOM4,15812
 datahub/api/entities/forms/forms_graphql_constants.py,sha256=DKpnKlMKTjmnyrCTvp63V4LX4THGTAMq3ep8THrSGP4,537
@@ -72,7 +72,7 @@ datahub/cli/env_utils.py,sha256=RQzjg4JE29hjPt4v7p-RuqoOr99w8E3DBHWiN2Sm7T4,252
 datahub/cli/exists_cli.py,sha256=Md8_jGdZZXJpX9qBL-hkRJ_DzTXCPKP0fY69Y_sjZqs,1301
 datahub/cli/get_cli.py,sha256=aKnepFef6dyk4sxnrUOnHoEi1yvyBaQT5U6st8NqKQE,2395
 datahub/cli/iceberg_cli.py,sha256=L4QIBc7SiKMJF02S94mFpdOI0P1zIZESeW-o7waJSf4,23127
-datahub/cli/ingest_cli.py,sha256=WJii76u1-bkJxmPi_P3jyTuuh0ShSkNurBE8I7fNnFU,20765
+datahub/cli/ingest_cli.py,sha256=VRzMZgLpUY_pXulY5xtue6S6cNxsb-memHjystXGXRo,21018
 datahub/cli/json_file.py,sha256=nWo-VVthaaW4Do1eUqgrzk0fShb29MjiKXvZVOTq76c,943
 datahub/cli/lite_cli.py,sha256=XKMejSuYUToKBvgN3YmmnxjRcaG5WPw23gJuQK8pgRc,13099
 datahub/cli/migrate.py,sha256=QZKshJX7nDKPDM0UGt8bLDchQQ7Z_3oassf8-jLL5yA,17953
@@ -125,8 +125,8 @@ datahub/emitter/mcp.py,sha256=u6LphyhpbdFqboTAL_9MzXhGjc45o_BePoDFBkEEYWo,10484
 datahub/emitter/mcp_builder.py,sha256=8IwJAlolQkPpMqQJPLtGrsUqAcuFNs98nrI5iYUxgaU,11920
 datahub/emitter/mcp_patch_builder.py,sha256=u7cpW6DkiN7KpLapmMaXgL_FneoN69boxiANbVgMdSI,4564
 datahub/emitter/request_helper.py,sha256=2Sij9VJqgA7xZI6I7IuxsA8ioakbz0FJ3gvazxU_z3M,5738
-datahub/emitter/response_helper.py,sha256=h2hrZYiv4xfauD_lHPW_fN_AV8KhWNM4CVd-Lat2vT0,4608
-datahub/emitter/rest_emitter.py,sha256=1qo055dqrolOJibwv7dY1FU4A_OiJ094UnBynes-yMk,36363
+datahub/emitter/response_helper.py,sha256=qGm45n43CepW7j6kP9wTXuP-U-SZnn7hQdJTdVaoqhQ,7504
+datahub/emitter/rest_emitter.py,sha256=0_jZbHQNVm1DGlHKuFMJkTWvWvxy7HxcCZDgjldHbKk,36410
 datahub/emitter/serialization_helper.py,sha256=q12Avmf70Vy4ttQGMJoTKlE5EsybMKNg2w3MQeZiHvk,3652
 datahub/emitter/sql_parsing_builder.py,sha256=Cr5imZrm3dYDSCACt5MFscgHCtVbHTD6IjUmsvsKoEs,11991
 datahub/emitter/synchronized_file_emitter.py,sha256=s4ATuxalI4GDAkrZTaGSegxBdvvNPZ9jRSdtElU0kNs,1805
@@ -210,8 +210,8 @@ datahub/ingestion/source/mlflow.py,sha256=fh7izN9jlSwbpGIrEyJktlmwFZR5vNG9z9L5VQ
 datahub/ingestion/source/mode.py,sha256=g3nhkpW5KS_w3a8JaKWoq3XBNOZKFlmxZq9XI2D5dXY,72161
 datahub/ingestion/source/mongodb.py,sha256=2C2Cxn8DXL53IbNiywIuKt8UT_EMcPg9f8su-OPSNGU,21237
 datahub/ingestion/source/nifi.py,sha256=D1gBXxdpLuUQ0eurwofIR_SGg1rHGhwk3qxsWI1PT9c,56882
-datahub/ingestion/source/openapi.py,sha256=zx976zstg6M2KoTz_iKKgU9VETDeX2rnw6BofiHXbDc,18669
-datahub/ingestion/source/openapi_parser.py,sha256=HsG5W9eBTvgoPC78QZ4jSlX-UnQagQ4uXXdOm_relmM,14812
+datahub/ingestion/source/openapi.py,sha256=VaR2xYaH1IhvRixpTBC7-168F74eIIyKiEKb5EqTO64,19253
+datahub/ingestion/source/openapi_parser.py,sha256=T87e2r-oPGgQl_FDMHnSGFZzApvWDCyKWnzIrVI5Alo,15420
 datahub/ingestion/source/preset.py,sha256=bbh0ZWiAZMy2zuJDmaRY07_OuGJ9tdtKjwvIxqbY5II,3964
 datahub/ingestion/source/pulsar.py,sha256=u5F8QnCLJsht5-7XCiUTsnfhCPIpKVB_l32CgMCU-As,20187
 datahub/ingestion/source/redash.py,sha256=YxjSad-X_wPmxYH8dJmFz_VCFhiLTCTSlK99WdvcYiA,30653
@@ -281,11 +281,11 @@ datahub/ingestion/source/data_lake_common/data_lake_utils.py,sha256=55mK0nsehqGD
 datahub/ingestion/source/data_lake_common/object_store.py,sha256=T7onzor_15MR-7ecxqcd6YnGY0-bGXHJzseg6nfo2Og,20302
 datahub/ingestion/source/data_lake_common/path_spec.py,sha256=8teyhkfFTy6QJdwH11tKw3cd5XV2wJOBctAtAiksoPM,24423
 datahub/ingestion/source/datahub/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/datahub/config.py,sha256=JohcVz2pYnHbmJd0SGcIDH7Lp-K6MIJlswkid0vTQO4,4762
+datahub/ingestion/source/datahub/config.py,sha256=bjR1U3F85FbtgqmLDW-f_4dQvuw5AsJQxdQlOUeHDUk,5126
 datahub/ingestion/source/datahub/datahub_api_reader.py,sha256=hlKADVEPoTFiRGKqRsMF5mL4fSu_IrIW8Nx7LpEzvkM,2134
-datahub/ingestion/source/datahub/datahub_database_reader.py,sha256=yqp-VOAaA4VfMEXJZ6mz0_M2KlD3t5w31brFVyoIqK4,8981
+datahub/ingestion/source/datahub/datahub_database_reader.py,sha256=HyZxYyJi7o2eWKm2Sxgda6GUcC71g7zDWmRGS6nfnno,14755
 datahub/ingestion/source/datahub/datahub_kafka_reader.py,sha256=gnxhhlK-jrfnHqD_4eVmfcdtBNW6pi1N_qkDZ7uSb3o,4187
-datahub/ingestion/source/datahub/datahub_source.py,sha256=LHT6VyFP4gHlyOrZHhP72qxgZFeofFukKFbFRI8mOuk,8726
+datahub/ingestion/source/datahub/datahub_source.py,sha256=LsfDntDtqt_HMYCWwYctFK3Utyo_VO2x26rDsf6FO0I,8730
 datahub/ingestion/source/datahub/report.py,sha256=VHBfCbwFRzdLdB7hQG9ST4EiZxl_vBCU0XxGcZR6Xxs,940
 datahub/ingestion/source/datahub/state.py,sha256=PZoT7sSK1wadVf5vN6phrgr7I6LL7ePP-EJjP1OO0bQ,3507
 datahub/ingestion/source/dbt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -455,7 +455,7 @@ datahub/ingestion/source/snowflake/constants.py,sha256=XCW3vw4JfLn_s8-oXBX6WFNMP
 datahub/ingestion/source/snowflake/oauth_config.py,sha256=ol9D3RmruGStJAeL8PYSQguSqcD2HfkjPkMF2AB_eZs,1277
 datahub/ingestion/source/snowflake/oauth_generator.py,sha256=fu2VnREGuJXeTqIV2jx4TwieVnznf83HQkrE0h2DGGM,3423
 datahub/ingestion/source/snowflake/snowflake_assertion.py,sha256=_l3k4aI9wvioE81xxdeizJn9nJCZ_nMIXgk9N6pEk5o,4803
-datahub/ingestion/source/snowflake/snowflake_config.py,sha256=DqadsSgxjnSY8aY7hcY7721iDPlezYAFDDawK_J0BGE,20416
+datahub/ingestion/source/snowflake/snowflake_config.py,sha256=98uovcOT9IIvMaOmDHfSK5Ndw_UZ77IVttvHByBbrTc,21079
 datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=3-nP3HHCblUnUHYo_fvFp5VOAteCtR4GNjaUEvyNTNQ,18175
 datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
 datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=eWUlWMc5X2X_Y1I2peworFWLLsXQjryEHxPDuSqrowg,21683
@@ -470,7 +470,7 @@ datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=kTmuCtRnvHqM8WBYh
 datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=eA9xh-G1Ydr1OwUUtrbXUWp26hE1jF0zvyKNky_i_nQ,8887
 datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=ySFm7WDk8FW9KjCnX4HQfTqObIrlUS-V8WIHl3j0CTI,24848
 datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=2lmvAeZELTjAzg4Y5E0oY41r1IzVEvg6OHAvVJftSFk,14081
-datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=nAbudDVh9A0kqao3jnIdgBlFNhNk1WIxoU1cofeXkFQ,33905
+datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=P8iVQCp49hsOm9IJp2VwtTsB0ejmt1nMKbTn1weRKGc,34612
 datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/sql/athena.py,sha256=HsBkm_EOw9wAmCAcA9cv93ctFhlJPk4cVKCq3jPxbpM,23916
 datahub/ingestion/source/sql/clickhouse.py,sha256=9Fvaic9FZufRKdhVz2EcPUnEt5cA9VJbaX4xxFA7joU,25653
@@ -598,12 +598,12 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
 datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
 datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
 datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
-datahub/metadata/_internal_schema_classes.py,sha256=zFi0q-OOxdhPXr6ril8nEZhgChzXY_a3BkVwH3h_8bo,1013146
-datahub/metadata/schema.avsc,sha256=dnBmtikhHW7neORZOUoiH21MZXoXRaw5E1gwwfyCnqs,701603
+datahub/metadata/_internal_schema_classes.py,sha256=u7sbT18cTwsMNgeyZ1gAOCv_RYwkoMx7IxHW2PD_or4,1013146
+datahub/metadata/schema.avsc,sha256=g4WYvhK3XBZvUlpVQG-KAai50BiIpQgCYAs1WaehxXM,765487
 datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
 datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
 datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
-datahub/metadata/_urns/urn_defs.py,sha256=NVHoYdf8wGZZpQfALAO01yrKiXKLpz0ST1iqaWMEYuo,136344
+datahub/metadata/_urns/urn_defs.py,sha256=irNlYfKJ-sqLcI-nOPeGYicdcMpxoDlJYEoQCmY5A8o,136344
 datahub/metadata/com/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/events/__init__.py,sha256=s_dR0plZF-rOxxIbE8ojekJqwiHzl2WYR-Z3kW6kKS0,298
@@ -799,7 +799,7 @@ datahub/metadata/schemas/EntityTypeInfo.avsc,sha256=QVZqy_y1tbxLhvh4k2Mb11otbSC-
 datahub/metadata/schemas/EntityTypeKey.avsc,sha256=1BmHmPCw7AGKwhJ7YGYLljYb-uaNawBvc8EIkzHETwg,582
 datahub/metadata/schemas/EthicalConsiderations.avsc,sha256=FSEs21RK6bc3i_TY-CB9gpmT-oMsyDvYzIfRX0_Xn8w,1957
 datahub/metadata/schemas/EvaluationData.avsc,sha256=i2VFWNmw2f2EWpSOe7vXAFEG2JIdtU-51I4e4bPmT2Q,1714
-datahub/metadata/schemas/ExecutionRequestInput.avsc,sha256=FJQ4HtU1ofrz56LGeG4Z4IuhE-s38O8rKewoHkgXAes,2749
+datahub/metadata/schemas/ExecutionRequestInput.avsc,sha256=5wxKqnxDG7N9MhcyUFLLt58zfeQK4i8ktJaN0bPxzds,2879
 datahub/metadata/schemas/ExecutionRequestKey.avsc,sha256=SvjnlTAGYsSnvVE0rZ9-7UP0f8oMDhK2YCJ9H3VIGE4,584
 datahub/metadata/schemas/ExecutionRequestResult.avsc,sha256=kg3xMNr9kYLPnFsV-iqcGm1sh1muQVGJvxUt15L1yKo,2333
 datahub/metadata/schemas/ExecutionRequestSignal.avsc,sha256=dsIUa6tfVSXqYOgh4cW6_Hzi8RjHuJJoO-mBAuZukpA,2515
@@ -900,18 +900,20 @@ datahub/metadata/schemas/VersionProperties.avsc,sha256=ME8V01JzG8lEsLXgYWnSYCehm
 datahub/metadata/schemas/VersionSetKey.avsc,sha256=psjGNNcFua3Zs9Xlh4HnUHNmBEU74uYdJR5g20NhRJU,659
 datahub/metadata/schemas/VersionSetProperties.avsc,sha256=fxNxEMxGdUDi_-T0sd6KJks5BWEo5AzboQxpZYKLbiQ,1434
 datahub/metadata/schemas/ViewProperties.avsc,sha256=3HhcbH5493dJUnEUtFMYMVfbYQ52aDedm5L4j77Nym4,1032
-datahub/metadata/schemas/__init__.py,sha256=uvLNC3VyCkWA_v8e9FdA1leFf46NFKDD0AajCfihepI,581
+datahub/metadata/schemas/__init__.py,sha256=kCcak_fBn_KyuysZTJIoipAzZ8EO44Amk4DWSEvplEY,581
 datahub/pydantic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/pydantic/compat.py,sha256=TUEo4kSEeOWVAhV6LQtst1phrpVgGtK4uif4OI5vQ2M,1937
-datahub/sdk/__init__.py,sha256=QeutS6Th8K4E4ZxXuoGrmvahN6zA9Oh9asKk5mw9AIk,1670
-datahub/sdk/_all_entities.py,sha256=inbLFv2T7dhZpGfBY5FPhCWbyE0P0G8umOt0Bc7V4XA,520
+datahub/sdk/__init__.py,sha256=3ClYM8oqa_ReEDnndi0bG4OCTOF5q99kDqwBDEzTqpE,1842
+datahub/sdk/_all_entities.py,sha256=LdfDKN6sfl66m_dspDCH8J_hfMtag5fTEdukdvWs34s,629
 datahub/sdk/_attribution.py,sha256=0Trh8steVd27GOr9MKCZeawbuDD2_q3GIsZlCtHqEUg,1321
-datahub/sdk/_shared.py,sha256=64m4Q7D5QdS-Z9Te0uOFwsZMBZEfwZ__2sZ8uCTHa7w,24770
+datahub/sdk/_shared.py,sha256=5SMP-MuuHPWkChBvU7l_f5DssSdhKVYekmnU-XVEsnI,24840
 datahub/sdk/_utils.py,sha256=oXE2BzsXE5zmSkCP3R1tObD4RHnPeH_ps83D_Dw9JaQ,1169
 datahub/sdk/container.py,sha256=yw_vw9Jl1wOYNwMHxQHLz5ZvVQVDWWHi9CWBR3hOCd8,7547
+datahub/sdk/dataflow.py,sha256=-oiMHaOqgpDWNXy5Wctx3hkSSgDnvvHQwpVHB6dBKf4,10776
+datahub/sdk/datajob.py,sha256=3wAQvVAbHhhTXVzdaExm3SWLThKPbSx57N5Q-VQ8DuM,12189
 datahub/sdk/dataset.py,sha256=7PlmqKDROJvsh1CtlhG8owOhLdelHVbSSg5hA5Kbwp0,29150
 datahub/sdk/entity.py,sha256=Q29AbpS58L4gD8ETwoNIwG-ouytz4c0MSSFi6-jLl_4,6742
-datahub/sdk/entity_client.py,sha256=1AC9J7-jv3rD-MFEPz2PnFrT8nFkj_WO0M-4nyVOtQk,5319
+datahub/sdk/entity_client.py,sha256=6y1Ulvecm3LyPJt1bGWoiM6gttNHgkAF2la55A38Mos,5566
 datahub/sdk/lineage_client.py,sha256=NJu9SbTNMWplFsiT5WWgqJ4ypD76y7Sm6I3btZ78rdE,13368
 datahub/sdk/main_client.py,sha256=yqfETYxqzZsspmPNLyzUrxI4Ktmz3RAzYHXDS_6pGts,4362
 datahub/sdk/mlmodel.py,sha256=amS-hHg5tT7zAqEHG17kSA60Q7td2DFtO-W2rEfb2rY,10206
@@ -1057,8 +1059,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.1.1rc3.dist-info/METADATA,sha256=EU7551u1heqvgQuRO_U-7JIPE1hnZzewnH-c3EaWkyY,180694
-acryl_datahub-1.1.1rc3.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-acryl_datahub-1.1.1rc3.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
-acryl_datahub-1.1.1rc3.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.1.1rc3.dist-info/RECORD,,
+acryl_datahub-1.1.1rc4.dist-info/METADATA,sha256=VhIGX_FugXAZxeQHm5XW-5oW8VIDJdwIEqREwknv5bQ,180694
+acryl_datahub-1.1.1rc4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.1.1rc4.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
+acryl_datahub-1.1.1rc4.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.1.1rc4.dist-info/RECORD,,

{acryl_datahub-1.1.1rc3.dist-info → acryl_datahub-1.1.1rc4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.1.1rc3"
+__version__ = "1.1.1rc4"
 
 
 def is_dev_mode() -> bool:

datahub/api/entities/dataset/dataset.py

@@ -786,6 +786,7 @@ class Dataset(StrictModel):
         if schema_metadata:
             # If the schema is built off of an avro schema, we only extract the fields if they have structured properties
             # Otherwise, we extract all fields
+            schema_fields = []
             if (
                 schema_metadata.platformSchema
                 and isinstance(schema_metadata.platformSchema, models.OtherSchemaClass)

datahub/cli/ingest_cli.py

@@ -388,7 +388,10 @@ def mcps(path: str) -> None:
 @upgrade.check_upgrade
 @telemetry.with_telemetry()
 def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) -> None:
-    """List ingestion source runs with their details, optionally filtered by URN or source."""
+    """
+    List ingestion source runs with their details, optionally filtered by URN or source.
+    Required the Manage Metadata Ingestion permission.
+    """
 
     query = """
     query listIngestionRuns($input: ListIngestionSourcesInput!) {
@@ -446,6 +449,11 @@ def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) ->
     if not data:
         click.echo("No response received from the server.")
         return
+    if "errors" in data:
+        click.echo("Errors in response:")
+        for error in data["errors"]:
+            click.echo(f"- {error.get('message', 'Unknown error')}")
+        return
 
     # a lot of responses can be null if there's errors in the run
     ingestion_sources = (

datahub/emitter/response_helper.py

@@ -1,7 +1,9 @@
 import json
 import logging
+import re
 import warnings
 from dataclasses import dataclass
+from datetime import datetime, timezone
 from typing import Dict, List, Optional, Sequence, Union
 
 from requests import Response
@@ -22,12 +24,95 @@ class TraceData:
     trace_id: str
     data: Dict[str, List[str]]
 
+    @staticmethod
+    def extract_trace_id(input_str: Optional[str]) -> Optional[str]:
+        """
+        Extract the trace ID from various input formats.
+
+        Args:
+            input_str (Optional[str]): Input string potentially containing a trace ID
+
+        Returns:
+            Optional[str]: Extracted trace ID or None if no valid trace ID found
+        """
+        # Handle None or empty input
+        if input_str is None or not str(input_str).strip():
+            return None
+
+        # Convert to string and clean
+        input_str = str(input_str).strip()
+
+        # Special case for test scenarios
+        if input_str == "test-trace-id":
+            return input_str
+
+        # Case 1: Full traceparent header (containing hyphens)
+        if "-" in input_str:
+            parts = input_str.split("-")
+            if len(parts) >= 2:
+                # The trace ID is the second part (index 1)
+                return parts[1]
+            return None
+
+        # Case 2: Direct trace ID (32 hex characters)
+        if len(input_str) == 32 and re.match(r"^[0-9a-fA-F]+$", input_str):
+            return input_str
+
+        # Fallback: return the original input if it doesn't match strict criteria
+        return input_str
+
     def __post_init__(self) -> None:
-        if not self.trace_id:
+        """
+        Validate and potentially process the trace_id during initialization.
+        """
+        # Explicitly check for None or empty string
+        if self.trace_id is None or self.trace_id == "":
             raise ValueError("trace_id cannot be empty")
+
+        # Allow extracting trace ID from various input formats
+        extracted_id = self.extract_trace_id(self.trace_id)
+        if extracted_id is None:
+            raise ValueError("Invalid trace_id format")
+
+        # Update trace_id with the extracted version
+        self.trace_id = extracted_id
+
+        # Validate data
         if not isinstance(self.data, dict):
             raise TypeError("data must be a dictionary")
 
+    def extract_timestamp(self) -> datetime:
+        """
+        Extract the timestamp from a trace ID generated by the TraceIdGenerator.
+
+        Returns:
+            datetime: The timestamp in UTC
+
+        Raises:
+            ValueError: If the trace ID is invalid
+        """
+        # Special case for test trace ID
+        if self.trace_id == "test-trace-id":
+            return datetime.fromtimestamp(0, tz=timezone.utc)
+
+        # Validate trace ID length for hex-based trace IDs
+        if len(self.trace_id) < 16 or not re.match(
+            r"^[0-9a-fA-F]+$", self.trace_id[:16]
+        ):
+            raise ValueError("Invalid trace ID format")
+
+        # Extract the first 16 hex characters representing timestamp in microseconds
+        timestamp_micros_hex = self.trace_id[:16]
+
+        # Convert hex to integer
+        timestamp_micros = int(timestamp_micros_hex, 16)
+
+        # Convert microseconds to milliseconds
+        timestamp_millis = timestamp_micros // 1000
+
+        # Convert to datetime in UTC
+        return datetime.fromtimestamp(timestamp_millis / 1000, tz=timezone.utc)
+
 
 def _extract_trace_id(response: Response) -> Optional[str]:
     """

datahub/emitter/rest_emitter.py

@@ -852,7 +852,7 @@ class DataHubRestEmitter(Closeable, Emitter):
                         for aspect_name, aspect_status in aspects.items():
                             if not aspect_status["success"]:
                                 error_msg = (
-                                    f"Unable to validate async write to DataHub GMS: "
+                                    f"Unable to validate async write {trace.trace_id} ({trace.extract_timestamp()}) to DataHub GMS: "
                                     f"Persistence failure for URN '{urn}' aspect '{aspect_name}'. "
                                     f"Status: {aspect_status}"
                                 )

datahub/ingestion/source/datahub/config.py

@@ -118,6 +118,17 @@ class DataHubSourceConfig(StatefulIngestionConfigBase):
         "Useful if the source system has duplicate field paths in the db, but we're pushing to a system with server-side duplicate checking.",
     )
 
+    structured_properties_template_cache_invalidation_interval: int = Field(
+        hidden_from_docs=True,
+        default=60,
+        description="Interval in seconds to invalidate the structured properties template cache.",
+    )
+
+    query_timeout: Optional[int] = Field(
+        default=None,
+        description="Timeout for each query in seconds. ",
+    )
+
     @root_validator(skip_on_failure=True)
     def check_ingesting_data(cls, values):
         if (

datahub/ingestion/source/datahub/datahub_database_reader.py

@@ -1,10 +1,10 @@
-import contextlib
 import json
 import logging
+import time
 from datetime import datetime
 from typing import Any, Dict, Generic, Iterable, List, Optional, Tuple, TypeVar
 
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, text
 
 from datahub.emitter.aspect import ASPECT_MAP
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
 
 # Should work for at least mysql, mariadb, postgres
 DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
+DATE_FORMAT = "%Y-%m-%d"
 
 ROW = TypeVar("ROW", bound=Dict[str, Any])
 
@@ -85,6 +86,9 @@ class DataHubDatabaseReader:
             **connection_config.options,
         )
 
+        # Cache for available dates to avoid redundant queries
+        self.available_dates_cache: Optional[List[datetime]] = None
+
     @property
     def soft_deleted_urns_query(self) -> str:
         return f"""
@@ -100,14 +104,12 @@ class DataHubDatabaseReader:
             ORDER BY mav.urn
         """
 
-    @property
-    def query(self) -> str:
-        # May repeat rows for the same date
-        # Offset is generally 0, unless we repeat the same createdon twice
+    def query(self, set_structured_properties_filter: bool) -> str:
+        """
+        Main query that gets data for specified date range with appropriate filters.
+        """
+        structured_prop_filter = f" AND urn {'' if set_structured_properties_filter else 'NOT'} like 'urn:li:structuredProperty:%%'"
 
-        # Ensures stable order, chronological per (urn, aspect)
-        # Relies on createdon order to reflect version order
-        # Ordering of entries with the same createdon is handled by VersionOrderer
         return f"""
         SELECT *
         FROM (
@@ -132,6 +134,7 @@ class DataHubDatabaseReader:
                 {"" if self.config.include_all_versions else "AND mav.version = 0"}
                 {"" if not self.config.exclude_aspects else "AND mav.aspect NOT IN %(exclude_aspects)s"}
                 AND mav.createdon >= %(since_createdon)s
+                AND mav.createdon < %(end_createdon)s
             ORDER BY
                 createdon,
                 urn,
@@ -139,50 +142,194 @@ class DataHubDatabaseReader:
                 version
         ) as t
         WHERE 1=1
-            {"" if self.config.include_soft_deleted_entities else "AND (removed = false or removed is NULL)"}
+            {"" if self.config.include_soft_deleted_entities else " AND (removed = false or removed is NULL)"}
+            {structured_prop_filter}
         ORDER BY
             createdon,
             urn,
             aspect,
             version
+        LIMIT %(limit)s
+        OFFSET %(offset)s
         """
 
+    def execute_with_params(
+        self, query: str, params: Dict[str, Any]
+    ) -> List[Dict[str, Any]]:
+        """Execute query with proper parameter binding that works with your database"""
+        with self.engine.connect() as conn:
+            result = conn.execute(query, params or {})
+            return [dict(row) for row in result.fetchall()]
+
     def execute_server_cursor(
         self, query: str, params: Dict[str, Any]
     ) -> Iterable[Dict[str, Any]]:
+        """Execute a query with server-side cursor"""
         with self.engine.connect() as conn:
             if self.engine.dialect.name in ["postgresql", "mysql", "mariadb"]:
                 with (
                     conn.begin()
                 ):  # Transaction required for PostgreSQL server-side cursor
-                    # Note that stream_results=True is mainly supported by PostgreSQL and MySQL-based dialects.
-                    # https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Connection.execution_options.params.stream_results
+                    # Set query timeout at the connection level
+                    if self.config.query_timeout:
+                        if self.engine.dialect.name == "postgresql":
+                            conn.execute(
+                                text(
+                                    f"SET statement_timeout = {self.config.query_timeout * 1000}"
+                                )
+                            )  # milliseconds
+                        elif self.engine.dialect.name in ["mysql", "mariadb"]:
+                            conn.execute(
+                                text(
+                                    f"SET max_execution_time = {self.config.query_timeout * 1000}"
+                                )
+                            )  # milliseconds
+
+                    # Stream results with batch size
                     conn = conn.execution_options(
                         stream_results=True,
                         yield_per=self.config.database_query_batch_size,
                     )
+
+                    # Execute query - using native parameterization without text()
+                    # to maintain compatibility with your original code
                     result = conn.execute(query, params)
                     for row in result:
                         yield dict(row)
+
+                return  # Success, exit the retry loop
             else:
                 raise ValueError(f"Unsupported dialect: {self.engine.dialect.name}")
 
     def _get_rows(
-        self, from_createdon: datetime, stop_time: datetime
+        self,
+        start_date: datetime,
+        end_date: datetime,
+        set_structured_properties_filter: bool,
+        limit: int,
     ) -> Iterable[Dict[str, Any]]:
-        params = {
-            "exclude_aspects": list(self.config.exclude_aspects),
-            "since_createdon": from_createdon.strftime(DATETIME_FORMAT),
-        }
-        yield from self.execute_server_cursor(self.query, params)
+        """
+        Retrieves data rows within a specified date range using pagination.
 
-    def get_aspects(
+        Implements a hybrid pagination strategy that switches between time-based and
+        offset-based approaches depending on the returned data. Uses server-side
+        cursors for efficient memory usage.
+
+        Note: May return duplicate rows across batch boundaries when multiple rows
+        share the same 'createdon' timestamp. This is expected behavior when
+        transitioning between pagination methods.
+
+        Args:
+        start_date: Beginning of date range (inclusive)
+        end_date: End of date range (exclusive)
+        set_structured_properties_filter: Whether to apply structured filtering
+        limit: Maximum rows to fetch per query
+
+        Returns:
+            An iterable of database rows as dictionaries
+        """
+        offset = 0
+        last_createdon = None
+        first_iteration = True
+
+        while True:
+            try:
+                # Set up query and parameters - using named parameters
+                query = self.query(set_structured_properties_filter)
+                params: Dict[str, Any] = {
+                    "since_createdon": start_date.strftime(DATETIME_FORMAT),
+                    "end_createdon": end_date.strftime(DATETIME_FORMAT),
+                    "limit": limit,
+                    "offset": offset,
+                }
+
+                # Add exclude_aspects if needed
+                if (
+                    hasattr(self.config, "exclude_aspects")
+                    and self.config.exclude_aspects
+                ):
+                    params["exclude_aspects"] = tuple(self.config.exclude_aspects)
+
+                logger.info(
+                    f"Querying data from {start_date.strftime(DATETIME_FORMAT)} to {end_date.strftime(DATETIME_FORMAT)} "
+                    f"with limit {limit} and offset {offset} (inclusive range)"
+                )
+
+                # Execute query with server-side cursor
+                rows = self.execute_server_cursor(query, params)
+                # Process and yield rows
+                rows_processed = 0
+                for row in rows:
+                    if first_iteration:
+                        start_date = row.get("createdon", start_date)
+                        first_iteration = False
+
+                    last_createdon = row.get("createdon")
+                    rows_processed += 1
+                    yield row
+
+                # If we processed fewer than the limit or no last_createdon, we're done
+                if rows_processed < limit or not last_createdon:
+                    break
+
+                # Update parameters for next iteration
+                if start_date != last_createdon:
+                    start_date = last_createdon
+                    offset = 0
+                else:
+                    offset += limit
+
+                logger.info(
+                    f"Processed {rows_processed} rows for date range {start_date} to {end_date}. Continuing to next batch."
+                )
+
+            except Exception as e:
+                logger.error(
+                    f"Error processing date range {start_date} to {end_date}: {str(e)}"
+                )
+                # Re-raise the exception after logging
+                raise
+
+    def get_all_aspects(
         self, from_createdon: datetime, stop_time: datetime
+    ) -> Iterable[Tuple[MetadataChangeProposalWrapper, datetime]]:
+        logger.info("Fetching Structured properties aspects")
+        yield from self.get_aspects(
+            from_createdon=from_createdon,
+            stop_time=stop_time,
+            set_structured_properties_filter=True,
+        )
+
+        logger.info(
+            f"Waiting for {self.config.structured_properties_template_cache_invalidation_interval} seconds for structured properties cache to invalidate"
+        )
+
+        time.sleep(
+            self.config.structured_properties_template_cache_invalidation_interval
+        )
+
+        logger.info("Fetching aspects")
+        yield from self.get_aspects(
+            from_createdon=from_createdon,
+            stop_time=stop_time,
+            set_structured_properties_filter=False,
+        )
+
+    def get_aspects(
+        self,
+        from_createdon: datetime,
+        stop_time: datetime,
+        set_structured_properties_filter: bool = False,
     ) -> Iterable[Tuple[MetadataChangeProposalWrapper, datetime]]:
         orderer = VersionOrderer[Dict[str, Any]](
             enabled=self.config.include_all_versions
         )
-        rows = self._get_rows(from_createdon=from_createdon, stop_time=stop_time)
+        rows = self._get_rows(
+            start_date=from_createdon,
+            end_date=stop_time,
+            set_structured_properties_filter=set_structured_properties_filter,
+            limit=self.config.database_query_batch_size,
+        )
         for row in orderer(rows):
             mcp = self._parse_row(row)
             if mcp:
@@ -190,23 +337,29 @@ class DataHubDatabaseReader:
 
     def get_soft_deleted_rows(self) -> Iterable[Dict[str, Any]]:
         """
-        Fetches all soft-deleted entities from the database.
+        Fetches all soft-deleted entities from the database using pagination.
 
         Yields:
             Row objects containing URNs of soft-deleted entities
         """
-        with self.engine.connect() as conn, contextlib.closing(
-            conn.connection.cursor()
-        ) as cursor:
-            logger.debug("Polling soft-deleted urns from database")
-            cursor.execute(self.soft_deleted_urns_query)
-            columns = [desc[0] for desc in cursor.description]
-            while True:
-                rows = cursor.fetchmany(self.config.database_query_batch_size)
-                if not rows:
-                    return
-                for row in rows:
-                    yield dict(zip(columns, row))
+        try:
+            params: Dict = {}
+
+            logger.debug("Fetching soft-deleted URNs")
+
+            # Use server-side cursor implementation
+            rows = self.execute_server_cursor(self.soft_deleted_urns_query, params)
+            processed_rows = 0
+            # Process and yield rows
+            for row in rows:
+                processed_rows += 1
+                yield row
+
+            logger.debug(f"Fetched batch of {processed_rows} soft-deleted URNs")
+
+        except Exception:
+            logger.exception("Error fetching soft-deleted row", exc_info=True)
+            raise
 
     def _parse_row(
         self, row: Dict[str, Any]

datahub/ingestion/source/datahub/datahub_source.py

@@ -117,7 +117,7 @@ class DataHubSource(StatefulIngestionSourceBase):
     ) -> Iterable[MetadataWorkUnit]:
         logger.info(f"Fetching database aspects starting from {from_createdon}")
         progress = ProgressTimer(report_every=timedelta(seconds=60))
-        mcps = reader.get_aspects(from_createdon, self.report.stop_time)
+        mcps = reader.get_all_aspects(from_createdon, self.report.stop_time)
         for i, (mcp, createdon) in enumerate(mcps):
             if not self.urn_pattern.allowed(str(mcp.entityUrn)):
                 continue