acryl-datahub 1.3.0.1rc6__py3-none-any.whl → 1.3.0.1rc8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (35) hide show
  1. {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc8.dist-info}/METADATA +2679 -2680
  2. {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc8.dist-info}/RECORD +35 -33
  3. datahub/_version.py +1 -1
  4. datahub/cli/docker_check.py +1 -1
  5. datahub/emitter/mce_builder.py +6 -0
  6. datahub/ingestion/autogenerated/capability_summary.json +12 -12
  7. datahub/ingestion/source/bigquery_v2/bigquery_queries.py +2 -0
  8. datahub/ingestion/source/common/subtypes.py +2 -0
  9. datahub/ingestion/source/dremio/dremio_source.py +15 -15
  10. datahub/ingestion/source/dynamodb/dynamodb.py +1 -1
  11. datahub/ingestion/source/fivetran/config.py +33 -0
  12. datahub/ingestion/source/fivetran/fivetran.py +184 -13
  13. datahub/ingestion/source/fivetran/fivetran_log_api.py +20 -5
  14. datahub/ingestion/source/fivetran/fivetran_rest_api.py +65 -0
  15. datahub/ingestion/source/fivetran/response_models.py +97 -0
  16. datahub/ingestion/source/hex/hex.py +1 -1
  17. datahub/ingestion/source/iceberg/iceberg.py +1 -1
  18. datahub/ingestion/source/metabase.py +23 -4
  19. datahub/ingestion/source/mlflow.py +1 -1
  20. datahub/ingestion/source/s3/source.py +1 -1
  21. datahub/ingestion/source/salesforce.py +1 -1
  22. datahub/ingestion/source/slack/slack.py +1 -1
  23. datahub/ingestion/source/snowflake/snowflake_queries.py +3 -0
  24. datahub/ingestion/source/snowflake/snowflake_summary.py +1 -1
  25. datahub/ingestion/source/sql/mssql/source.py +41 -9
  26. datahub/ingestion/source/sql_queries.py +1 -1
  27. datahub/ingestion/source/unity/source.py +1 -1
  28. datahub/ingestion/source/vertexai/vertexai.py +1 -1
  29. datahub/metadata/schema.avsc +4 -2
  30. datahub/metadata/schemas/DataHubFileInfo.avsc +4 -2
  31. datahub/sdk/mlmodel.py +19 -0
  32. {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc8.dist-info}/WHEEL +0 -0
  33. {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc8.dist-info}/entry_points.txt +0 -0
  34. {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc8.dist-info}/licenses/LICENSE +0 -0
  35. {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc8.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
- acryl_datahub-1.3.0.1rc6.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
1
+ acryl_datahub-1.3.0.1rc8.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
2
2
  datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
3
3
  datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
4
- datahub/_version.py,sha256=PW9A4Uazfqf_qZ54rH_cG6i8GhrmicechIGtJYipA8Q,323
4
+ datahub/_version.py,sha256=3eJGiXMGVkMi1lpj9ZplnwELXbksY0tQWUHU5_VpRwU,323
5
5
  datahub/entrypoints.py,sha256=VcbU6Z47b_JKW1zI-WJMYIngm05FSogKLiuvFNtyNcI,9088
6
6
  datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
7
7
  datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -72,7 +72,7 @@ datahub/cli/cli_utils.py,sha256=0jTTAKuDZ8GzZwGHYytcT_MPR3Rb2DAcbr9n1H2T2sE,1617
72
72
  datahub/cli/config_utils.py,sha256=wzCKcqXKSdc9P-hzvhzEVb0rB2kI2x5B2_MrRiME05M,4864
73
73
  datahub/cli/container_cli.py,sha256=D0zWP3_3aww8_RTkMugOoOlILz3dPJ0TE9asQDLCm6E,1697
74
74
  datahub/cli/delete_cli.py,sha256=0YJeWuXPGY0kbSn1AXK1-8SfCGBxb78ZbO53RAgyjQg,26515
75
- datahub/cli/docker_check.py,sha256=51aLFrtv38SRoAjHPS0aQy7ekchqBqnonLFgOyCY4xw,13020
75
+ datahub/cli/docker_check.py,sha256=0LBY1VmdQG1wLR0HFgSFLNz23NahCQXuy7YOQKUnF48,13018
76
76
  datahub/cli/docker_cli.py,sha256=EdW6OgPf-ZiqTSAwqz6cxTG1AmxlPbBonWl2GcuhWRk,33178
77
77
  datahub/cli/env_utils.py,sha256=RQzjg4JE29hjPt4v7p-RuqoOr99w8E3DBHWiN2Sm7T4,252
78
78
  datahub/cli/exists_cli.py,sha256=1cUYNh3GqNgVHWTrfMRGJoo9tFZNXcLetMaDbLaig6o,1233
@@ -128,7 +128,7 @@ datahub/emitter/composite_emitter.py,sha256=ZU-IdlAXKGPtmyT0JJgYC09vRn-TmeNaA6VP
128
128
  datahub/emitter/enum_helpers.py,sha256=QBOEUu_hDCvyL_v4ayNQV8XwJbf5zKyu0Xat0mI1Kgo,376
129
129
  datahub/emitter/generic_emitter.py,sha256=i37ZFm9VR_tmiZm9kIypEkQEB_cLKbzj_tJvViN-fm8,828
130
130
  datahub/emitter/kafka_emitter.py,sha256=Uix1W1WaXF8VqUTUfzdRZKca2XrR1w50Anx2LVkROlc,5822
131
- datahub/emitter/mce_builder.py,sha256=S8M7gMWjechjz5oGydrIkYBHEPRosbhj-z8LMWy8yq4,16849
131
+ datahub/emitter/mce_builder.py,sha256=SR4hiL2kT10_ilYBh9WTVssA-NjIq7pA6wMq7D8_hK0,17047
132
132
  datahub/emitter/mcp.py,sha256=u6LphyhpbdFqboTAL_9MzXhGjc45o_BePoDFBkEEYWo,10484
133
133
  datahub/emitter/mcp_builder.py,sha256=8IwJAlolQkPpMqQJPLtGrsUqAcuFNs98nrI5iYUxgaU,11920
134
134
  datahub/emitter/mcp_patch_builder.py,sha256=u7cpW6DkiN7KpLapmMaXgL_FneoN69boxiANbVgMdSI,4564
@@ -162,7 +162,7 @@ datahub/ingestion/api/auto_work_units/auto_dataset_properties_aspect.py,sha256=I
162
162
  datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py,sha256=-667F-xWArmeVaW-3mGsoIMNKPrnLx6KM4OfzMGsMOo,18064
163
163
  datahub/ingestion/api/auto_work_units/auto_validate_input_fields.py,sha256=Xm7lhEPgicK_Isa2p0GtFbGCLJo5xpzmLc4AYEJPic0,3559
164
164
  datahub/ingestion/autogenerated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
165
- datahub/ingestion/autogenerated/capability_summary.json,sha256=9Ns5gYfVq0LYogaYtb0ioDPfu8SVhftiq9R7l0irQwg,111506
165
+ datahub/ingestion/autogenerated/capability_summary.json,sha256=n3VuRCSA_0pCMbtndm3zd57oLFtTVm8pQIeW_u-MIGk,111518
166
166
  datahub/ingestion/autogenerated/lineage.json,sha256=8BdZF-5V5kJbX4mfFav8Zg-jHjzfkAEGk-pu1atLN4I,10029
167
167
  datahub/ingestion/autogenerated/lineage_helper.py,sha256=I_k1pZSCCCjDbUVifPTfy6fkmV8jqdVhbirE8EkpmxI,4748
168
168
  datahub/ingestion/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -218,8 +218,8 @@ datahub/ingestion/source/ge_data_profiler.py,sha256=9lEQdLcMBa7znqa6Zz-QWA4Uiv8K
218
218
  datahub/ingestion/source/ge_profiling_config.py,sha256=FIuZtce0gRncSRKA1V9GLg8H5JyJPieZweFJS36Q_CI,11523
219
219
  datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suabri1yV5geaAPjzg2eORDc,2559
220
220
  datahub/ingestion/source/ldap.py,sha256=PKoA5pVjuIxFfW1TcbYNIWSm7-C7shK2FDn7Zo5mrVM,18705
221
- datahub/ingestion/source/metabase.py,sha256=txzrTtxD1hla3yspxY6GQRCZUFWOK03d0-wJqDmT9AQ,32695
222
- datahub/ingestion/source/mlflow.py,sha256=t7heUgivLXU7lxc-ndZxc1LZuoDKZgpSIe-x3ExXfMg,33340
221
+ datahub/ingestion/source/metabase.py,sha256=udWYNEomLVo5RjWUOlbq9eI43Pjhaf56OVF_YZBHBl0,33258
222
+ datahub/ingestion/source/mlflow.py,sha256=sAXOAxgrKOMQPkaBj3KVtlpf9qfhYy9J1op1JaNVXs4,33343
223
223
  datahub/ingestion/source/mode.py,sha256=VF3ps9qk5I4LZiAAh-TEDFj2mHmMic1VdbTBxkGPYOc,72748
224
224
  datahub/ingestion/source/mongodb.py,sha256=ykUA2Jyn0rxzOO-pCWosOqvFTIRgFmqkCTRHAsxpOYc,21423
225
225
  datahub/ingestion/source/nifi.py,sha256=waXqfxHp1aEZ-JzU-EVU6EIcPepI93tP1vCz8RbgCOg,56882
@@ -228,9 +228,9 @@ datahub/ingestion/source/openapi_parser.py,sha256=T87e2r-oPGgQl_FDMHnSGFZzApvWDC
228
228
  datahub/ingestion/source/preset.py,sha256=opltXJmopFsTpIDFdFVQsuZh5aqT41EETPymKoF1JxA,3949
229
229
  datahub/ingestion/source/pulsar.py,sha256=u5F8QnCLJsht5-7XCiUTsnfhCPIpKVB_l32CgMCU-As,20187
230
230
  datahub/ingestion/source/redash.py,sha256=C4cDikWymbL88fDqaIPX5WA3f2sIEtH7bmhJKkmXJsM,30652
231
- datahub/ingestion/source/salesforce.py,sha256=dMQ2jMu9P8r0rmREQA6KuFgAbegJ7WnHpUmMaUfHPDI,40942
231
+ datahub/ingestion/source/salesforce.py,sha256=0oTpAXys9nEQYzrbyFJIH60GaqsskDR5XPNqd3KSptY,40941
232
232
  datahub/ingestion/source/source_registry.py,sha256=a2mLjJPLkSI-gYCTb_7U7Jo4D8jGknNQ_yScPIihXFk,1208
233
- datahub/ingestion/source/sql_queries.py,sha256=RexNsG-COCiKNyL0mfoeNoMrkXxmB5UBOSJKJnHNHy0,14563
233
+ datahub/ingestion/source/sql_queries.py,sha256=xBJfoBoStB0BDk8G68UUfwE2qUiLyq5hbNYelWFkDrY,14581
234
234
  datahub/ingestion/source/superset.py,sha256=qZ1SMeejwiM_ZkPlERXi3OSmabWcrsJSxZe98Eo9tqA,57866
235
235
  datahub/ingestion/source/abs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
236
236
  datahub/ingestion/source/abs/config.py,sha256=WW9JWbzqAJDblAcJKtNeuBHqOeJsB57lW2PqSD65-BU,6729
@@ -268,7 +268,7 @@ datahub/ingestion/source/bigquery_v2/bigquery_connection.py,sha256=6XFCc0oxxU3R4
268
268
  datahub/ingestion/source/bigquery_v2/bigquery_data_reader.py,sha256=DeT3v_Z82__8En0FcZ0kavBAWQoRvSZ5Rppm9eeDAb8,2393
269
269
  datahub/ingestion/source/bigquery_v2/bigquery_helper.py,sha256=QER3gY8e_k1_eNVj7cBso7ZzrWl_vO5PYSa6CpvqNx8,1554
270
270
  datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py,sha256=9_sfX8BE2vt9RjBMyq27UxCxBaSlD5o3L4gQxrwlPvA,4961
271
- datahub/ingestion/source/bigquery_v2/bigquery_queries.py,sha256=2syDMaRpYEbtGUVejVAK5d6g8HqM54ZyEM908uLJ55o,3393
271
+ datahub/ingestion/source/bigquery_v2/bigquery_queries.py,sha256=VdYe2PYGq4Z8VSuwHfiSSx7bgoqQe2xsXWRHy1sQoQo,3509
272
272
  datahub/ingestion/source/bigquery_v2/bigquery_report.py,sha256=zlTkqOmt5zxnO40rVTYHF3fclj4OVlLtqUXwW5WIIcM,7855
273
273
  datahub/ingestion/source/bigquery_v2/bigquery_schema.py,sha256=zbYb1EYnCJxgvsU8oT_76l0q_BW1exVjMWM1GAgd1nc,32600
274
274
  datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py,sha256=PbSCMj5ACwEu_HQNe29IHs4y1bn15_nnz6ZW1Yt17wI,51796
@@ -289,7 +289,7 @@ datahub/ingestion/source/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
289
289
  datahub/ingestion/source/common/data_platforms.py,sha256=HhuP3YIEi2WpyKDjUU8RiM0a2qjHWQcvc8kcqub0cVo,548
290
290
  datahub/ingestion/source/common/data_reader.py,sha256=XbSxiRTYrk6seOz0ZjVjzSpGvP8lEjmqXrNI4cdYYmQ,1819
291
291
  datahub/ingestion/source/common/gcp_credentials_config.py,sha256=e1VgUC5mnmgSOnhtxAR6Z5HdLDqXWpA543osBi-0cbE,2431
292
- datahub/ingestion/source/common/subtypes.py,sha256=ZVeDW2MjEum4ueT0UAKib6RqNCXsgAkhnkivDvaGJfs,4759
292
+ datahub/ingestion/source/common/subtypes.py,sha256=yUduH43tGxgK8H2OYTzyRs6dXUH2KmhjuCFJHNYGaH4,4855
293
293
  datahub/ingestion/source/data_lake_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
294
294
  datahub/ingestion/source/data_lake_common/config.py,sha256=qUk83B01hjuBKHvVz8SmXnVCy5eFj-2-2QLEOrAdbgk,359
295
295
  datahub/ingestion/source/data_lake_common/data_lake_utils.py,sha256=IYr5y8vy_6CtMtITqzn6OqovzH1cpe1i30M-75PouXo,7768
@@ -323,11 +323,11 @@ datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py,sha256=MQk8BAHL
323
323
  datahub/ingestion/source/dremio/dremio_entities.py,sha256=1gZrNqTp3Pm6vqGDQaWt3HkxEuHKxpGYQ4geVoFvxWI,15147
324
324
  datahub/ingestion/source/dremio/dremio_profiling.py,sha256=TAcnpo8ZRKhLDHnQSJzJg3YdwTSyEa73LUAzENs7wG4,12287
325
325
  datahub/ingestion/source/dremio/dremio_reporting.py,sha256=UEj-6FMdIWsry5535_kM2hLze5aPRMatTwvI0Bd2BSo,2140
326
- datahub/ingestion/source/dremio/dremio_source.py,sha256=JFgzQiYcIkKcyiTJstYdfNxIVCGAcbi198kPQ55bYso,25596
326
+ datahub/ingestion/source/dremio/dremio_source.py,sha256=6QHiFtfNiLiiYDCotUGRMsfWcgsd2ZyVOavimXyx6W4,25416
327
327
  datahub/ingestion/source/dremio/dremio_sql_queries.py,sha256=wA1hqKk9cKMJDyEdZRQcDDLZPGYwuNqrvleUHTkWgrQ,10508
328
328
  datahub/ingestion/source/dynamodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
329
329
  datahub/ingestion/source/dynamodb/data_reader.py,sha256=vC77KpcP8LJN0g8wsPRDVw4sebv0ZWIP3tJkEIHaomA,3120
330
- datahub/ingestion/source/dynamodb/dynamodb.py,sha256=k7lBnoUAobLwpHkgbW358XbqmVHaR_NZbQ-9CNiDcrk,22928
330
+ datahub/ingestion/source/dynamodb/dynamodb.py,sha256=32Bt9y9-R60MCGt9gYikNZjrjRxCQZDmD0ra9by5EyM,22931
331
331
  datahub/ingestion/source/excel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
332
332
  datahub/ingestion/source/excel/config.py,sha256=4x7GCpGaESXR8E1VH5ug53Dqh9tbyjbueaJHi3a_E_E,3604
333
333
  datahub/ingestion/source/excel/excel_file.py,sha256=j-wr012nd6HhY-vbTfi7mY13mHJ0evV2OjUAYmHNcz4,18296
@@ -336,11 +336,13 @@ datahub/ingestion/source/excel/report.py,sha256=oEkeI8J6is7zB9iz4RqASu_-Q5xl36lA
336
336
  datahub/ingestion/source/excel/source.py,sha256=w_vOz4UD7BcXBBDKoo81_6-QFeOPITuXqkfjIMHCQj4,23827
337
337
  datahub/ingestion/source/excel/util.py,sha256=YYmadYuCiT-4_MfQM0YSE7wuDcE0k8o2KrlOKM9Z6eI,406
338
338
  datahub/ingestion/source/fivetran/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
339
- datahub/ingestion/source/fivetran/config.py,sha256=6yriUMtTPMZUHqbZ9gzyFduPVt6CxzirdYSg4k-ziYI,10285
339
+ datahub/ingestion/source/fivetran/config.py,sha256=wGoGsh8oCRTlENOoD5pMq6aFBl0S3A7D8AXF1OGS_Dg,11759
340
340
  datahub/ingestion/source/fivetran/data_classes.py,sha256=ecdUJH5BEze0yv-uFpKWPNaNmV1gORDA2XMFk0zhcBw,595
341
- datahub/ingestion/source/fivetran/fivetran.py,sha256=Up5wbLk7hBk9b0pqcHwW6b0H52UJj90cmLhn0QJeZ4g,14416
342
- datahub/ingestion/source/fivetran/fivetran_log_api.py,sha256=-ibtfgxFv08P5_X5PVqV4CocxAjRWmY858esQL5OaAQ,13697
341
+ datahub/ingestion/source/fivetran/fivetran.py,sha256=8uPD5qNdA-BjG8pmjxVYf2_oOWVLzpg5fiaIYQGJO38,22685
342
+ datahub/ingestion/source/fivetran/fivetran_log_api.py,sha256=WaDqnFoK6wo737V28iAzY5fqxYc7ch5HrTngSva8j9Q,14320
343
343
  datahub/ingestion/source/fivetran/fivetran_query.py,sha256=VJTka6cdIzlqy0aWyviMO2uSHcL0ZQFTCefUnnjv_Bk,6578
344
+ datahub/ingestion/source/fivetran/fivetran_rest_api.py,sha256=10laMOEc6kPxZcStDhWnwQNrohNVud1SywAo5Hl1kU0,2117
345
+ datahub/ingestion/source/fivetran/response_models.py,sha256=5D0fPuWRSSGah6UXUF966dCQiK-VUwlqDNIonasPRz0,3355
344
346
  datahub/ingestion/source/gc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
345
347
  datahub/ingestion/source/gc/datahub_gc.py,sha256=nJ6QbHnTPL0MamWxNTZA26FMZsnmvUdUwnQcDS81K9s,12723
346
348
  datahub/ingestion/source/gc/dataprocess_cleanup.py,sha256=mUWcMt-_FL1SYGIgI4lGZDZGXspUUTv__5GN1W2oJ3s,17118
@@ -364,12 +366,12 @@ datahub/ingestion/source/grafana/types.py,sha256=Bz0-FIPBXHaBjfFHYGJhE20c2vYZwAs
364
366
  datahub/ingestion/source/hex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
365
367
  datahub/ingestion/source/hex/api.py,sha256=rKr6GmhIs98_SQZYG4egZlS049WH4ZxW2-7Ueq8692Q,12940
366
368
  datahub/ingestion/source/hex/constants.py,sha256=8hUTMWyG5keTNfXoLu_Dh413Hw_mGGJX1atiiDZyKtg,271
367
- datahub/ingestion/source/hex/hex.py,sha256=NaMibmvFXoCD84G0-asEDILbiluZyulbZZg23moP0hI,13328
369
+ datahub/ingestion/source/hex/hex.py,sha256=rra2-6ICNkSlrnO4_SFDTIValgmTaoOdDObr3q2CChk,13331
368
370
  datahub/ingestion/source/hex/mapper.py,sha256=IyDAE-TzZUji3ICI_9gkYC3dQN3gl6kERRWNVRk80fQ,13905
369
371
  datahub/ingestion/source/hex/model.py,sha256=eri4aRo1eXcE2SWjzCnPFMhzPTiJ8w8zC4GN7Lgpr74,1864
370
372
  datahub/ingestion/source/hex/query_fetcher.py,sha256=r9UvF_qwswkRlNY7AI8p46eqAYSxVtjVE2e7eO4XagA,13384
371
373
  datahub/ingestion/source/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
372
- datahub/ingestion/source/iceberg/iceberg.py,sha256=ySHJZFyYuf-0S0-mDBTvnNGS7UpPiNa1OrLIFzf-tgE,37941
374
+ datahub/ingestion/source/iceberg/iceberg.py,sha256=JdFTVHDGJra1-L-z3JGBAT-PH978NEZZ5ZxGU93N1vA,37944
373
375
  datahub/ingestion/source/iceberg/iceberg_common.py,sha256=CD_yHQ_wEgivyLQUTRO9BZJB29S7j5fUVllki-BPwUU,12292
374
376
  datahub/ingestion/source/iceberg/iceberg_profiler.py,sha256=24zhWNa-x8dismkdGWuutruRN4VLWkM-4doTFH9-SEU,9936
375
377
  datahub/ingestion/source/identity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -464,7 +466,7 @@ datahub/ingestion/source/s3/config.py,sha256=lElFXgEpKDT9SVoiXvtx98wV6Gp880qP4pL
464
466
  datahub/ingestion/source/s3/datalake_profiler_config.py,sha256=FfrcgK-JEF94vw-l3q6pN6FENXb-wZzW2w1VUZVkwW8,3620
465
467
  datahub/ingestion/source/s3/profiling.py,sha256=yKNCKpr6w7qpCH-baeSkNE9VjkN6eBot_weD-2_Jxzk,17579
466
468
  datahub/ingestion/source/s3/report.py,sha256=9Ej1UCChw963UpGw1-7asi5vFrOM232gfgG8bRdKPp0,667
467
- datahub/ingestion/source/s3/source.py,sha256=rokdUHb2rPXc6JbHwq5GDYdK4COlhm6_E84jHNk-EAE,58063
469
+ datahub/ingestion/source/s3/source.py,sha256=UCz9z0FBEvGYiBzea2kmJXdkSwlsIuTyqSuCdTwIC1w,58062
468
470
  datahub/ingestion/source/sac/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
469
471
  datahub/ingestion/source/sac/sac.py,sha256=0s_JxHGOhit3Wvgbg7qQi-Z9j9_TgBX_I1yOR3L6-rA,30243
470
472
  datahub/ingestion/source/sac/sac_common.py,sha256=-xQTDBtgH56AnpRXWGDnlmQqUuLRx-7wF1U1kQFWtX8,998
@@ -484,7 +486,7 @@ datahub/ingestion/source/sigma/data_classes.py,sha256=2uiPdTOYc3IW-WD0wSlhZqKcXl
484
486
  datahub/ingestion/source/sigma/sigma.py,sha256=7dbkwk8_wp94XH9mhmtI_8ihR35cqYywtU-dc_oK4Cc,25386
485
487
  datahub/ingestion/source/sigma/sigma_api.py,sha256=7PK5AQa838hYeaQ5L0dioi4n4bLrpN-r7COKTTNUYw8,19837
486
488
  datahub/ingestion/source/slack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
487
- datahub/ingestion/source/slack/slack.py,sha256=JWanUfzFGynV_PWcH0YzJIbRcmL880DA6dEI9QW-QiQ,25800
489
+ datahub/ingestion/source/slack/slack.py,sha256=ECiVkegoL6L3wTFOQo7WF4MVvMZthQUTMYrBRg5hUOI,25802
488
490
  datahub/ingestion/source/snaplogic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
489
491
  datahub/ingestion/source/snaplogic/snaplogic.py,sha256=T72zCO97xbQkC6DLq6Ka_h_kI5ZQZG6YRFJAHl_N5c0,13540
490
492
  datahub/ingestion/source/snaplogic/snaplogic_config.py,sha256=lYXKeaD0lvLoZ8eJmPNHhuLlOnCPbelZMyihgUXbxD8,1445
@@ -501,13 +503,13 @@ datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=uSHdPqigRzjeNx
501
503
  datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
502
504
  datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=nam-bYV6wL9LfR-Tt50Qe_Kea61IuWS-lLu5__aDxk8,21853
503
505
  datahub/ingestion/source/snowflake/snowflake_profiler.py,sha256=PmQi-qDlRhdJ-PsJ7x-EScIiswWRAxDDOKHydvN3mTY,7404
504
- datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=AiLEbCs6zXbIoejFdLsbSupkvqHj5BOtPz9lubGKLv8,47142
506
+ datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=e2fKNwKAeECdx6G9m9geeXhKVLQKDTGbMzOIz7WxFqw,47316
505
507
  datahub/ingestion/source/snowflake/snowflake_query.py,sha256=wLDaYZrWJ0794KKn69rB_QF0_8Bzu5l_7L6mD77KVc4,40469
506
508
  datahub/ingestion/source/snowflake/snowflake_report.py,sha256=fA6C-p9wM-jyTsXE_suTbCtrE_lle-5LI52S7wFYf00,6701
507
509
  datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=a6avRJXbj2qwnu28oK1YotmJo68zEG-1S7vonsUUJy4,41473
508
510
  datahub/ingestion/source/snowflake/snowflake_schema_gen.py,sha256=-JBfYgAXKMbVYu3f2viQoOQ0O2wv6GWfw1giOiETbpc,59091
509
511
  datahub/ingestion/source/snowflake/snowflake_shares.py,sha256=maZyFkfrbVogEFM0tTKRiNp9c_1muv6YfleSd3q0umI,6341
510
- datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=5Li4H8KuS4qBKR98L2P-JZI79UXsOjcAFxZZyio9NU0,5787
512
+ datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=LFCb0RqbrylmZXiFc4vczS6YrsoXBKs7FOebwNB65jg,5786
511
513
  datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=eA9xh-G1Ydr1OwUUtrbXUWp26hE1jF0zvyKNky_i_nQ,8887
512
514
  datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=mM0v9b4PHRJAT-SdRids3wdzc5O96gWCCww3e42itV8,24982
513
515
  datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=1c1YNmAxxOwAKy8IEFqVdp6x-EvCYJkN6UZ_RwUUVv0,15062
@@ -543,7 +545,7 @@ datahub/ingestion/source/sql/two_tier_sql_source.py,sha256=xlpQ9thbEn6uta6eVMzsH
543
545
  datahub/ingestion/source/sql/vertica.py,sha256=LUpIzDpzFRtMcXCSG2EC5CBZbveBOvnB27P_pDrXC9o,33501
544
546
  datahub/ingestion/source/sql/mssql/__init__.py,sha256=1agpl8S_uDW40olkhCX_W19dbr5GO9qgjS3R7pLRZSk,87
545
547
  datahub/ingestion/source/sql/mssql/job_models.py,sha256=ozEMc86e2oWRRF8Vjasjj32onVaVOQ98f00mbCYKmY8,9401
546
- datahub/ingestion/source/sql/mssql/source.py,sha256=N3ccX621byKLhthQw1-_aCkrn5teV4H1hc8u1gUMcy0,44762
548
+ datahub/ingestion/source/sql/mssql/source.py,sha256=6dVWAFaaXDR4z2LJZozusg4lEFjlGLn2ELK0JoJbfYA,46342
547
549
  datahub/ingestion/source/sql/stored_procedures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
548
550
  datahub/ingestion/source/sql/stored_procedures/base.py,sha256=n0l5OaTuW-m3TRvkxs3TqvgMeWF6BagzW3tjyWUcC1A,8631
549
551
  datahub/ingestion/source/sql/stored_procedures/lineage.py,sha256=fryLhuAlsjr9SHIjHJ-PmtCMx89bjzWVnJZ3f1bwQVU,1905
@@ -581,7 +583,7 @@ datahub/ingestion/source/unity/proxy_patch.py,sha256=gVYl5Fm_ase0iwBf3yDg7PE3bbT
581
583
  datahub/ingestion/source/unity/proxy_profiling.py,sha256=WLqvYP6MziaisA4LYL4T_GA-kPt6Xdde7bfaYsjYw40,9663
582
584
  datahub/ingestion/source/unity/proxy_types.py,sha256=dp7fRqIjaFCn6ivbgXOGHcw9bQQhZg6u-fdTK053oFM,10163
583
585
  datahub/ingestion/source/unity/report.py,sha256=wa5ER1hLc-362iWS9MNwDJf_adfAb3tfge_O3wSAaTc,3291
584
- datahub/ingestion/source/unity/source.py,sha256=TojTsImuCazZIegfvkePklAWFHUiTQChhHP2KYsuIdk,56247
586
+ datahub/ingestion/source/unity/source.py,sha256=smIDyLJ_WN_JuGi72rsO63YzIfJ94oaRHXCl-7jZ9Cs,56246
585
587
  datahub/ingestion/source/unity/tag_entities.py,sha256=-Z-XYc1XhquE-Eoksn9v0o11ZjV9CWz8n6zeXLbzluQ,7275
586
588
  datahub/ingestion/source/unity/usage.py,sha256=5CMQ0uOwuXfRq0rdxQbiHwrfSkvmQ5R2GNLqqZ2dOJw,13351
587
589
  datahub/ingestion/source/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -589,7 +591,7 @@ datahub/ingestion/source/usage/clickhouse_usage.py,sha256=M6YVQqwJoFqJPxlTr62lFw
589
591
  datahub/ingestion/source/usage/starburst_trino_usage.py,sha256=E5wlRQ-jkwSqFaHWgwnWRPu5IiWwxOnPXX1bCv_v__E,10815
590
592
  datahub/ingestion/source/usage/usage_common.py,sha256=glh_pdELYi7ZF8XXTQMlcTPXMTgJcB9lTfr6grDSkjA,9901
591
593
  datahub/ingestion/source/vertexai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
592
- datahub/ingestion/source/vertexai/vertexai.py,sha256=RuHda0mbc1DElYZIZ_W_hvkN7Eg4LIvI1fRFMvpHPB0,56012
594
+ datahub/ingestion/source/vertexai/vertexai.py,sha256=8WvICNO0s1g16p6zG1Pn0WQHxmnJTVy8Gwoy4Lzd6C8,56015
593
595
  datahub/ingestion/source/vertexai/vertexai_config.py,sha256=uMnsv3b6TsPRH26u_JE_v1u0db7ANEAFlVxU5A6ELRM,989
594
596
  datahub/ingestion/source/vertexai/vertexai_result_type_utils.py,sha256=dJwRxuDA3flmTKjV5EUbmDFfxE0S8K1CEPB_EYUfNfI,3578
595
597
  datahub/ingestion/source_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -647,7 +649,7 @@ datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1
647
649
  datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
648
650
  datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
649
651
  datahub/metadata/_internal_schema_classes.py,sha256=O8MG_weYzOYtGIZsOr7c6EO33NDo9liE8D8kyotoq8Q,1084754
650
- datahub/metadata/schema.avsc,sha256=YMMQSOELeCC4ZptYnoEfMFHRL-DguCR9KmjZ6NYWMlQ,780967
652
+ datahub/metadata/schema.avsc,sha256=_1lV6ZE52FCGSYWFneSaD_jXQ9vAp0A5bnPsGtc1foM,781031
651
653
  datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
652
654
  datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
653
655
  datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
@@ -779,7 +781,7 @@ datahub/metadata/schemas/DataHubAccessTokenKey.avsc,sha256=3EspNIxgb_I4WwV0a2o4N
779
781
  datahub/metadata/schemas/DataHubActionKey.avsc,sha256=bjiKcoyvUPQKaGUi2ICBMJ_ukwnt7dh0szJS4WBZE0A,448
780
782
  datahub/metadata/schemas/DataHubConnectionDetails.avsc,sha256=IvZj6OA7HRvy-ZIIn0UbXdJNnyt_oTn16XIe5ZlcqGk,1661
781
783
  datahub/metadata/schemas/DataHubConnectionKey.avsc,sha256=VwbamVFoEdp6epz1lJm_UShBl6ksBxoA7jAYuPI5u3M,522
782
- datahub/metadata/schemas/DataHubFileInfo.avsc,sha256=Vt1t5L9QyaIkSh3GUkrQs4I81o2MYAJJu194h1M97tw,6314
784
+ datahub/metadata/schemas/DataHubFileInfo.avsc,sha256=tylUV_qx3kmoqEjQX747FUD0ZP-65kXBM7VidtJ9km8,6374
783
785
  datahub/metadata/schemas/DataHubFileKey.avsc,sha256=-qxMFdpRPSHpA88adIFIJ35PZrN5qXOEo8RK-xTzkSQ,422
784
786
  datahub/metadata/schemas/DataHubIngestionSourceInfo.avsc,sha256=4wac7sluRIq-0ZjODE5SmuVKuQeW8ajLJNRpqEBRyio,4601
785
787
  datahub/metadata/schemas/DataHubIngestionSourceKey.avsc,sha256=TGmm9WEGTaABs7kt5Uc-N-kbc5Sd-2sQwx-JpfAptvw,545
@@ -984,7 +986,7 @@ datahub/sdk/entity.py,sha256=Q29AbpS58L4gD8ETwoNIwG-ouytz4c0MSSFi6-jLl_4,6742
984
986
  datahub/sdk/entity_client.py,sha256=LtFu0lYOl5s_B2G7HXoYY6uXaTBld-MC8Z_UeVKYCbc,9770
985
987
  datahub/sdk/lineage_client.py,sha256=qSe2TEt4HKRVytAsDokkfzqErZiL46c0TMe6g2C5hAg,33766
986
988
  datahub/sdk/main_client.py,sha256=LAymeMOkrjjJjQQ8Nc7G3hvF3P8Y0k0AXrDEGDGt4iU,5706
987
- datahub/sdk/mlmodel.py,sha256=cO5R8BYVljmQ0w33RIOuZmj4nq8OJCDVAZGTQI6YFS8,12628
989
+ datahub/sdk/mlmodel.py,sha256=LyAyuj3x0YDpjWBytlioP8dzl7hXy00MBTQJZVnwCGg,13377
988
990
  datahub/sdk/mlmodelgroup.py,sha256=wlZZHny0UORpF0fRYuVkWLSQwIHX_fWl5lPb1NKR6dM,8194
989
991
  datahub/sdk/resolver_client.py,sha256=nKMAZJt2tRSGfKSzoREIh43PXqjM3umLiYkYHJjo1io,3243
990
992
  datahub/sdk/search_client.py,sha256=f2L_aOy-pPB9Mk7WdLSJ6Htp3OT4cEgDIrqnCweNtM8,3592
@@ -1131,8 +1133,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
1131
1133
  datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
1132
1134
  datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
1133
1135
  datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
1134
- acryl_datahub-1.3.0.1rc6.dist-info/METADATA,sha256=JfkdfNc1P5L7jaGrlIUM-DqN9KcYjFuKfxwfg-IvTQI,184688
1135
- acryl_datahub-1.3.0.1rc6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
1136
- acryl_datahub-1.3.0.1rc6.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
1137
- acryl_datahub-1.3.0.1rc6.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
1138
- acryl_datahub-1.3.0.1rc6.dist-info/RECORD,,
1136
+ acryl_datahub-1.3.0.1rc8.dist-info/METADATA,sha256=lOSilMTQwZ3rvmq-XCw-gmoitXmWUW_muDrbc-GqlKY,184619
1137
+ acryl_datahub-1.3.0.1rc8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
1138
+ acryl_datahub-1.3.0.1rc8.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
1139
+ acryl_datahub-1.3.0.1rc8.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
1140
+ acryl_datahub-1.3.0.1rc8.dist-info/RECORD,,
datahub/_version.py CHANGED
@@ -1,6 +1,6 @@
1
1
  # Published at https://pypi.org/project/acryl-datahub/.
2
2
  __package_name__ = "acryl-datahub"
3
- __version__ = "1.3.0.1rc6"
3
+ __version__ = "1.3.0.1rc8"
4
4
 
5
5
 
6
6
  def is_dev_mode() -> bool:
@@ -111,7 +111,7 @@ def run_quickstart_preflight_checks(client: docker.DockerClient) -> None:
111
111
 
112
112
  result = client.containers.run(
113
113
  "alpine:latest",
114
- "sh -c \"df -B1 / | tail -1 | awk '{print $2, $4}'\"", # total, available
114
+ "sh -c \"df -B1 -P / | awk 'NR==2{print $2, $4}'\"", # total, available
115
115
  remove=True,
116
116
  stdout=True,
117
117
  stderr=True,
@@ -374,6 +374,12 @@ def make_domain_urn(domain: str) -> str:
374
374
  return f"urn:li:domain:{domain}"
375
375
 
376
376
 
377
+ def make_data_product_urn(data_product_id: str) -> str:
378
+ if data_product_id.startswith("urn:li:dataProduct:"):
379
+ return data_product_id
380
+ return f"urn:li:dataProduct:{data_product_id}"
381
+
382
+
377
383
  def make_ml_primary_key_urn(feature_table_name: str, primary_key_name: str) -> str:
378
384
  return f"urn:li:mlPrimaryKey:({feature_table_name},{primary_key_name})"
379
385
 
@@ -1,5 +1,5 @@
1
1
  {
2
- "generated_at": "2025-08-28T07:13:19.226648+00:00",
2
+ "generated_at": "2025-10-23T14:26:01.879155+00:00",
3
3
  "generated_by": "metadata-ingestion/scripts/capability_summary.py",
4
4
  "plugin_details": {
5
5
  "abs": {
@@ -815,7 +815,7 @@
815
815
  "classname": "datahub.ingestion.source.dynamodb.dynamodb.DynamoDBSource",
816
816
  "platform_id": "dynamodb",
817
817
  "platform_name": "DynamoDB",
818
- "support_status": "TESTING"
818
+ "support_status": "INCUBATING"
819
819
  },
820
820
  "elasticsearch": {
821
821
  "capabilities": [
@@ -945,7 +945,7 @@
945
945
  "classname": "datahub.ingestion.source.fivetran.fivetran.FivetranSource",
946
946
  "platform_id": "fivetran",
947
947
  "platform_name": "Fivetran",
948
- "support_status": "INCUBATING"
948
+ "support_status": "CERTIFIED"
949
949
  },
950
950
  "gcs": {
951
951
  "capabilities": [
@@ -1197,7 +1197,7 @@
1197
1197
  "classname": "datahub.ingestion.source.hex.hex.HexSource",
1198
1198
  "platform_id": "hex",
1199
1199
  "platform_name": "Hex",
1200
- "support_status": "TESTING"
1200
+ "support_status": "INCUBATING"
1201
1201
  },
1202
1202
  "hive": {
1203
1203
  "capabilities": [
@@ -1395,7 +1395,7 @@
1395
1395
  "classname": "datahub.ingestion.source.iceberg.iceberg.IcebergSource",
1396
1396
  "platform_id": "iceberg",
1397
1397
  "platform_name": "Iceberg",
1398
- "support_status": "TESTING"
1398
+ "support_status": "INCUBATING"
1399
1399
  },
1400
1400
  "json-schema": {
1401
1401
  "capabilities": [
@@ -1787,7 +1787,7 @@
1787
1787
  "classname": "datahub.ingestion.source.mlflow.MLflowSource",
1788
1788
  "platform_id": "mlflow",
1789
1789
  "platform_name": "MLflow",
1790
- "support_status": "TESTING"
1790
+ "support_status": "INCUBATING"
1791
1791
  },
1792
1792
  "mode": {
1793
1793
  "capabilities": [
@@ -2775,7 +2775,7 @@
2775
2775
  "classname": "datahub.ingestion.source.s3.source.S3Source",
2776
2776
  "platform_id": "s3",
2777
2777
  "platform_name": "S3 / Local Files",
2778
- "support_status": "INCUBATING"
2778
+ "support_status": "CERTIFIED"
2779
2779
  },
2780
2780
  "sac": {
2781
2781
  "capabilities": [
@@ -2888,7 +2888,7 @@
2888
2888
  "classname": "datahub.ingestion.source.salesforce.SalesforceSource",
2889
2889
  "platform_id": "salesforce",
2890
2890
  "platform_name": "Salesforce",
2891
- "support_status": "INCUBATING"
2891
+ "support_status": "CERTIFIED"
2892
2892
  },
2893
2893
  "sigma": {
2894
2894
  "capabilities": [
@@ -2966,7 +2966,7 @@
2966
2966
  "classname": "datahub.ingestion.source.slack.slack.SlackSource",
2967
2967
  "platform_id": "slack",
2968
2968
  "platform_name": "Slack",
2969
- "support_status": "TESTING"
2969
+ "support_status": "CERTIFIED"
2970
2970
  },
2971
2971
  "snaplogic": {
2972
2972
  "capabilities": [
@@ -3550,7 +3550,7 @@
3550
3550
  "classname": "datahub.ingestion.source.unity.source.UnityCatalogSource",
3551
3551
  "platform_id": "databricks",
3552
3552
  "platform_name": "Databricks",
3553
- "support_status": "INCUBATING"
3553
+ "support_status": "CERTIFIED"
3554
3554
  },
3555
3555
  "vertexai": {
3556
3556
  "capabilities": [
@@ -3564,7 +3564,7 @@
3564
3564
  "classname": "datahub.ingestion.source.vertexai.vertexai.VertexAISource",
3565
3565
  "platform_id": "vertexai",
3566
3566
  "platform_name": "Vertex AI",
3567
- "support_status": "TESTING"
3567
+ "support_status": "INCUBATING"
3568
3568
  },
3569
3569
  "vertica": {
3570
3570
  "capabilities": [
@@ -3649,4 +3649,4 @@
3649
3649
  "support_status": "CERTIFIED"
3650
3650
  }
3651
3651
  }
3652
- }
3652
+ }
@@ -7,6 +7,7 @@ from typing_extensions import Self
7
7
 
8
8
  from datahub.configuration.time_window_config import BaseTimeWindowConfig
9
9
  from datahub.ingestion.api.common import PipelineContext
10
+ from datahub.ingestion.api.decorators import SupportStatus, support_status
10
11
  from datahub.ingestion.api.source import Source, SourceReport
11
12
  from datahub.ingestion.api.workunit import MetadataWorkUnit
12
13
  from datahub.ingestion.source.bigquery_v2.bigquery_config import (
@@ -50,6 +51,7 @@ class BigQueryQueriesSourceConfig(
50
51
  )
51
52
 
52
53
 
54
+ @support_status(SupportStatus.CERTIFIED)
53
55
  class BigQueryQueriesSource(Source):
54
56
  def __init__(self, ctx: PipelineContext, config: BigQueryQueriesSourceConfig):
55
57
  self.ctx = ctx
@@ -34,6 +34,8 @@ class DatasetSubTypes(StrEnum):
34
34
  API_ENDPOINT = "API Endpoint"
35
35
  SLACK_CHANNEL = "Slack Channel"
36
36
  PROJECTIONS = "Projections"
37
+ GOOGLE_SHEETS = "Google Sheets"
38
+ GOOGLE_SHEETS_NAMED_RANGE = "Google Sheets Named Range"
37
39
 
38
40
  # TODO: Create separate entity...
39
41
  NOTEBOOK = "Notebook"
@@ -338,10 +338,10 @@ class DremioSource(StatefulIngestionSourceBase):
338
338
  return
339
339
 
340
340
  dataset_urn = make_dataset_urn_with_platform_instance(
341
- platform=make_data_platform_urn(self.get_platform()),
342
- name=f"dremio.{dataset_name}",
343
- env=self.config.env,
341
+ platform=self.get_platform(),
342
+ name=dataset_name,
344
343
  platform_instance=self.config.platform_instance,
344
+ env=self.config.env,
345
345
  )
346
346
 
347
347
  for dremio_mcp in self.dremio_aspects.populate_dataset_mcp(
@@ -421,10 +421,10 @@ class DremioSource(StatefulIngestionSourceBase):
421
421
  schema_str = ".".join(dataset_info.path)
422
422
  dataset_name = f"{schema_str}.{dataset_info.resource_name}".lower()
423
423
  dataset_urn = make_dataset_urn_with_platform_instance(
424
- platform=make_data_platform_urn(self.get_platform()),
425
- name=f"dremio.{dataset_name}",
426
- env=self.config.env,
424
+ platform=self.get_platform(),
425
+ name=dataset_name,
427
426
  platform_instance=self.config.platform_instance,
427
+ env=self.config.env,
428
428
  )
429
429
  yield from self.profiler.get_workunits(dataset_info, dataset_urn)
430
430
 
@@ -436,10 +436,10 @@ class DremioSource(StatefulIngestionSourceBase):
436
436
  """
437
437
  upstream_urns = [
438
438
  make_dataset_urn_with_platform_instance(
439
- platform=make_data_platform_urn(self.get_platform()),
440
- name=f"dremio.{upstream_table.lower()}",
441
- env=self.config.env,
439
+ platform=self.get_platform(),
440
+ name=upstream_table.lower(),
442
441
  platform_instance=self.config.platform_instance,
442
+ env=self.config.env,
443
443
  )
444
444
  for upstream_table in parents
445
445
  ]
@@ -498,19 +498,19 @@ class DremioSource(StatefulIngestionSourceBase):
498
498
  if query.query and query.affected_dataset:
499
499
  upstream_urns = [
500
500
  make_dataset_urn_with_platform_instance(
501
- platform=make_data_platform_urn(self.get_platform()),
502
- name=f"dremio.{ds.lower()}",
503
- env=self.config.env,
501
+ platform=self.get_platform(),
502
+ name=ds.lower(),
504
503
  platform_instance=self.config.platform_instance,
504
+ env=self.config.env,
505
505
  )
506
506
  for ds in query.queried_datasets
507
507
  ]
508
508
 
509
509
  downstream_urn = make_dataset_urn_with_platform_instance(
510
- platform=make_data_platform_urn(self.get_platform()),
511
- name=f"dremio.{query.affected_dataset.lower()}",
512
- env=self.config.env,
510
+ platform=self.get_platform(),
511
+ name=query.affected_dataset.lower(),
513
512
  platform_instance=self.config.platform_instance,
513
+ env=self.config.env,
514
514
  )
515
515
 
516
516
  # Add query to SqlParsingAggregator
@@ -163,7 +163,7 @@ _attribute_type_to_field_type_mapping: Dict[str, Type] = {
163
163
 
164
164
  @platform_name("DynamoDB", id="dynamodb")
165
165
  @config_class(DynamoDBConfig)
166
- @support_status(SupportStatus.TESTING)
166
+ @support_status(SupportStatus.INCUBATING)
167
167
  @capability(
168
168
  SourceCapability.PLATFORM_INSTANCE,
169
169
  "By default, platform_instance will use the AWS account id",
@@ -68,14 +68,22 @@ class Constant:
68
68
  SUCCESSFUL = "SUCCESSFUL"
69
69
  FAILURE_WITH_TASK = "FAILURE_WITH_TASK"
70
70
  CANCELED = "CANCELED"
71
+ GOOGLE_SHEETS_CONNECTOR_TYPE = "google_sheets"
71
72
 
72
73
 
74
+ # Key: Connector Type, Value: Platform ID/Name
73
75
  KNOWN_DATA_PLATFORM_MAPPING = {
74
76
  "google_cloud_postgresql": "postgres",
75
77
  "postgres": "postgres",
76
78
  "snowflake": "snowflake",
79
+ Constant.GOOGLE_SHEETS_CONNECTOR_TYPE: Constant.GOOGLE_SHEETS_CONNECTOR_TYPE,
77
80
  }
78
81
 
82
+ # Note: (As of Oct 2025) Fivetran Platform Connector has stale lineage metadata for Google Sheets column data (deleted/renamed).
83
+ # Ref: https://fivetran.com/docs/connectors/files/google-sheets#deletingdata
84
+ # TODO: Remove Google Sheets connector type from DISABLE_LINEAGE_FOR_CONNECTOR_TYPES
85
+ DISABLE_COL_LINEAGE_FOR_CONNECTOR_TYPES = [Constant.GOOGLE_SHEETS_CONNECTOR_TYPE]
86
+
79
87
 
80
88
  class SnowflakeDestinationConfig(SnowflakeConnectionConfig):
81
89
  database: str = Field(description="The fivetran connector log database.")
@@ -97,6 +105,17 @@ class DatabricksDestinationConfig(UnityCatalogConnectionConfig):
97
105
  return warehouse_id
98
106
 
99
107
 
108
+ class FivetranAPIConfig(ConfigModel):
109
+ api_key: str = Field(description="Fivetran API key")
110
+ api_secret: str = Field(description="Fivetran API secret")
111
+ base_url: str = Field(
112
+ default="https://api.fivetran.com", description="Fivetran API base URL"
113
+ )
114
+ request_timeout_sec: int = Field(
115
+ default=30, description="Request timeout in seconds"
116
+ )
117
+
118
+
100
119
  class FivetranLogConfig(ConfigModel):
101
120
  destination_platform: Literal["snowflake", "bigquery", "databricks"] = (
102
121
  pydantic.Field(
@@ -163,6 +182,7 @@ class MetadataExtractionPerfReport(Report):
163
182
  @dataclasses.dataclass
164
183
  class FivetranSourceReport(StaleEntityRemovalSourceReport):
165
184
  connectors_scanned: int = 0
185
+ fivetran_rest_api_call_count: int = 0
166
186
  filtered_connectors: LossyList[str] = dataclasses.field(default_factory=LossyList)
167
187
  metadata_extraction_perf: MetadataExtractionPerfReport = dataclasses.field(
168
188
  default_factory=MetadataExtractionPerfReport
@@ -174,6 +194,9 @@ class FivetranSourceReport(StaleEntityRemovalSourceReport):
174
194
  def report_connectors_dropped(self, connector: str) -> None:
175
195
  self.filtered_connectors.append(connector)
176
196
 
197
+ def report_fivetran_rest_api_call_count(self) -> None:
198
+ self.fivetran_rest_api_call_count += 1
199
+
177
200
 
178
201
  class PlatformDetail(ConfigModel):
179
202
  platform: Optional[str] = pydantic.Field(
@@ -234,6 +257,16 @@ class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin
234
257
  description="A mapping of destination id to its platform/instance/env details.",
235
258
  )
236
259
 
260
+ """
261
+ Use Fivetran REST API to get :
262
+ - Google Sheets Connector details and emit related entities
263
+ Fivetran Platform Connector syncs limited information about the Google Sheets Connector.
264
+ """
265
+ api_config: Optional[FivetranAPIConfig] = Field(
266
+ default=None,
267
+ description="Fivetran REST API configuration, used to provide wider support for connections.",
268
+ )
269
+
237
270
  @pydantic.root_validator(pre=True)
238
271
  def compat_sources_to_database(cls, values: Dict) -> Dict:
239
272
  if "sources_to_database" in values: