acryl-datahub 1.3.0.1rc6__py3-none-any.whl → 1.3.0.1rc7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc7.dist-info}/METADATA +2457 -2458
- {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc7.dist-info}/RECORD +34 -32
- datahub/_version.py +1 -1
- datahub/cli/docker_check.py +1 -1
- datahub/emitter/mce_builder.py +6 -0
- datahub/ingestion/autogenerated/capability_summary.json +12 -12
- datahub/ingestion/source/bigquery_v2/bigquery_queries.py +2 -0
- datahub/ingestion/source/common/subtypes.py +2 -0
- datahub/ingestion/source/dremio/dremio_source.py +15 -15
- datahub/ingestion/source/dynamodb/dynamodb.py +1 -1
- datahub/ingestion/source/fivetran/config.py +33 -0
- datahub/ingestion/source/fivetran/fivetran.py +184 -13
- datahub/ingestion/source/fivetran/fivetran_log_api.py +20 -5
- datahub/ingestion/source/fivetran/fivetran_rest_api.py +65 -0
- datahub/ingestion/source/fivetran/response_models.py +97 -0
- datahub/ingestion/source/hex/hex.py +1 -1
- datahub/ingestion/source/iceberg/iceberg.py +1 -1
- datahub/ingestion/source/metabase.py +23 -4
- datahub/ingestion/source/mlflow.py +1 -1
- datahub/ingestion/source/s3/source.py +1 -1
- datahub/ingestion/source/salesforce.py +1 -1
- datahub/ingestion/source/slack/slack.py +1 -1
- datahub/ingestion/source/snowflake/snowflake_queries.py +3 -0
- datahub/ingestion/source/snowflake/snowflake_summary.py +1 -1
- datahub/ingestion/source/sql_queries.py +1 -1
- datahub/ingestion/source/unity/source.py +1 -1
- datahub/ingestion/source/vertexai/vertexai.py +1 -1
- datahub/metadata/schema.avsc +4 -2
- datahub/metadata/schemas/DataHubFileInfo.avsc +4 -2
- datahub/sdk/mlmodel.py +19 -0
- {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc7.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc7.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc7.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.3.0.1rc6.dist-info → acryl_datahub-1.3.0.1rc7.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
acryl_datahub-1.3.0.
|
|
1
|
+
acryl_datahub-1.3.0.1rc7.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
|
|
2
2
|
datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
|
|
3
3
|
datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
|
|
4
|
-
datahub/_version.py,sha256=
|
|
4
|
+
datahub/_version.py,sha256=661Az3oTCAOu_uY9mbHEJZPDcekVKKHGjBByQxBb0CM,323
|
|
5
5
|
datahub/entrypoints.py,sha256=VcbU6Z47b_JKW1zI-WJMYIngm05FSogKLiuvFNtyNcI,9088
|
|
6
6
|
datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
|
|
7
7
|
datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -72,7 +72,7 @@ datahub/cli/cli_utils.py,sha256=0jTTAKuDZ8GzZwGHYytcT_MPR3Rb2DAcbr9n1H2T2sE,1617
|
|
|
72
72
|
datahub/cli/config_utils.py,sha256=wzCKcqXKSdc9P-hzvhzEVb0rB2kI2x5B2_MrRiME05M,4864
|
|
73
73
|
datahub/cli/container_cli.py,sha256=D0zWP3_3aww8_RTkMugOoOlILz3dPJ0TE9asQDLCm6E,1697
|
|
74
74
|
datahub/cli/delete_cli.py,sha256=0YJeWuXPGY0kbSn1AXK1-8SfCGBxb78ZbO53RAgyjQg,26515
|
|
75
|
-
datahub/cli/docker_check.py,sha256=
|
|
75
|
+
datahub/cli/docker_check.py,sha256=0LBY1VmdQG1wLR0HFgSFLNz23NahCQXuy7YOQKUnF48,13018
|
|
76
76
|
datahub/cli/docker_cli.py,sha256=EdW6OgPf-ZiqTSAwqz6cxTG1AmxlPbBonWl2GcuhWRk,33178
|
|
77
77
|
datahub/cli/env_utils.py,sha256=RQzjg4JE29hjPt4v7p-RuqoOr99w8E3DBHWiN2Sm7T4,252
|
|
78
78
|
datahub/cli/exists_cli.py,sha256=1cUYNh3GqNgVHWTrfMRGJoo9tFZNXcLetMaDbLaig6o,1233
|
|
@@ -128,7 +128,7 @@ datahub/emitter/composite_emitter.py,sha256=ZU-IdlAXKGPtmyT0JJgYC09vRn-TmeNaA6VP
|
|
|
128
128
|
datahub/emitter/enum_helpers.py,sha256=QBOEUu_hDCvyL_v4ayNQV8XwJbf5zKyu0Xat0mI1Kgo,376
|
|
129
129
|
datahub/emitter/generic_emitter.py,sha256=i37ZFm9VR_tmiZm9kIypEkQEB_cLKbzj_tJvViN-fm8,828
|
|
130
130
|
datahub/emitter/kafka_emitter.py,sha256=Uix1W1WaXF8VqUTUfzdRZKca2XrR1w50Anx2LVkROlc,5822
|
|
131
|
-
datahub/emitter/mce_builder.py,sha256=
|
|
131
|
+
datahub/emitter/mce_builder.py,sha256=SR4hiL2kT10_ilYBh9WTVssA-NjIq7pA6wMq7D8_hK0,17047
|
|
132
132
|
datahub/emitter/mcp.py,sha256=u6LphyhpbdFqboTAL_9MzXhGjc45o_BePoDFBkEEYWo,10484
|
|
133
133
|
datahub/emitter/mcp_builder.py,sha256=8IwJAlolQkPpMqQJPLtGrsUqAcuFNs98nrI5iYUxgaU,11920
|
|
134
134
|
datahub/emitter/mcp_patch_builder.py,sha256=u7cpW6DkiN7KpLapmMaXgL_FneoN69boxiANbVgMdSI,4564
|
|
@@ -162,7 +162,7 @@ datahub/ingestion/api/auto_work_units/auto_dataset_properties_aspect.py,sha256=I
|
|
|
162
162
|
datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py,sha256=-667F-xWArmeVaW-3mGsoIMNKPrnLx6KM4OfzMGsMOo,18064
|
|
163
163
|
datahub/ingestion/api/auto_work_units/auto_validate_input_fields.py,sha256=Xm7lhEPgicK_Isa2p0GtFbGCLJo5xpzmLc4AYEJPic0,3559
|
|
164
164
|
datahub/ingestion/autogenerated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
165
|
-
datahub/ingestion/autogenerated/capability_summary.json,sha256=
|
|
165
|
+
datahub/ingestion/autogenerated/capability_summary.json,sha256=n3VuRCSA_0pCMbtndm3zd57oLFtTVm8pQIeW_u-MIGk,111518
|
|
166
166
|
datahub/ingestion/autogenerated/lineage.json,sha256=8BdZF-5V5kJbX4mfFav8Zg-jHjzfkAEGk-pu1atLN4I,10029
|
|
167
167
|
datahub/ingestion/autogenerated/lineage_helper.py,sha256=I_k1pZSCCCjDbUVifPTfy6fkmV8jqdVhbirE8EkpmxI,4748
|
|
168
168
|
datahub/ingestion/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -218,8 +218,8 @@ datahub/ingestion/source/ge_data_profiler.py,sha256=9lEQdLcMBa7znqa6Zz-QWA4Uiv8K
|
|
|
218
218
|
datahub/ingestion/source/ge_profiling_config.py,sha256=FIuZtce0gRncSRKA1V9GLg8H5JyJPieZweFJS36Q_CI,11523
|
|
219
219
|
datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suabri1yV5geaAPjzg2eORDc,2559
|
|
220
220
|
datahub/ingestion/source/ldap.py,sha256=PKoA5pVjuIxFfW1TcbYNIWSm7-C7shK2FDn7Zo5mrVM,18705
|
|
221
|
-
datahub/ingestion/source/metabase.py,sha256=
|
|
222
|
-
datahub/ingestion/source/mlflow.py,sha256=
|
|
221
|
+
datahub/ingestion/source/metabase.py,sha256=udWYNEomLVo5RjWUOlbq9eI43Pjhaf56OVF_YZBHBl0,33258
|
|
222
|
+
datahub/ingestion/source/mlflow.py,sha256=sAXOAxgrKOMQPkaBj3KVtlpf9qfhYy9J1op1JaNVXs4,33343
|
|
223
223
|
datahub/ingestion/source/mode.py,sha256=VF3ps9qk5I4LZiAAh-TEDFj2mHmMic1VdbTBxkGPYOc,72748
|
|
224
224
|
datahub/ingestion/source/mongodb.py,sha256=ykUA2Jyn0rxzOO-pCWosOqvFTIRgFmqkCTRHAsxpOYc,21423
|
|
225
225
|
datahub/ingestion/source/nifi.py,sha256=waXqfxHp1aEZ-JzU-EVU6EIcPepI93tP1vCz8RbgCOg,56882
|
|
@@ -228,9 +228,9 @@ datahub/ingestion/source/openapi_parser.py,sha256=T87e2r-oPGgQl_FDMHnSGFZzApvWDC
|
|
|
228
228
|
datahub/ingestion/source/preset.py,sha256=opltXJmopFsTpIDFdFVQsuZh5aqT41EETPymKoF1JxA,3949
|
|
229
229
|
datahub/ingestion/source/pulsar.py,sha256=u5F8QnCLJsht5-7XCiUTsnfhCPIpKVB_l32CgMCU-As,20187
|
|
230
230
|
datahub/ingestion/source/redash.py,sha256=C4cDikWymbL88fDqaIPX5WA3f2sIEtH7bmhJKkmXJsM,30652
|
|
231
|
-
datahub/ingestion/source/salesforce.py,sha256=
|
|
231
|
+
datahub/ingestion/source/salesforce.py,sha256=0oTpAXys9nEQYzrbyFJIH60GaqsskDR5XPNqd3KSptY,40941
|
|
232
232
|
datahub/ingestion/source/source_registry.py,sha256=a2mLjJPLkSI-gYCTb_7U7Jo4D8jGknNQ_yScPIihXFk,1208
|
|
233
|
-
datahub/ingestion/source/sql_queries.py,sha256=
|
|
233
|
+
datahub/ingestion/source/sql_queries.py,sha256=xBJfoBoStB0BDk8G68UUfwE2qUiLyq5hbNYelWFkDrY,14581
|
|
234
234
|
datahub/ingestion/source/superset.py,sha256=qZ1SMeejwiM_ZkPlERXi3OSmabWcrsJSxZe98Eo9tqA,57866
|
|
235
235
|
datahub/ingestion/source/abs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
236
236
|
datahub/ingestion/source/abs/config.py,sha256=WW9JWbzqAJDblAcJKtNeuBHqOeJsB57lW2PqSD65-BU,6729
|
|
@@ -268,7 +268,7 @@ datahub/ingestion/source/bigquery_v2/bigquery_connection.py,sha256=6XFCc0oxxU3R4
|
|
|
268
268
|
datahub/ingestion/source/bigquery_v2/bigquery_data_reader.py,sha256=DeT3v_Z82__8En0FcZ0kavBAWQoRvSZ5Rppm9eeDAb8,2393
|
|
269
269
|
datahub/ingestion/source/bigquery_v2/bigquery_helper.py,sha256=QER3gY8e_k1_eNVj7cBso7ZzrWl_vO5PYSa6CpvqNx8,1554
|
|
270
270
|
datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py,sha256=9_sfX8BE2vt9RjBMyq27UxCxBaSlD5o3L4gQxrwlPvA,4961
|
|
271
|
-
datahub/ingestion/source/bigquery_v2/bigquery_queries.py,sha256=
|
|
271
|
+
datahub/ingestion/source/bigquery_v2/bigquery_queries.py,sha256=VdYe2PYGq4Z8VSuwHfiSSx7bgoqQe2xsXWRHy1sQoQo,3509
|
|
272
272
|
datahub/ingestion/source/bigquery_v2/bigquery_report.py,sha256=zlTkqOmt5zxnO40rVTYHF3fclj4OVlLtqUXwW5WIIcM,7855
|
|
273
273
|
datahub/ingestion/source/bigquery_v2/bigquery_schema.py,sha256=zbYb1EYnCJxgvsU8oT_76l0q_BW1exVjMWM1GAgd1nc,32600
|
|
274
274
|
datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py,sha256=PbSCMj5ACwEu_HQNe29IHs4y1bn15_nnz6ZW1Yt17wI,51796
|
|
@@ -289,7 +289,7 @@ datahub/ingestion/source/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
|
|
|
289
289
|
datahub/ingestion/source/common/data_platforms.py,sha256=HhuP3YIEi2WpyKDjUU8RiM0a2qjHWQcvc8kcqub0cVo,548
|
|
290
290
|
datahub/ingestion/source/common/data_reader.py,sha256=XbSxiRTYrk6seOz0ZjVjzSpGvP8lEjmqXrNI4cdYYmQ,1819
|
|
291
291
|
datahub/ingestion/source/common/gcp_credentials_config.py,sha256=e1VgUC5mnmgSOnhtxAR6Z5HdLDqXWpA543osBi-0cbE,2431
|
|
292
|
-
datahub/ingestion/source/common/subtypes.py,sha256=
|
|
292
|
+
datahub/ingestion/source/common/subtypes.py,sha256=yUduH43tGxgK8H2OYTzyRs6dXUH2KmhjuCFJHNYGaH4,4855
|
|
293
293
|
datahub/ingestion/source/data_lake_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
294
294
|
datahub/ingestion/source/data_lake_common/config.py,sha256=qUk83B01hjuBKHvVz8SmXnVCy5eFj-2-2QLEOrAdbgk,359
|
|
295
295
|
datahub/ingestion/source/data_lake_common/data_lake_utils.py,sha256=IYr5y8vy_6CtMtITqzn6OqovzH1cpe1i30M-75PouXo,7768
|
|
@@ -323,11 +323,11 @@ datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py,sha256=MQk8BAHL
|
|
|
323
323
|
datahub/ingestion/source/dremio/dremio_entities.py,sha256=1gZrNqTp3Pm6vqGDQaWt3HkxEuHKxpGYQ4geVoFvxWI,15147
|
|
324
324
|
datahub/ingestion/source/dremio/dremio_profiling.py,sha256=TAcnpo8ZRKhLDHnQSJzJg3YdwTSyEa73LUAzENs7wG4,12287
|
|
325
325
|
datahub/ingestion/source/dremio/dremio_reporting.py,sha256=UEj-6FMdIWsry5535_kM2hLze5aPRMatTwvI0Bd2BSo,2140
|
|
326
|
-
datahub/ingestion/source/dremio/dremio_source.py,sha256=
|
|
326
|
+
datahub/ingestion/source/dremio/dremio_source.py,sha256=6QHiFtfNiLiiYDCotUGRMsfWcgsd2ZyVOavimXyx6W4,25416
|
|
327
327
|
datahub/ingestion/source/dremio/dremio_sql_queries.py,sha256=wA1hqKk9cKMJDyEdZRQcDDLZPGYwuNqrvleUHTkWgrQ,10508
|
|
328
328
|
datahub/ingestion/source/dynamodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
329
329
|
datahub/ingestion/source/dynamodb/data_reader.py,sha256=vC77KpcP8LJN0g8wsPRDVw4sebv0ZWIP3tJkEIHaomA,3120
|
|
330
|
-
datahub/ingestion/source/dynamodb/dynamodb.py,sha256=
|
|
330
|
+
datahub/ingestion/source/dynamodb/dynamodb.py,sha256=32Bt9y9-R60MCGt9gYikNZjrjRxCQZDmD0ra9by5EyM,22931
|
|
331
331
|
datahub/ingestion/source/excel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
332
332
|
datahub/ingestion/source/excel/config.py,sha256=4x7GCpGaESXR8E1VH5ug53Dqh9tbyjbueaJHi3a_E_E,3604
|
|
333
333
|
datahub/ingestion/source/excel/excel_file.py,sha256=j-wr012nd6HhY-vbTfi7mY13mHJ0evV2OjUAYmHNcz4,18296
|
|
@@ -336,11 +336,13 @@ datahub/ingestion/source/excel/report.py,sha256=oEkeI8J6is7zB9iz4RqASu_-Q5xl36lA
|
|
|
336
336
|
datahub/ingestion/source/excel/source.py,sha256=w_vOz4UD7BcXBBDKoo81_6-QFeOPITuXqkfjIMHCQj4,23827
|
|
337
337
|
datahub/ingestion/source/excel/util.py,sha256=YYmadYuCiT-4_MfQM0YSE7wuDcE0k8o2KrlOKM9Z6eI,406
|
|
338
338
|
datahub/ingestion/source/fivetran/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
339
|
-
datahub/ingestion/source/fivetran/config.py,sha256=
|
|
339
|
+
datahub/ingestion/source/fivetran/config.py,sha256=wGoGsh8oCRTlENOoD5pMq6aFBl0S3A7D8AXF1OGS_Dg,11759
|
|
340
340
|
datahub/ingestion/source/fivetran/data_classes.py,sha256=ecdUJH5BEze0yv-uFpKWPNaNmV1gORDA2XMFk0zhcBw,595
|
|
341
|
-
datahub/ingestion/source/fivetran/fivetran.py,sha256=
|
|
342
|
-
datahub/ingestion/source/fivetran/fivetran_log_api.py,sha256
|
|
341
|
+
datahub/ingestion/source/fivetran/fivetran.py,sha256=8uPD5qNdA-BjG8pmjxVYf2_oOWVLzpg5fiaIYQGJO38,22685
|
|
342
|
+
datahub/ingestion/source/fivetran/fivetran_log_api.py,sha256=WaDqnFoK6wo737V28iAzY5fqxYc7ch5HrTngSva8j9Q,14320
|
|
343
343
|
datahub/ingestion/source/fivetran/fivetran_query.py,sha256=VJTka6cdIzlqy0aWyviMO2uSHcL0ZQFTCefUnnjv_Bk,6578
|
|
344
|
+
datahub/ingestion/source/fivetran/fivetran_rest_api.py,sha256=10laMOEc6kPxZcStDhWnwQNrohNVud1SywAo5Hl1kU0,2117
|
|
345
|
+
datahub/ingestion/source/fivetran/response_models.py,sha256=5D0fPuWRSSGah6UXUF966dCQiK-VUwlqDNIonasPRz0,3355
|
|
344
346
|
datahub/ingestion/source/gc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
345
347
|
datahub/ingestion/source/gc/datahub_gc.py,sha256=nJ6QbHnTPL0MamWxNTZA26FMZsnmvUdUwnQcDS81K9s,12723
|
|
346
348
|
datahub/ingestion/source/gc/dataprocess_cleanup.py,sha256=mUWcMt-_FL1SYGIgI4lGZDZGXspUUTv__5GN1W2oJ3s,17118
|
|
@@ -364,12 +366,12 @@ datahub/ingestion/source/grafana/types.py,sha256=Bz0-FIPBXHaBjfFHYGJhE20c2vYZwAs
|
|
|
364
366
|
datahub/ingestion/source/hex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
365
367
|
datahub/ingestion/source/hex/api.py,sha256=rKr6GmhIs98_SQZYG4egZlS049WH4ZxW2-7Ueq8692Q,12940
|
|
366
368
|
datahub/ingestion/source/hex/constants.py,sha256=8hUTMWyG5keTNfXoLu_Dh413Hw_mGGJX1atiiDZyKtg,271
|
|
367
|
-
datahub/ingestion/source/hex/hex.py,sha256=
|
|
369
|
+
datahub/ingestion/source/hex/hex.py,sha256=rra2-6ICNkSlrnO4_SFDTIValgmTaoOdDObr3q2CChk,13331
|
|
368
370
|
datahub/ingestion/source/hex/mapper.py,sha256=IyDAE-TzZUji3ICI_9gkYC3dQN3gl6kERRWNVRk80fQ,13905
|
|
369
371
|
datahub/ingestion/source/hex/model.py,sha256=eri4aRo1eXcE2SWjzCnPFMhzPTiJ8w8zC4GN7Lgpr74,1864
|
|
370
372
|
datahub/ingestion/source/hex/query_fetcher.py,sha256=r9UvF_qwswkRlNY7AI8p46eqAYSxVtjVE2e7eO4XagA,13384
|
|
371
373
|
datahub/ingestion/source/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
372
|
-
datahub/ingestion/source/iceberg/iceberg.py,sha256=
|
|
374
|
+
datahub/ingestion/source/iceberg/iceberg.py,sha256=JdFTVHDGJra1-L-z3JGBAT-PH978NEZZ5ZxGU93N1vA,37944
|
|
373
375
|
datahub/ingestion/source/iceberg/iceberg_common.py,sha256=CD_yHQ_wEgivyLQUTRO9BZJB29S7j5fUVllki-BPwUU,12292
|
|
374
376
|
datahub/ingestion/source/iceberg/iceberg_profiler.py,sha256=24zhWNa-x8dismkdGWuutruRN4VLWkM-4doTFH9-SEU,9936
|
|
375
377
|
datahub/ingestion/source/identity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -464,7 +466,7 @@ datahub/ingestion/source/s3/config.py,sha256=lElFXgEpKDT9SVoiXvtx98wV6Gp880qP4pL
|
|
|
464
466
|
datahub/ingestion/source/s3/datalake_profiler_config.py,sha256=FfrcgK-JEF94vw-l3q6pN6FENXb-wZzW2w1VUZVkwW8,3620
|
|
465
467
|
datahub/ingestion/source/s3/profiling.py,sha256=yKNCKpr6w7qpCH-baeSkNE9VjkN6eBot_weD-2_Jxzk,17579
|
|
466
468
|
datahub/ingestion/source/s3/report.py,sha256=9Ej1UCChw963UpGw1-7asi5vFrOM232gfgG8bRdKPp0,667
|
|
467
|
-
datahub/ingestion/source/s3/source.py,sha256=
|
|
469
|
+
datahub/ingestion/source/s3/source.py,sha256=UCz9z0FBEvGYiBzea2kmJXdkSwlsIuTyqSuCdTwIC1w,58062
|
|
468
470
|
datahub/ingestion/source/sac/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
469
471
|
datahub/ingestion/source/sac/sac.py,sha256=0s_JxHGOhit3Wvgbg7qQi-Z9j9_TgBX_I1yOR3L6-rA,30243
|
|
470
472
|
datahub/ingestion/source/sac/sac_common.py,sha256=-xQTDBtgH56AnpRXWGDnlmQqUuLRx-7wF1U1kQFWtX8,998
|
|
@@ -484,7 +486,7 @@ datahub/ingestion/source/sigma/data_classes.py,sha256=2uiPdTOYc3IW-WD0wSlhZqKcXl
|
|
|
484
486
|
datahub/ingestion/source/sigma/sigma.py,sha256=7dbkwk8_wp94XH9mhmtI_8ihR35cqYywtU-dc_oK4Cc,25386
|
|
485
487
|
datahub/ingestion/source/sigma/sigma_api.py,sha256=7PK5AQa838hYeaQ5L0dioi4n4bLrpN-r7COKTTNUYw8,19837
|
|
486
488
|
datahub/ingestion/source/slack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
487
|
-
datahub/ingestion/source/slack/slack.py,sha256=
|
|
489
|
+
datahub/ingestion/source/slack/slack.py,sha256=ECiVkegoL6L3wTFOQo7WF4MVvMZthQUTMYrBRg5hUOI,25802
|
|
488
490
|
datahub/ingestion/source/snaplogic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
489
491
|
datahub/ingestion/source/snaplogic/snaplogic.py,sha256=T72zCO97xbQkC6DLq6Ka_h_kI5ZQZG6YRFJAHl_N5c0,13540
|
|
490
492
|
datahub/ingestion/source/snaplogic/snaplogic_config.py,sha256=lYXKeaD0lvLoZ8eJmPNHhuLlOnCPbelZMyihgUXbxD8,1445
|
|
@@ -501,13 +503,13 @@ datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=uSHdPqigRzjeNx
|
|
|
501
503
|
datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
|
|
502
504
|
datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=nam-bYV6wL9LfR-Tt50Qe_Kea61IuWS-lLu5__aDxk8,21853
|
|
503
505
|
datahub/ingestion/source/snowflake/snowflake_profiler.py,sha256=PmQi-qDlRhdJ-PsJ7x-EScIiswWRAxDDOKHydvN3mTY,7404
|
|
504
|
-
datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=
|
|
506
|
+
datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=e2fKNwKAeECdx6G9m9geeXhKVLQKDTGbMzOIz7WxFqw,47316
|
|
505
507
|
datahub/ingestion/source/snowflake/snowflake_query.py,sha256=wLDaYZrWJ0794KKn69rB_QF0_8Bzu5l_7L6mD77KVc4,40469
|
|
506
508
|
datahub/ingestion/source/snowflake/snowflake_report.py,sha256=fA6C-p9wM-jyTsXE_suTbCtrE_lle-5LI52S7wFYf00,6701
|
|
507
509
|
datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=a6avRJXbj2qwnu28oK1YotmJo68zEG-1S7vonsUUJy4,41473
|
|
508
510
|
datahub/ingestion/source/snowflake/snowflake_schema_gen.py,sha256=-JBfYgAXKMbVYu3f2viQoOQ0O2wv6GWfw1giOiETbpc,59091
|
|
509
511
|
datahub/ingestion/source/snowflake/snowflake_shares.py,sha256=maZyFkfrbVogEFM0tTKRiNp9c_1muv6YfleSd3q0umI,6341
|
|
510
|
-
datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=
|
|
512
|
+
datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=LFCb0RqbrylmZXiFc4vczS6YrsoXBKs7FOebwNB65jg,5786
|
|
511
513
|
datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=eA9xh-G1Ydr1OwUUtrbXUWp26hE1jF0zvyKNky_i_nQ,8887
|
|
512
514
|
datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=mM0v9b4PHRJAT-SdRids3wdzc5O96gWCCww3e42itV8,24982
|
|
513
515
|
datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=1c1YNmAxxOwAKy8IEFqVdp6x-EvCYJkN6UZ_RwUUVv0,15062
|
|
@@ -581,7 +583,7 @@ datahub/ingestion/source/unity/proxy_patch.py,sha256=gVYl5Fm_ase0iwBf3yDg7PE3bbT
|
|
|
581
583
|
datahub/ingestion/source/unity/proxy_profiling.py,sha256=WLqvYP6MziaisA4LYL4T_GA-kPt6Xdde7bfaYsjYw40,9663
|
|
582
584
|
datahub/ingestion/source/unity/proxy_types.py,sha256=dp7fRqIjaFCn6ivbgXOGHcw9bQQhZg6u-fdTK053oFM,10163
|
|
583
585
|
datahub/ingestion/source/unity/report.py,sha256=wa5ER1hLc-362iWS9MNwDJf_adfAb3tfge_O3wSAaTc,3291
|
|
584
|
-
datahub/ingestion/source/unity/source.py,sha256=
|
|
586
|
+
datahub/ingestion/source/unity/source.py,sha256=smIDyLJ_WN_JuGi72rsO63YzIfJ94oaRHXCl-7jZ9Cs,56246
|
|
585
587
|
datahub/ingestion/source/unity/tag_entities.py,sha256=-Z-XYc1XhquE-Eoksn9v0o11ZjV9CWz8n6zeXLbzluQ,7275
|
|
586
588
|
datahub/ingestion/source/unity/usage.py,sha256=5CMQ0uOwuXfRq0rdxQbiHwrfSkvmQ5R2GNLqqZ2dOJw,13351
|
|
587
589
|
datahub/ingestion/source/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -589,7 +591,7 @@ datahub/ingestion/source/usage/clickhouse_usage.py,sha256=M6YVQqwJoFqJPxlTr62lFw
|
|
|
589
591
|
datahub/ingestion/source/usage/starburst_trino_usage.py,sha256=E5wlRQ-jkwSqFaHWgwnWRPu5IiWwxOnPXX1bCv_v__E,10815
|
|
590
592
|
datahub/ingestion/source/usage/usage_common.py,sha256=glh_pdELYi7ZF8XXTQMlcTPXMTgJcB9lTfr6grDSkjA,9901
|
|
591
593
|
datahub/ingestion/source/vertexai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
592
|
-
datahub/ingestion/source/vertexai/vertexai.py,sha256=
|
|
594
|
+
datahub/ingestion/source/vertexai/vertexai.py,sha256=8WvICNO0s1g16p6zG1Pn0WQHxmnJTVy8Gwoy4Lzd6C8,56015
|
|
593
595
|
datahub/ingestion/source/vertexai/vertexai_config.py,sha256=uMnsv3b6TsPRH26u_JE_v1u0db7ANEAFlVxU5A6ELRM,989
|
|
594
596
|
datahub/ingestion/source/vertexai/vertexai_result_type_utils.py,sha256=dJwRxuDA3flmTKjV5EUbmDFfxE0S8K1CEPB_EYUfNfI,3578
|
|
595
597
|
datahub/ingestion/source_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -647,7 +649,7 @@ datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1
|
|
|
647
649
|
datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
|
|
648
650
|
datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
|
|
649
651
|
datahub/metadata/_internal_schema_classes.py,sha256=O8MG_weYzOYtGIZsOr7c6EO33NDo9liE8D8kyotoq8Q,1084754
|
|
650
|
-
datahub/metadata/schema.avsc,sha256=
|
|
652
|
+
datahub/metadata/schema.avsc,sha256=_1lV6ZE52FCGSYWFneSaD_jXQ9vAp0A5bnPsGtc1foM,781031
|
|
651
653
|
datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
|
|
652
654
|
datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
|
|
653
655
|
datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
|
|
@@ -779,7 +781,7 @@ datahub/metadata/schemas/DataHubAccessTokenKey.avsc,sha256=3EspNIxgb_I4WwV0a2o4N
|
|
|
779
781
|
datahub/metadata/schemas/DataHubActionKey.avsc,sha256=bjiKcoyvUPQKaGUi2ICBMJ_ukwnt7dh0szJS4WBZE0A,448
|
|
780
782
|
datahub/metadata/schemas/DataHubConnectionDetails.avsc,sha256=IvZj6OA7HRvy-ZIIn0UbXdJNnyt_oTn16XIe5ZlcqGk,1661
|
|
781
783
|
datahub/metadata/schemas/DataHubConnectionKey.avsc,sha256=VwbamVFoEdp6epz1lJm_UShBl6ksBxoA7jAYuPI5u3M,522
|
|
782
|
-
datahub/metadata/schemas/DataHubFileInfo.avsc,sha256=
|
|
784
|
+
datahub/metadata/schemas/DataHubFileInfo.avsc,sha256=tylUV_qx3kmoqEjQX747FUD0ZP-65kXBM7VidtJ9km8,6374
|
|
783
785
|
datahub/metadata/schemas/DataHubFileKey.avsc,sha256=-qxMFdpRPSHpA88adIFIJ35PZrN5qXOEo8RK-xTzkSQ,422
|
|
784
786
|
datahub/metadata/schemas/DataHubIngestionSourceInfo.avsc,sha256=4wac7sluRIq-0ZjODE5SmuVKuQeW8ajLJNRpqEBRyio,4601
|
|
785
787
|
datahub/metadata/schemas/DataHubIngestionSourceKey.avsc,sha256=TGmm9WEGTaABs7kt5Uc-N-kbc5Sd-2sQwx-JpfAptvw,545
|
|
@@ -984,7 +986,7 @@ datahub/sdk/entity.py,sha256=Q29AbpS58L4gD8ETwoNIwG-ouytz4c0MSSFi6-jLl_4,6742
|
|
|
984
986
|
datahub/sdk/entity_client.py,sha256=LtFu0lYOl5s_B2G7HXoYY6uXaTBld-MC8Z_UeVKYCbc,9770
|
|
985
987
|
datahub/sdk/lineage_client.py,sha256=qSe2TEt4HKRVytAsDokkfzqErZiL46c0TMe6g2C5hAg,33766
|
|
986
988
|
datahub/sdk/main_client.py,sha256=LAymeMOkrjjJjQQ8Nc7G3hvF3P8Y0k0AXrDEGDGt4iU,5706
|
|
987
|
-
datahub/sdk/mlmodel.py,sha256=
|
|
989
|
+
datahub/sdk/mlmodel.py,sha256=LyAyuj3x0YDpjWBytlioP8dzl7hXy00MBTQJZVnwCGg,13377
|
|
988
990
|
datahub/sdk/mlmodelgroup.py,sha256=wlZZHny0UORpF0fRYuVkWLSQwIHX_fWl5lPb1NKR6dM,8194
|
|
989
991
|
datahub/sdk/resolver_client.py,sha256=nKMAZJt2tRSGfKSzoREIh43PXqjM3umLiYkYHJjo1io,3243
|
|
990
992
|
datahub/sdk/search_client.py,sha256=f2L_aOy-pPB9Mk7WdLSJ6Htp3OT4cEgDIrqnCweNtM8,3592
|
|
@@ -1131,8 +1133,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
|
|
|
1131
1133
|
datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
|
|
1132
1134
|
datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
|
|
1133
1135
|
datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
|
|
1134
|
-
acryl_datahub-1.3.0.
|
|
1135
|
-
acryl_datahub-1.3.0.
|
|
1136
|
-
acryl_datahub-1.3.0.
|
|
1137
|
-
acryl_datahub-1.3.0.
|
|
1138
|
-
acryl_datahub-1.3.0.
|
|
1136
|
+
acryl_datahub-1.3.0.1rc7.dist-info/METADATA,sha256=9QVlP4lgQfARG_mUrqs38beedc4VXd6o4r7YiXWiF2w,184619
|
|
1137
|
+
acryl_datahub-1.3.0.1rc7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
1138
|
+
acryl_datahub-1.3.0.1rc7.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
|
|
1139
|
+
acryl_datahub-1.3.0.1rc7.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
|
|
1140
|
+
acryl_datahub-1.3.0.1rc7.dist-info/RECORD,,
|
datahub/_version.py
CHANGED
datahub/cli/docker_check.py
CHANGED
|
@@ -111,7 +111,7 @@ def run_quickstart_preflight_checks(client: docker.DockerClient) -> None:
|
|
|
111
111
|
|
|
112
112
|
result = client.containers.run(
|
|
113
113
|
"alpine:latest",
|
|
114
|
-
"sh -c \"df -B1 / |
|
|
114
|
+
"sh -c \"df -B1 -P / | awk 'NR==2{print $2, $4}'\"", # total, available
|
|
115
115
|
remove=True,
|
|
116
116
|
stdout=True,
|
|
117
117
|
stderr=True,
|
datahub/emitter/mce_builder.py
CHANGED
|
@@ -374,6 +374,12 @@ def make_domain_urn(domain: str) -> str:
|
|
|
374
374
|
return f"urn:li:domain:{domain}"
|
|
375
375
|
|
|
376
376
|
|
|
377
|
+
def make_data_product_urn(data_product_id: str) -> str:
|
|
378
|
+
if data_product_id.startswith("urn:li:dataProduct:"):
|
|
379
|
+
return data_product_id
|
|
380
|
+
return f"urn:li:dataProduct:{data_product_id}"
|
|
381
|
+
|
|
382
|
+
|
|
377
383
|
def make_ml_primary_key_urn(feature_table_name: str, primary_key_name: str) -> str:
|
|
378
384
|
return f"urn:li:mlPrimaryKey:({feature_table_name},{primary_key_name})"
|
|
379
385
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"generated_at": "2025-
|
|
2
|
+
"generated_at": "2025-10-23T14:26:01.879155+00:00",
|
|
3
3
|
"generated_by": "metadata-ingestion/scripts/capability_summary.py",
|
|
4
4
|
"plugin_details": {
|
|
5
5
|
"abs": {
|
|
@@ -815,7 +815,7 @@
|
|
|
815
815
|
"classname": "datahub.ingestion.source.dynamodb.dynamodb.DynamoDBSource",
|
|
816
816
|
"platform_id": "dynamodb",
|
|
817
817
|
"platform_name": "DynamoDB",
|
|
818
|
-
"support_status": "
|
|
818
|
+
"support_status": "INCUBATING"
|
|
819
819
|
},
|
|
820
820
|
"elasticsearch": {
|
|
821
821
|
"capabilities": [
|
|
@@ -945,7 +945,7 @@
|
|
|
945
945
|
"classname": "datahub.ingestion.source.fivetran.fivetran.FivetranSource",
|
|
946
946
|
"platform_id": "fivetran",
|
|
947
947
|
"platform_name": "Fivetran",
|
|
948
|
-
"support_status": "
|
|
948
|
+
"support_status": "CERTIFIED"
|
|
949
949
|
},
|
|
950
950
|
"gcs": {
|
|
951
951
|
"capabilities": [
|
|
@@ -1197,7 +1197,7 @@
|
|
|
1197
1197
|
"classname": "datahub.ingestion.source.hex.hex.HexSource",
|
|
1198
1198
|
"platform_id": "hex",
|
|
1199
1199
|
"platform_name": "Hex",
|
|
1200
|
-
"support_status": "
|
|
1200
|
+
"support_status": "INCUBATING"
|
|
1201
1201
|
},
|
|
1202
1202
|
"hive": {
|
|
1203
1203
|
"capabilities": [
|
|
@@ -1395,7 +1395,7 @@
|
|
|
1395
1395
|
"classname": "datahub.ingestion.source.iceberg.iceberg.IcebergSource",
|
|
1396
1396
|
"platform_id": "iceberg",
|
|
1397
1397
|
"platform_name": "Iceberg",
|
|
1398
|
-
"support_status": "
|
|
1398
|
+
"support_status": "INCUBATING"
|
|
1399
1399
|
},
|
|
1400
1400
|
"json-schema": {
|
|
1401
1401
|
"capabilities": [
|
|
@@ -1787,7 +1787,7 @@
|
|
|
1787
1787
|
"classname": "datahub.ingestion.source.mlflow.MLflowSource",
|
|
1788
1788
|
"platform_id": "mlflow",
|
|
1789
1789
|
"platform_name": "MLflow",
|
|
1790
|
-
"support_status": "
|
|
1790
|
+
"support_status": "INCUBATING"
|
|
1791
1791
|
},
|
|
1792
1792
|
"mode": {
|
|
1793
1793
|
"capabilities": [
|
|
@@ -2775,7 +2775,7 @@
|
|
|
2775
2775
|
"classname": "datahub.ingestion.source.s3.source.S3Source",
|
|
2776
2776
|
"platform_id": "s3",
|
|
2777
2777
|
"platform_name": "S3 / Local Files",
|
|
2778
|
-
"support_status": "
|
|
2778
|
+
"support_status": "CERTIFIED"
|
|
2779
2779
|
},
|
|
2780
2780
|
"sac": {
|
|
2781
2781
|
"capabilities": [
|
|
@@ -2888,7 +2888,7 @@
|
|
|
2888
2888
|
"classname": "datahub.ingestion.source.salesforce.SalesforceSource",
|
|
2889
2889
|
"platform_id": "salesforce",
|
|
2890
2890
|
"platform_name": "Salesforce",
|
|
2891
|
-
"support_status": "
|
|
2891
|
+
"support_status": "CERTIFIED"
|
|
2892
2892
|
},
|
|
2893
2893
|
"sigma": {
|
|
2894
2894
|
"capabilities": [
|
|
@@ -2966,7 +2966,7 @@
|
|
|
2966
2966
|
"classname": "datahub.ingestion.source.slack.slack.SlackSource",
|
|
2967
2967
|
"platform_id": "slack",
|
|
2968
2968
|
"platform_name": "Slack",
|
|
2969
|
-
"support_status": "
|
|
2969
|
+
"support_status": "CERTIFIED"
|
|
2970
2970
|
},
|
|
2971
2971
|
"snaplogic": {
|
|
2972
2972
|
"capabilities": [
|
|
@@ -3550,7 +3550,7 @@
|
|
|
3550
3550
|
"classname": "datahub.ingestion.source.unity.source.UnityCatalogSource",
|
|
3551
3551
|
"platform_id": "databricks",
|
|
3552
3552
|
"platform_name": "Databricks",
|
|
3553
|
-
"support_status": "
|
|
3553
|
+
"support_status": "CERTIFIED"
|
|
3554
3554
|
},
|
|
3555
3555
|
"vertexai": {
|
|
3556
3556
|
"capabilities": [
|
|
@@ -3564,7 +3564,7 @@
|
|
|
3564
3564
|
"classname": "datahub.ingestion.source.vertexai.vertexai.VertexAISource",
|
|
3565
3565
|
"platform_id": "vertexai",
|
|
3566
3566
|
"platform_name": "Vertex AI",
|
|
3567
|
-
"support_status": "
|
|
3567
|
+
"support_status": "INCUBATING"
|
|
3568
3568
|
},
|
|
3569
3569
|
"vertica": {
|
|
3570
3570
|
"capabilities": [
|
|
@@ -3649,4 +3649,4 @@
|
|
|
3649
3649
|
"support_status": "CERTIFIED"
|
|
3650
3650
|
}
|
|
3651
3651
|
}
|
|
3652
|
-
}
|
|
3652
|
+
}
|
|
@@ -7,6 +7,7 @@ from typing_extensions import Self
|
|
|
7
7
|
|
|
8
8
|
from datahub.configuration.time_window_config import BaseTimeWindowConfig
|
|
9
9
|
from datahub.ingestion.api.common import PipelineContext
|
|
10
|
+
from datahub.ingestion.api.decorators import SupportStatus, support_status
|
|
10
11
|
from datahub.ingestion.api.source import Source, SourceReport
|
|
11
12
|
from datahub.ingestion.api.workunit import MetadataWorkUnit
|
|
12
13
|
from datahub.ingestion.source.bigquery_v2.bigquery_config import (
|
|
@@ -50,6 +51,7 @@ class BigQueryQueriesSourceConfig(
|
|
|
50
51
|
)
|
|
51
52
|
|
|
52
53
|
|
|
54
|
+
@support_status(SupportStatus.CERTIFIED)
|
|
53
55
|
class BigQueryQueriesSource(Source):
|
|
54
56
|
def __init__(self, ctx: PipelineContext, config: BigQueryQueriesSourceConfig):
|
|
55
57
|
self.ctx = ctx
|
|
@@ -34,6 +34,8 @@ class DatasetSubTypes(StrEnum):
|
|
|
34
34
|
API_ENDPOINT = "API Endpoint"
|
|
35
35
|
SLACK_CHANNEL = "Slack Channel"
|
|
36
36
|
PROJECTIONS = "Projections"
|
|
37
|
+
GOOGLE_SHEETS = "Google Sheets"
|
|
38
|
+
GOOGLE_SHEETS_NAMED_RANGE = "Google Sheets Named Range"
|
|
37
39
|
|
|
38
40
|
# TODO: Create separate entity...
|
|
39
41
|
NOTEBOOK = "Notebook"
|
|
@@ -338,10 +338,10 @@ class DremioSource(StatefulIngestionSourceBase):
|
|
|
338
338
|
return
|
|
339
339
|
|
|
340
340
|
dataset_urn = make_dataset_urn_with_platform_instance(
|
|
341
|
-
platform=
|
|
342
|
-
name=
|
|
343
|
-
env=self.config.env,
|
|
341
|
+
platform=self.get_platform(),
|
|
342
|
+
name=dataset_name,
|
|
344
343
|
platform_instance=self.config.platform_instance,
|
|
344
|
+
env=self.config.env,
|
|
345
345
|
)
|
|
346
346
|
|
|
347
347
|
for dremio_mcp in self.dremio_aspects.populate_dataset_mcp(
|
|
@@ -421,10 +421,10 @@ class DremioSource(StatefulIngestionSourceBase):
|
|
|
421
421
|
schema_str = ".".join(dataset_info.path)
|
|
422
422
|
dataset_name = f"{schema_str}.{dataset_info.resource_name}".lower()
|
|
423
423
|
dataset_urn = make_dataset_urn_with_platform_instance(
|
|
424
|
-
platform=
|
|
425
|
-
name=
|
|
426
|
-
env=self.config.env,
|
|
424
|
+
platform=self.get_platform(),
|
|
425
|
+
name=dataset_name,
|
|
427
426
|
platform_instance=self.config.platform_instance,
|
|
427
|
+
env=self.config.env,
|
|
428
428
|
)
|
|
429
429
|
yield from self.profiler.get_workunits(dataset_info, dataset_urn)
|
|
430
430
|
|
|
@@ -436,10 +436,10 @@ class DremioSource(StatefulIngestionSourceBase):
|
|
|
436
436
|
"""
|
|
437
437
|
upstream_urns = [
|
|
438
438
|
make_dataset_urn_with_platform_instance(
|
|
439
|
-
platform=
|
|
440
|
-
name=
|
|
441
|
-
env=self.config.env,
|
|
439
|
+
platform=self.get_platform(),
|
|
440
|
+
name=upstream_table.lower(),
|
|
442
441
|
platform_instance=self.config.platform_instance,
|
|
442
|
+
env=self.config.env,
|
|
443
443
|
)
|
|
444
444
|
for upstream_table in parents
|
|
445
445
|
]
|
|
@@ -498,19 +498,19 @@ class DremioSource(StatefulIngestionSourceBase):
|
|
|
498
498
|
if query.query and query.affected_dataset:
|
|
499
499
|
upstream_urns = [
|
|
500
500
|
make_dataset_urn_with_platform_instance(
|
|
501
|
-
platform=
|
|
502
|
-
name=
|
|
503
|
-
env=self.config.env,
|
|
501
|
+
platform=self.get_platform(),
|
|
502
|
+
name=ds.lower(),
|
|
504
503
|
platform_instance=self.config.platform_instance,
|
|
504
|
+
env=self.config.env,
|
|
505
505
|
)
|
|
506
506
|
for ds in query.queried_datasets
|
|
507
507
|
]
|
|
508
508
|
|
|
509
509
|
downstream_urn = make_dataset_urn_with_platform_instance(
|
|
510
|
-
platform=
|
|
511
|
-
name=
|
|
512
|
-
env=self.config.env,
|
|
510
|
+
platform=self.get_platform(),
|
|
511
|
+
name=query.affected_dataset.lower(),
|
|
513
512
|
platform_instance=self.config.platform_instance,
|
|
513
|
+
env=self.config.env,
|
|
514
514
|
)
|
|
515
515
|
|
|
516
516
|
# Add query to SqlParsingAggregator
|
|
@@ -163,7 +163,7 @@ _attribute_type_to_field_type_mapping: Dict[str, Type] = {
|
|
|
163
163
|
|
|
164
164
|
@platform_name("DynamoDB", id="dynamodb")
|
|
165
165
|
@config_class(DynamoDBConfig)
|
|
166
|
-
@support_status(SupportStatus.
|
|
166
|
+
@support_status(SupportStatus.INCUBATING)
|
|
167
167
|
@capability(
|
|
168
168
|
SourceCapability.PLATFORM_INSTANCE,
|
|
169
169
|
"By default, platform_instance will use the AWS account id",
|
|
@@ -68,14 +68,22 @@ class Constant:
|
|
|
68
68
|
SUCCESSFUL = "SUCCESSFUL"
|
|
69
69
|
FAILURE_WITH_TASK = "FAILURE_WITH_TASK"
|
|
70
70
|
CANCELED = "CANCELED"
|
|
71
|
+
GOOGLE_SHEETS_CONNECTOR_TYPE = "google_sheets"
|
|
71
72
|
|
|
72
73
|
|
|
74
|
+
# Key: Connector Type, Value: Platform ID/Name
|
|
73
75
|
KNOWN_DATA_PLATFORM_MAPPING = {
|
|
74
76
|
"google_cloud_postgresql": "postgres",
|
|
75
77
|
"postgres": "postgres",
|
|
76
78
|
"snowflake": "snowflake",
|
|
79
|
+
Constant.GOOGLE_SHEETS_CONNECTOR_TYPE: Constant.GOOGLE_SHEETS_CONNECTOR_TYPE,
|
|
77
80
|
}
|
|
78
81
|
|
|
82
|
+
# Note: (As of Oct 2025) Fivetran Platform Connector has stale lineage metadata for Google Sheets column data (deleted/renamed).
|
|
83
|
+
# Ref: https://fivetran.com/docs/connectors/files/google-sheets#deletingdata
|
|
84
|
+
# TODO: Remove Google Sheets connector type from DISABLE_LINEAGE_FOR_CONNECTOR_TYPES
|
|
85
|
+
DISABLE_COL_LINEAGE_FOR_CONNECTOR_TYPES = [Constant.GOOGLE_SHEETS_CONNECTOR_TYPE]
|
|
86
|
+
|
|
79
87
|
|
|
80
88
|
class SnowflakeDestinationConfig(SnowflakeConnectionConfig):
|
|
81
89
|
database: str = Field(description="The fivetran connector log database.")
|
|
@@ -97,6 +105,17 @@ class DatabricksDestinationConfig(UnityCatalogConnectionConfig):
|
|
|
97
105
|
return warehouse_id
|
|
98
106
|
|
|
99
107
|
|
|
108
|
+
class FivetranAPIConfig(ConfigModel):
|
|
109
|
+
api_key: str = Field(description="Fivetran API key")
|
|
110
|
+
api_secret: str = Field(description="Fivetran API secret")
|
|
111
|
+
base_url: str = Field(
|
|
112
|
+
default="https://api.fivetran.com", description="Fivetran API base URL"
|
|
113
|
+
)
|
|
114
|
+
request_timeout_sec: int = Field(
|
|
115
|
+
default=30, description="Request timeout in seconds"
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
100
119
|
class FivetranLogConfig(ConfigModel):
|
|
101
120
|
destination_platform: Literal["snowflake", "bigquery", "databricks"] = (
|
|
102
121
|
pydantic.Field(
|
|
@@ -163,6 +182,7 @@ class MetadataExtractionPerfReport(Report):
|
|
|
163
182
|
@dataclasses.dataclass
|
|
164
183
|
class FivetranSourceReport(StaleEntityRemovalSourceReport):
|
|
165
184
|
connectors_scanned: int = 0
|
|
185
|
+
fivetran_rest_api_call_count: int = 0
|
|
166
186
|
filtered_connectors: LossyList[str] = dataclasses.field(default_factory=LossyList)
|
|
167
187
|
metadata_extraction_perf: MetadataExtractionPerfReport = dataclasses.field(
|
|
168
188
|
default_factory=MetadataExtractionPerfReport
|
|
@@ -174,6 +194,9 @@ class FivetranSourceReport(StaleEntityRemovalSourceReport):
|
|
|
174
194
|
def report_connectors_dropped(self, connector: str) -> None:
|
|
175
195
|
self.filtered_connectors.append(connector)
|
|
176
196
|
|
|
197
|
+
def report_fivetran_rest_api_call_count(self) -> None:
|
|
198
|
+
self.fivetran_rest_api_call_count += 1
|
|
199
|
+
|
|
177
200
|
|
|
178
201
|
class PlatformDetail(ConfigModel):
|
|
179
202
|
platform: Optional[str] = pydantic.Field(
|
|
@@ -234,6 +257,16 @@ class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin
|
|
|
234
257
|
description="A mapping of destination id to its platform/instance/env details.",
|
|
235
258
|
)
|
|
236
259
|
|
|
260
|
+
"""
|
|
261
|
+
Use Fivetran REST API to get :
|
|
262
|
+
- Google Sheets Connector details and emit related entities
|
|
263
|
+
Fivetran Platform Connector syncs limited information about the Google Sheets Connector.
|
|
264
|
+
"""
|
|
265
|
+
api_config: Optional[FivetranAPIConfig] = Field(
|
|
266
|
+
default=None,
|
|
267
|
+
description="Fivetran REST API configuration, used to provide wider support for connections.",
|
|
268
|
+
)
|
|
269
|
+
|
|
237
270
|
@pydantic.root_validator(pre=True)
|
|
238
271
|
def compat_sources_to_database(cls, values: Dict) -> Dict:
|
|
239
272
|
if "sources_to_database" in values:
|