acryl-datahub 1.1.0.4rc1__py3-none-any.whl → 1.1.0.4rc3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of acryl-datahub might be problematic. Click here for more details.

Files changed (58) hide show
  1. {acryl_datahub-1.1.0.4rc1.dist-info → acryl_datahub-1.1.0.4rc3.dist-info}/METADATA +2581 -2581
  2. {acryl_datahub-1.1.0.4rc1.dist-info → acryl_datahub-1.1.0.4rc3.dist-info}/RECORD +58 -58
  3. datahub/_version.py +1 -1
  4. datahub/emitter/rest_emitter.py +18 -1
  5. datahub/ingestion/api/source.py +2 -0
  6. datahub/ingestion/source/bigquery_v2/bigquery.py +18 -0
  7. datahub/ingestion/source/dbt/dbt_cloud.py +3 -0
  8. datahub/ingestion/source/dbt/dbt_common.py +3 -1
  9. datahub/ingestion/source/dbt/dbt_core.py +3 -0
  10. datahub/ingestion/source/dremio/dremio_api.py +98 -68
  11. datahub/ingestion/source/dremio/dremio_config.py +2 -0
  12. datahub/ingestion/source/dremio/dremio_reporting.py +23 -2
  13. datahub/ingestion/source/dremio/dremio_source.py +90 -77
  14. datahub/ingestion/source/dremio/dremio_sql_queries.py +82 -21
  15. datahub/ingestion/source/file.py +3 -0
  16. datahub/ingestion/source/ge_data_profiler.py +48 -8
  17. datahub/ingestion/source/iceberg/iceberg.py +3 -1
  18. datahub/ingestion/source/kafka/kafka.py +1 -0
  19. datahub/ingestion/source/looker/looker_source.py +1 -0
  20. datahub/ingestion/source/powerbi/powerbi.py +1 -0
  21. datahub/ingestion/source/qlik_sense/qlik_sense.py +1 -0
  22. datahub/ingestion/source/redshift/redshift.py +21 -1
  23. datahub/ingestion/source/sac/sac.py +3 -1
  24. datahub/ingestion/source/sigma/sigma.py +1 -0
  25. datahub/ingestion/source/snowflake/snowflake_config.py +3 -6
  26. datahub/ingestion/source/snowflake/snowflake_utils.py +2 -7
  27. datahub/ingestion/source/snowflake/snowflake_v2.py +2 -0
  28. datahub/ingestion/source/sql/clickhouse.py +3 -1
  29. datahub/ingestion/source/sql/cockroachdb.py +0 -1
  30. datahub/ingestion/source/sql/hana.py +3 -1
  31. datahub/ingestion/source/sql/hive_metastore.py +3 -1
  32. datahub/ingestion/source/sql/mariadb.py +0 -1
  33. datahub/ingestion/source/sql/mssql/source.py +8 -1
  34. datahub/ingestion/source/sql/mysql.py +0 -9
  35. datahub/ingestion/source/sql/postgres.py +0 -1
  36. datahub/ingestion/source/sql/sql_common.py +12 -0
  37. datahub/ingestion/source/tableau/tableau.py +1 -0
  38. datahub/ingestion/source/unity/source.py +1 -0
  39. datahub/ingestion/source/usage/clickhouse_usage.py +4 -1
  40. datahub/ingestion/source/usage/starburst_trino_usage.py +3 -0
  41. datahub/metadata/_internal_schema_classes.py +25 -0
  42. datahub/metadata/schema.avsc +18 -1
  43. datahub/metadata/schemas/ContainerProperties.avsc +6 -0
  44. datahub/metadata/schemas/DataFlowInfo.avsc +6 -0
  45. datahub/metadata/schemas/DataHubPolicyInfo.avsc +12 -1
  46. datahub/metadata/schemas/DataJobInfo.avsc +6 -0
  47. datahub/metadata/schemas/DataProcessKey.avsc +6 -0
  48. datahub/metadata/schemas/DatasetKey.avsc +6 -0
  49. datahub/metadata/schemas/IcebergWarehouseInfo.avsc +6 -0
  50. datahub/metadata/schemas/MLModelDeploymentKey.avsc +6 -0
  51. datahub/metadata/schemas/MLModelGroupKey.avsc +6 -0
  52. datahub/metadata/schemas/MLModelKey.avsc +6 -0
  53. datahub/metadata/schemas/MetadataChangeEvent.avsc +18 -1
  54. datahub/utilities/stats_collections.py +4 -0
  55. {acryl_datahub-1.1.0.4rc1.dist-info → acryl_datahub-1.1.0.4rc3.dist-info}/WHEEL +0 -0
  56. {acryl_datahub-1.1.0.4rc1.dist-info → acryl_datahub-1.1.0.4rc3.dist-info}/entry_points.txt +0 -0
  57. {acryl_datahub-1.1.0.4rc1.dist-info → acryl_datahub-1.1.0.4rc3.dist-info}/licenses/LICENSE +0 -0
  58. {acryl_datahub-1.1.0.4rc1.dist-info → acryl_datahub-1.1.0.4rc3.dist-info}/top_level.txt +0 -0
datahub/metadata/schemas/DataJobInfo.avsc CHANGED
@@ -213,13 +213,16 @@
213
213
  "DEV": "Designates development fabrics",
214
214
  "EI": "Designates early-integration fabrics",
215
215
  "NON_PROD": "Designates non-production fabrics",
216
+ "PRD": "Alternative Prod spelling",
216
217
  "PRE": "Designates pre-production fabrics",
217
218
  "PROD": "Designates production fabrics",
218
219
  "QA": "Designates quality assurance fabrics",
219
220
  "RVW": "Designates review fabrics",
220
221
  "SANDBOX": "Designates sandbox fabrics",
222
+ "SIT": "System Integration Testing",
221
223
  "STG": "Designates staging fabrics",
222
224
  "TEST": "Designates testing fabrics",
225
+ "TST": "Alternative Test spelling",
223
226
  "UAT": "Designates user acceptance testing fabrics"
224
227
  },
225
228
  "name": "FabricType",
@@ -236,6 +239,9 @@
236
239
  "PROD",
237
240
  "CORP",
238
241
  "RVW",
242
+ "PRD",
243
+ "TST",
244
+ "SIT",
239
245
  "SANDBOX"
240
246
  ],
241
247
  "doc": "Fabric group type"
datahub/metadata/schemas/DataProcessKey.avsc CHANGED
@@ -46,13 +46,16 @@
46
46
  "DEV": "Designates development fabrics",
47
47
  "EI": "Designates early-integration fabrics",
48
48
  "NON_PROD": "Designates non-production fabrics",
49
+ "PRD": "Alternative Prod spelling",
49
50
  "PRE": "Designates pre-production fabrics",
50
51
  "PROD": "Designates production fabrics",
51
52
  "QA": "Designates quality assurance fabrics",
52
53
  "RVW": "Designates review fabrics",
53
54
  "SANDBOX": "Designates sandbox fabrics",
55
+ "SIT": "System Integration Testing",
54
56
  "STG": "Designates staging fabrics",
55
57
  "TEST": "Designates testing fabrics",
58
+ "TST": "Alternative Test spelling",
56
59
  "UAT": "Designates user acceptance testing fabrics"
57
60
  },
58
61
  "name": "FabricType",
@@ -69,6 +72,9 @@
69
72
  "PROD",
70
73
  "CORP",
71
74
  "RVW",
75
+ "PRD",
76
+ "TST",
77
+ "SIT",
72
78
  "SANDBOX"
73
79
  ],
74
80
  "doc": "Fabric group type"
datahub/metadata/schemas/DatasetKey.avsc CHANGED
@@ -83,13 +83,16 @@
83
83
  "DEV": "Designates development fabrics",
84
84
  "EI": "Designates early-integration fabrics",
85
85
  "NON_PROD": "Designates non-production fabrics",
86
+ "PRD": "Alternative Prod spelling",
86
87
  "PRE": "Designates pre-production fabrics",
87
88
  "PROD": "Designates production fabrics",
88
89
  "QA": "Designates quality assurance fabrics",
89
90
  "RVW": "Designates review fabrics",
90
91
  "SANDBOX": "Designates sandbox fabrics",
92
+ "SIT": "System Integration Testing",
91
93
  "STG": "Designates staging fabrics",
92
94
  "TEST": "Designates testing fabrics",
95
+ "TST": "Alternative Test spelling",
93
96
  "UAT": "Designates user acceptance testing fabrics"
94
97
  },
95
98
  "name": "FabricType",
@@ -106,6 +109,9 @@
106
109
  "PROD",
107
110
  "CORP",
108
111
  "RVW",
112
+ "PRD",
113
+ "TST",
114
+ "SIT",
109
115
  "SANDBOX"
110
116
  ],
111
117
  "doc": "Fabric group type"
datahub/metadata/schemas/IcebergWarehouseInfo.avsc CHANGED
@@ -58,13 +58,16 @@
58
58
  "DEV": "Designates development fabrics",
59
59
  "EI": "Designates early-integration fabrics",
60
60
  "NON_PROD": "Designates non-production fabrics",
61
+ "PRD": "Alternative Prod spelling",
61
62
  "PRE": "Designates pre-production fabrics",
62
63
  "PROD": "Designates production fabrics",
63
64
  "QA": "Designates quality assurance fabrics",
64
65
  "RVW": "Designates review fabrics",
65
66
  "SANDBOX": "Designates sandbox fabrics",
67
+ "SIT": "System Integration Testing",
66
68
  "STG": "Designates staging fabrics",
67
69
  "TEST": "Designates testing fabrics",
70
+ "TST": "Alternative Test spelling",
68
71
  "UAT": "Designates user acceptance testing fabrics"
69
72
  },
70
73
  "name": "FabricType",
@@ -81,6 +84,9 @@
81
84
  "PROD",
82
85
  "CORP",
83
86
  "RVW",
87
+ "PRD",
88
+ "TST",
89
+ "SIT",
84
90
  "SANDBOX"
85
91
  ],
86
92
  "doc": "Fabric group type"
datahub/metadata/schemas/MLModelDeploymentKey.avsc CHANGED
@@ -54,13 +54,16 @@
54
54
  "DEV": "Designates development fabrics",
55
55
  "EI": "Designates early-integration fabrics",
56
56
  "NON_PROD": "Designates non-production fabrics",
57
+ "PRD": "Alternative Prod spelling",
57
58
  "PRE": "Designates pre-production fabrics",
58
59
  "PROD": "Designates production fabrics",
59
60
  "QA": "Designates quality assurance fabrics",
60
61
  "RVW": "Designates review fabrics",
61
62
  "SANDBOX": "Designates sandbox fabrics",
63
+ "SIT": "System Integration Testing",
62
64
  "STG": "Designates staging fabrics",
63
65
  "TEST": "Designates testing fabrics",
66
+ "TST": "Alternative Test spelling",
64
67
  "UAT": "Designates user acceptance testing fabrics"
65
68
  },
66
69
  "name": "FabricType",
@@ -77,6 +80,9 @@
77
80
  "PROD",
78
81
  "CORP",
79
82
  "RVW",
83
+ "PRD",
84
+ "TST",
85
+ "SIT",
80
86
  "SANDBOX"
81
87
  ],
82
88
  "doc": "Fabric group type"
datahub/metadata/schemas/MLModelGroupKey.avsc CHANGED
@@ -61,13 +61,16 @@
61
61
  "DEV": "Designates development fabrics",
62
62
  "EI": "Designates early-integration fabrics",
63
63
  "NON_PROD": "Designates non-production fabrics",
64
+ "PRD": "Alternative Prod spelling",
64
65
  "PRE": "Designates pre-production fabrics",
65
66
  "PROD": "Designates production fabrics",
66
67
  "QA": "Designates quality assurance fabrics",
67
68
  "RVW": "Designates review fabrics",
68
69
  "SANDBOX": "Designates sandbox fabrics",
70
+ "SIT": "System Integration Testing",
69
71
  "STG": "Designates staging fabrics",
70
72
  "TEST": "Designates testing fabrics",
73
+ "TST": "Alternative Test spelling",
71
74
  "UAT": "Designates user acceptance testing fabrics"
72
75
  },
73
76
  "name": "FabricType",
@@ -84,6 +87,9 @@
84
87
  "PROD",
85
88
  "CORP",
86
89
  "RVW",
90
+ "PRD",
91
+ "TST",
92
+ "SIT",
87
93
  "SANDBOX"
88
94
  ],
89
95
  "doc": "Fabric group type"
datahub/metadata/schemas/MLModelKey.avsc CHANGED
@@ -75,13 +75,16 @@
75
75
  "DEV": "Designates development fabrics",
76
76
  "EI": "Designates early-integration fabrics",
77
77
  "NON_PROD": "Designates non-production fabrics",
78
+ "PRD": "Alternative Prod spelling",
78
79
  "PRE": "Designates pre-production fabrics",
79
80
  "PROD": "Designates production fabrics",
80
81
  "QA": "Designates quality assurance fabrics",
81
82
  "RVW": "Designates review fabrics",
82
83
  "SANDBOX": "Designates sandbox fabrics",
84
+ "SIT": "System Integration Testing",
83
85
  "STG": "Designates staging fabrics",
84
86
  "TEST": "Designates testing fabrics",
87
+ "TST": "Alternative Test spelling",
85
88
  "UAT": "Designates user acceptance testing fabrics"
86
89
  },
87
90
  "name": "FabricType",
@@ -98,6 +101,9 @@
98
101
  "PROD",
99
102
  "CORP",
100
103
  "RVW",
104
+ "PRD",
105
+ "TST",
106
+ "SIT",
101
107
  "SANDBOX"
102
108
  ],
103
109
  "doc": "Fabric group type"
datahub/metadata/schemas/MetadataChangeEvent.avsc CHANGED
@@ -2424,13 +2424,16 @@
2424
2424
  "DEV": "Designates development fabrics",
2425
2425
  "EI": "Designates early-integration fabrics",
2426
2426
  "NON_PROD": "Designates non-production fabrics",
2427
+ "PRD": "Alternative Prod spelling",
2427
2428
  "PRE": "Designates pre-production fabrics",
2428
2429
  "PROD": "Designates production fabrics",
2429
2430
  "QA": "Designates quality assurance fabrics",
2430
2431
  "RVW": "Designates review fabrics",
2431
2432
  "SANDBOX": "Designates sandbox fabrics",
2433
+ "SIT": "System Integration Testing",
2432
2434
  "STG": "Designates staging fabrics",
2433
2435
  "TEST": "Designates testing fabrics",
2436
+ "TST": "Alternative Test spelling",
2434
2437
  "UAT": "Designates user acceptance testing fabrics"
2435
2438
  },
2436
2439
  "name": "FabricType",
@@ -2447,6 +2450,9 @@
2447
2450
  "PROD",
2448
2451
  "CORP",
2449
2452
  "RVW",
2453
+ "PRD",
2454
+ "TST",
2455
+ "SIT",
2450
2456
  "SANDBOX"
2451
2457
  ],
2452
2458
  "doc": "Fabric group type"
@@ -7733,13 +7739,15 @@
7733
7739
  "type": "enum",
7734
7740
  "symbolDocs": {
7735
7741
  "EQUALS": "Whether the field matches the value",
7742
+ "NOT_EQUALS": "Whether the field does not match the value",
7736
7743
  "STARTS_WITH": "Whether the field value starts with the value"
7737
7744
  },
7738
7745
  "name": "PolicyMatchCondition",
7739
7746
  "namespace": "com.linkedin.pegasus2avro.policy",
7740
7747
  "symbols": [
7741
7748
  "EQUALS",
7742
- "STARTS_WITH"
7749
+ "STARTS_WITH",
7750
+ "NOT_EQUALS"
7743
7751
  ],
7744
7752
  "doc": "The matching condition in a filter criterion"
7745
7753
  },
@@ -7761,6 +7769,15 @@
7761
7769
  "name": "filter",
7762
7770
  "default": null,
7763
7771
  "doc": "Filter to apply privileges to"
7772
+ },
7773
+ {
7774
+ "type": [
7775
+ "null",
7776
+ "com.linkedin.pegasus2avro.policy.PolicyMatchFilter"
7777
+ ],
7778
+ "name": "privilegeConstraints",
7779
+ "default": null,
7780
+ "doc": "Constraints around what sub-resources operations are allowed to modify, i.e. NOT_EQUALS - cannot modify a particular defined tag, EQUALS - can only modify a particular defined tag, STARTS_WITH - can only modify a tag starting with xyz"
7764
7781
  }
7765
7782
  ],
7766
7783
  "doc": "Information used to filter DataHub resource."
datahub/utilities/stats_collections.py CHANGED
@@ -56,3 +56,7 @@ class TopKDict(DefaultDict[_KT, _VT]):
56
56
 
57
57
  def int_top_k_dict() -> TopKDict[str, int]:
58
58
  return TopKDict(int)
59
+
60
+
61
+ def float_top_k_dict() -> TopKDict[str, float]:
62
+ return TopKDict(float)