acryl-datahub 1.1.0.3rc1__py3-none-any.whl → 1.1.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (68) hide show
  1. {acryl_datahub-1.1.0.3rc1.dist-info → acryl_datahub-1.1.0.4.dist-info}/METADATA +2474 -2474
  2. {acryl_datahub-1.1.0.3rc1.dist-info → acryl_datahub-1.1.0.4.dist-info}/RECORD +68 -68
  3. datahub/_version.py +1 -1
  4. datahub/cli/check_cli.py +27 -0
  5. datahub/cli/delete_cli.py +117 -19
  6. datahub/emitter/rest_emitter.py +18 -1
  7. datahub/ingestion/api/source.py +2 -0
  8. datahub/ingestion/glossary/classification_mixin.py +5 -0
  9. datahub/ingestion/graph/client.py +42 -2
  10. datahub/ingestion/source/bigquery_v2/bigquery.py +18 -0
  11. datahub/ingestion/source/bigquery_v2/common.py +1 -1
  12. datahub/ingestion/source/dbt/dbt_cloud.py +3 -0
  13. datahub/ingestion/source/dbt/dbt_common.py +3 -1
  14. datahub/ingestion/source/dbt/dbt_core.py +3 -0
  15. datahub/ingestion/source/dremio/dremio_api.py +98 -68
  16. datahub/ingestion/source/dremio/dremio_config.py +2 -0
  17. datahub/ingestion/source/dremio/dremio_reporting.py +23 -2
  18. datahub/ingestion/source/dremio/dremio_source.py +90 -77
  19. datahub/ingestion/source/dremio/dremio_sql_queries.py +82 -21
  20. datahub/ingestion/source/file.py +3 -0
  21. datahub/ingestion/source/ge_data_profiler.py +48 -8
  22. datahub/ingestion/source/ge_profiling_config.py +11 -0
  23. datahub/ingestion/source/iceberg/iceberg.py +3 -1
  24. datahub/ingestion/source/kafka/kafka.py +16 -0
  25. datahub/ingestion/source/looker/looker_source.py +1 -0
  26. datahub/ingestion/source/powerbi/powerbi.py +1 -0
  27. datahub/ingestion/source/qlik_sense/qlik_sense.py +1 -0
  28. datahub/ingestion/source/redshift/redshift.py +21 -1
  29. datahub/ingestion/source/sac/sac.py +3 -1
  30. datahub/ingestion/source/sigma/sigma.py +1 -0
  31. datahub/ingestion/source/snowflake/snowflake_config.py +3 -6
  32. datahub/ingestion/source/snowflake/snowflake_summary.py +5 -0
  33. datahub/ingestion/source/snowflake/snowflake_utils.py +2 -7
  34. datahub/ingestion/source/snowflake/snowflake_v2.py +2 -0
  35. datahub/ingestion/source/sql/clickhouse.py +3 -1
  36. datahub/ingestion/source/sql/cockroachdb.py +0 -1
  37. datahub/ingestion/source/sql/hana.py +3 -1
  38. datahub/ingestion/source/sql/hive_metastore.py +3 -1
  39. datahub/ingestion/source/sql/mariadb.py +0 -1
  40. datahub/ingestion/source/sql/mssql/source.py +8 -1
  41. datahub/ingestion/source/sql/mysql.py +0 -1
  42. datahub/ingestion/source/sql/postgres.py +0 -1
  43. datahub/ingestion/source/sql/sql_common.py +12 -0
  44. datahub/ingestion/source/superset.py +1 -1
  45. datahub/ingestion/source/tableau/tableau.py +1 -0
  46. datahub/ingestion/source/unity/source.py +1 -0
  47. datahub/ingestion/source/usage/clickhouse_usage.py +4 -1
  48. datahub/ingestion/source/usage/starburst_trino_usage.py +3 -0
  49. datahub/metadata/_internal_schema_classes.py +25 -0
  50. datahub/metadata/schema.avsc +18 -1
  51. datahub/metadata/schemas/ContainerProperties.avsc +6 -0
  52. datahub/metadata/schemas/DataFlowInfo.avsc +6 -0
  53. datahub/metadata/schemas/DataHubPolicyInfo.avsc +12 -1
  54. datahub/metadata/schemas/DataJobInfo.avsc +6 -0
  55. datahub/metadata/schemas/DataProcessKey.avsc +6 -0
  56. datahub/metadata/schemas/DatasetKey.avsc +6 -0
  57. datahub/metadata/schemas/IcebergWarehouseInfo.avsc +6 -0
  58. datahub/metadata/schemas/MLModelDeploymentKey.avsc +6 -0
  59. datahub/metadata/schemas/MLModelGroupKey.avsc +6 -0
  60. datahub/metadata/schemas/MLModelKey.avsc +6 -0
  61. datahub/metadata/schemas/MetadataChangeEvent.avsc +18 -1
  62. datahub/sdk/main_client.py +9 -10
  63. datahub/sql_parsing/sqlglot_lineage.py +22 -0
  64. datahub/utilities/stats_collections.py +4 -0
  65. {acryl_datahub-1.1.0.3rc1.dist-info → acryl_datahub-1.1.0.4.dist-info}/WHEEL +0 -0
  66. {acryl_datahub-1.1.0.3rc1.dist-info → acryl_datahub-1.1.0.4.dist-info}/entry_points.txt +0 -0
  67. {acryl_datahub-1.1.0.3rc1.dist-info → acryl_datahub-1.1.0.4.dist-info}/licenses/LICENSE +0 -0
  68. {acryl_datahub-1.1.0.3rc1.dist-info → acryl_datahub-1.1.0.4.dist-info}/top_level.txt +0 -0
@@ -54,13 +54,16 @@
54
54
  "DEV": "Designates development fabrics",
55
55
  "EI": "Designates early-integration fabrics",
56
56
  "NON_PROD": "Designates non-production fabrics",
57
+ "PRD": "Alternative Prod spelling",
57
58
  "PRE": "Designates pre-production fabrics",
58
59
  "PROD": "Designates production fabrics",
59
60
  "QA": "Designates quality assurance fabrics",
60
61
  "RVW": "Designates review fabrics",
61
62
  "SANDBOX": "Designates sandbox fabrics",
63
+ "SIT": "System Integration Testing",
62
64
  "STG": "Designates staging fabrics",
63
65
  "TEST": "Designates testing fabrics",
66
+ "TST": "Alternative Test spelling",
64
67
  "UAT": "Designates user acceptance testing fabrics"
65
68
  },
66
69
  "name": "FabricType",
@@ -77,6 +80,9 @@
77
80
  "PROD",
78
81
  "CORP",
79
82
  "RVW",
83
+ "PRD",
84
+ "TST",
85
+ "SIT",
80
86
  "SANDBOX"
81
87
  ],
82
88
  "doc": "Fabric group type"
@@ -61,13 +61,16 @@
61
61
  "DEV": "Designates development fabrics",
62
62
  "EI": "Designates early-integration fabrics",
63
63
  "NON_PROD": "Designates non-production fabrics",
64
+ "PRD": "Alternative Prod spelling",
64
65
  "PRE": "Designates pre-production fabrics",
65
66
  "PROD": "Designates production fabrics",
66
67
  "QA": "Designates quality assurance fabrics",
67
68
  "RVW": "Designates review fabrics",
68
69
  "SANDBOX": "Designates sandbox fabrics",
70
+ "SIT": "System Integration Testing",
69
71
  "STG": "Designates staging fabrics",
70
72
  "TEST": "Designates testing fabrics",
73
+ "TST": "Alternative Test spelling",
71
74
  "UAT": "Designates user acceptance testing fabrics"
72
75
  },
73
76
  "name": "FabricType",
@@ -84,6 +87,9 @@
84
87
  "PROD",
85
88
  "CORP",
86
89
  "RVW",
90
+ "PRD",
91
+ "TST",
92
+ "SIT",
87
93
  "SANDBOX"
88
94
  ],
89
95
  "doc": "Fabric group type"
@@ -75,13 +75,16 @@
75
75
  "DEV": "Designates development fabrics",
76
76
  "EI": "Designates early-integration fabrics",
77
77
  "NON_PROD": "Designates non-production fabrics",
78
+ "PRD": "Alternative Prod spelling",
78
79
  "PRE": "Designates pre-production fabrics",
79
80
  "PROD": "Designates production fabrics",
80
81
  "QA": "Designates quality assurance fabrics",
81
82
  "RVW": "Designates review fabrics",
82
83
  "SANDBOX": "Designates sandbox fabrics",
84
+ "SIT": "System Integration Testing",
83
85
  "STG": "Designates staging fabrics",
84
86
  "TEST": "Designates testing fabrics",
87
+ "TST": "Alternative Test spelling",
85
88
  "UAT": "Designates user acceptance testing fabrics"
86
89
  },
87
90
  "name": "FabricType",
@@ -98,6 +101,9 @@
98
101
  "PROD",
99
102
  "CORP",
100
103
  "RVW",
104
+ "PRD",
105
+ "TST",
106
+ "SIT",
101
107
  "SANDBOX"
102
108
  ],
103
109
  "doc": "Fabric group type"
@@ -2424,13 +2424,16 @@
2424
2424
  "DEV": "Designates development fabrics",
2425
2425
  "EI": "Designates early-integration fabrics",
2426
2426
  "NON_PROD": "Designates non-production fabrics",
2427
+ "PRD": "Alternative Prod spelling",
2427
2428
  "PRE": "Designates pre-production fabrics",
2428
2429
  "PROD": "Designates production fabrics",
2429
2430
  "QA": "Designates quality assurance fabrics",
2430
2431
  "RVW": "Designates review fabrics",
2431
2432
  "SANDBOX": "Designates sandbox fabrics",
2433
+ "SIT": "System Integration Testing",
2432
2434
  "STG": "Designates staging fabrics",
2433
2435
  "TEST": "Designates testing fabrics",
2436
+ "TST": "Alternative Test spelling",
2434
2437
  "UAT": "Designates user acceptance testing fabrics"
2435
2438
  },
2436
2439
  "name": "FabricType",
@@ -2447,6 +2450,9 @@
2447
2450
  "PROD",
2448
2451
  "CORP",
2449
2452
  "RVW",
2453
+ "PRD",
2454
+ "TST",
2455
+ "SIT",
2450
2456
  "SANDBOX"
2451
2457
  ],
2452
2458
  "doc": "Fabric group type"
@@ -7733,13 +7739,15 @@
7733
7739
  "type": "enum",
7734
7740
  "symbolDocs": {
7735
7741
  "EQUALS": "Whether the field matches the value",
7742
+ "NOT_EQUALS": "Whether the field does not match the value",
7736
7743
  "STARTS_WITH": "Whether the field value starts with the value"
7737
7744
  },
7738
7745
  "name": "PolicyMatchCondition",
7739
7746
  "namespace": "com.linkedin.pegasus2avro.policy",
7740
7747
  "symbols": [
7741
7748
  "EQUALS",
7742
- "STARTS_WITH"
7749
+ "STARTS_WITH",
7750
+ "NOT_EQUALS"
7743
7751
  ],
7744
7752
  "doc": "The matching condition in a filter criterion"
7745
7753
  },
@@ -7761,6 +7769,15 @@
7761
7769
  "name": "filter",
7762
7770
  "default": null,
7763
7771
  "doc": "Filter to apply privileges to"
7772
+ },
7773
+ {
7774
+ "type": [
7775
+ "null",
7776
+ "com.linkedin.pegasus2avro.policy.PolicyMatchFilter"
7777
+ ],
7778
+ "name": "privilegeConstraints",
7779
+ "default": null,
7780
+ "doc": "Constraints around what sub-resources operations are allowed to modify, i.e. NOT_EQUALS - cannot modify a particular defined tag, EQUALS - can only modify a particular defined tag, STARTS_WITH - can only modify a tag starting with xyz"
7764
7781
  }
7765
7782
  ],
7766
7783
  "doc": "Information used to filter DataHub resource."
@@ -9,15 +9,6 @@ from datahub.sdk.entity_client import EntityClient
9
9
  from datahub.sdk.lineage_client import LineageClient
10
10
  from datahub.sdk.search_client import SearchClient
11
11
 
12
- try:
13
- from acryl_datahub_cloud.sdk import ( # type: ignore[import-not-found]
14
- ResolverClient,
15
- )
16
- except ImportError:
17
- from datahub.sdk.resolver_client import ( # type: ignore[assignment] # If the client is not installed, use the one from the SDK
18
- ResolverClient,
19
- )
20
-
21
12
 
22
13
  class DataHubClient:
23
14
  """Main client for interacting with DataHub.
@@ -101,7 +92,15 @@ class DataHubClient:
101
92
  return EntityClient(self)
102
93
 
103
94
  @property
104
- def resolve(self) -> ResolverClient:
95
+ def resolve(self): # type: ignore[report-untyped-call] # Not available due to circular import issues
96
+ try:
97
+ from acryl_datahub_cloud.sdk import ( # type: ignore[import-not-found]
98
+ ResolverClient,
99
+ )
100
+ except ImportError:
101
+ from datahub.sdk.resolver_client import ( # type: ignore[assignment] # If the client is not installed, use the one from the SDK
102
+ ResolverClient,
103
+ )
105
104
  return ResolverClient(self)
106
105
 
107
106
  @property
@@ -1486,6 +1486,28 @@ def _sqlglot_lineage_nocache(
1486
1486
  )
1487
1487
  except Exception as e:
1488
1488
  return SqlParsingResult.make_from_error(e)
1489
+ except BaseException as e:
1490
+ # Check if this is a PanicException from SQLGlot's Rust tokenizer
1491
+ # We use runtime type checking instead of isinstance() because pyo3_runtime
1492
+ # is only available when sqlglot[rs] is installed and may not be importable
1493
+ # at module load time, but the exception can still be raised at runtime
1494
+ if (
1495
+ e.__class__.__name__ == "PanicException"
1496
+ and e.__class__.__module__ == "pyo3_runtime"
1497
+ ):
1498
+ # Handle pyo3_runtime.PanicException from SQLGlot's Rust tokenizer.
1499
+ # pyo3_runtime.PanicException inherits from BaseException (like SystemExit or
1500
+ # KeyboardInterrupt) rather than Exception, so it bypasses normal exception handling.
1501
+ # Avoid catching BaseException, as it includes KeyboardInterrupt
1502
+ # and would prevent Ctrl+C from working.
1503
+ wrapped_exception = Exception(
1504
+ f"pyo3_runtime.PanicException during SQL parsing: {e}"
1505
+ )
1506
+ wrapped_exception.__cause__ = e
1507
+ return SqlParsingResult.make_from_error(wrapped_exception)
1508
+ else:
1509
+ # Re-raise other BaseException types (SystemExit, KeyboardInterrupt, etc.)
1510
+ raise
1489
1511
 
1490
1512
 
1491
1513
  _sqlglot_lineage_cached = functools.lru_cache(maxsize=SQL_PARSE_RESULT_CACHE_SIZE)(
@@ -56,3 +56,7 @@ class TopKDict(DefaultDict[_KT, _VT]):
56
56
 
57
57
  def int_top_k_dict() -> TopKDict[str, int]:
58
58
  return TopKDict(int)
59
+
60
+
61
+ def float_top_k_dict() -> TopKDict[str, float]:
62
+ return TopKDict(float)