acryl-datahub 1.3.0rc4__py3-none-any.whl → 1.3.0.1rc1__py3-none-any.whl

{acryl_datahub-1.3.0rc4.dist-info → acryl_datahub-1.3.0.1rc1.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.3.0rc4.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.3.0.1rc1.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=IE1A1EIFh0zNNTxCNZuHp84ueLDs5dQeRmsZJgB-YLw,321
+datahub/_version.py,sha256=R7AO3BvsCQaRcQe_NV5EtywuCZj3jLQL4SqalUnrmTc,323
 datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -100,7 +100,7 @@ datahub/cli/specific/structuredproperties_cli.py,sha256=rgx8fhI7WYq8QLwIttkMysX7
 datahub/cli/specific/user_cli.py,sha256=HaASGg25b82Q18hKyOn98gPoQfmj1gW0utDMs0iR5WY,1897
 datahub/configuration/__init__.py,sha256=5TN3a7CWNsLRHpdj-sv2bxKWF2IslvJwE6EpNMFrIS4,123
 datahub/configuration/_config_enum.py,sha256=ul2hr5gMmdLvBINicFkMNMi1ApmnmZSwNdUYYted5nk,1447
-datahub/configuration/common.py,sha256=HvMrc-q2ZocbAHj5KgBHUaQOjdWLvk_wRGDkTUVbhtU,11742
+datahub/configuration/common.py,sha256=i34Ec4tkTc2iy0ay82ZqPUeapUR3-ZKItR3gzI-0zTg,11730
 datahub/configuration/config_loader.py,sha256=hRzPFxkz-w9IqkpSa5vwCzSra1p49DyfeJNeyqGa8-4,6827
 datahub/configuration/connection_resolver.py,sha256=UsnV1_X8yivOykiifllkoKRn19eO6j_NTBWHC2Ob5Xg,1625
 datahub/configuration/datetimes.py,sha256=nayNc0mmlVKH6oVv9ud6C1dDUiZPGabW-YZxvrkosPg,2870
@@ -334,7 +334,7 @@ datahub/ingestion/source/excel/report.py,sha256=oEkeI8J6is7zB9iz4RqASu_-Q5xl36lA
 datahub/ingestion/source/excel/source.py,sha256=w_vOz4UD7BcXBBDKoo81_6-QFeOPITuXqkfjIMHCQj4,23827
 datahub/ingestion/source/excel/util.py,sha256=YYmadYuCiT-4_MfQM0YSE7wuDcE0k8o2KrlOKM9Z6eI,406
 datahub/ingestion/source/fivetran/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/fivetran/config.py,sha256=vNmnQM3oekr2dOLPria-wjCLmp27bcYypIfoA6xx5k8,10290
+datahub/ingestion/source/fivetran/config.py,sha256=6yriUMtTPMZUHqbZ9gzyFduPVt6CxzirdYSg4k-ziYI,10285
 datahub/ingestion/source/fivetran/data_classes.py,sha256=ecdUJH5BEze0yv-uFpKWPNaNmV1gORDA2XMFk0zhcBw,595
 datahub/ingestion/source/fivetran/fivetran.py,sha256=Up5wbLk7hBk9b0pqcHwW6b0H52UJj90cmLhn0QJeZ4g,14416
 datahub/ingestion/source/fivetran/fivetran_log_api.py,sha256=-ibtfgxFv08P5_X5PVqV4CocxAjRWmY858esQL5OaAQ,13697
@@ -568,7 +568,8 @@ datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=wsVD0SkGUwb-H9
 datahub/ingestion/source/tableau/tableau_validation.py,sha256=Hjbfc1AMIkGgzo5ffWXtNRjrxSxzHvw7-dYZDt4d3WE,1819
 datahub/ingestion/source/unity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/unity/analyze_profiler.py,sha256=2pqkFY30CfN4aHgFZZntjeG0hNhBytZJvXC13VfTc1I,4689
-datahub/ingestion/source/unity/config.py,sha256=A5lkm-koBDOnBSSCTzOvYlsSIxT-xbK3NcJMS6xJMaQ,20914
+datahub/ingestion/source/unity/config.py,sha256=lHvr-PGVcZ0P_2e0RuwmfSRlQRJ81astx4hQZkNrX_k,18713
+datahub/ingestion/source/unity/connection.py,sha256=iCsQhZ1vxzv1qQKTl_sFUZdmBLLIrNdu2X2V8hT7IGI,2441
 datahub/ingestion/source/unity/connection_test.py,sha256=Dwpz4AIc6ZDwq6pWmRCSCuDUgNjPP_bVAVJumgAAS4w,2661
 datahub/ingestion/source/unity/ge_profiler.py,sha256=NBRHZceq-f95iUn7u0h7cgcd9nAc48Aa-lmp_BqE0As,8409
 datahub/ingestion/source/unity/hive_metastore_proxy.py,sha256=IAWWJjaW0si_UF52Se2D7wmdYRY_afUG4QlVmQu6xaw,15351
@@ -594,7 +595,7 @@ datahub/ingestion/source_config/csv_enricher.py,sha256=IROxxfFJA56dHkmmbjjhb7h1p
 datahub/ingestion/source_config/operation_config.py,sha256=hxF2RM0jk0HUPXYiliMniXBC-wz-ZPcs90ZGLfHT8rE,3924
 datahub/ingestion/source_config/pulsar.py,sha256=zi3QTAw8CzzuwXgU-GUCuLyneT5pxHsLqZFyd15ECYs,5604
 datahub/ingestion/source_report/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source_report/ingestion_stage.py,sha256=0MY39QetRovYd1iBNSy0OW11YyaOsPaqhQi-1svmAcY,3106
+datahub/ingestion/source_report/ingestion_stage.py,sha256=7r_WUQ3mHH8nAlVu7QUZcj8CwTRORXRhAaATTgkVZ8c,3111
 datahub/ingestion/source_report/pulsar.py,sha256=f6CMNw8TyPp3tuSGsLLPEhSvoQLXwxtaaM6GmNvsANU,1119
 datahub/ingestion/source_report/time_window.py,sha256=9yI5l2S1DcF7ClvUHLeN8m62I5vlhV9k-aQqSZh2l7w,229
 datahub/ingestion/transformer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -1018,7 +1019,7 @@ datahub/sql_parsing/sql_parsing_aggregator.py,sha256=kxxSVe3YNoz_T2OG6-F30ZuXNSX
 datahub/sql_parsing/sql_parsing_common.py,sha256=cZ4WvVyHZuXDGjnBvKMX2_fz2EMextB5WQWcK0_saBo,3155
 datahub/sql_parsing/sql_parsing_result_utils.py,sha256=prwWTj1EB2fRPv1eMB4EkpFNafIYAt-X8TIK0NWqank,796
 datahub/sql_parsing/sqlglot_lineage.py,sha256=Zli78TtN8ow-uqNl_oloWT642a5bDGn22-FF0O0tqrs,66658
-datahub/sql_parsing/sqlglot_utils.py,sha256=zH8V9tAcSVO7Y8I3sIKPhs0D_9HzdNBlranBDmk1NB4,15454
+datahub/sql_parsing/sqlglot_utils.py,sha256=2CwrnDsmjcDlwjdBbmaiTVbWZjEAm3fr4ulEvTr3cZQ,15343
 datahub/sql_parsing/tool_meta_extractor.py,sha256=5JsLPcKjuXSrPGxNIhRvX72dFPmlV33-hyvhJwlWxCY,7543
 datahub/telemetry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/telemetry/stats.py,sha256=TwaQisQlD2Bk0uw__pP6u3Ovz9r-Ip4pCwpnto4r5e0,959
@@ -1125,8 +1126,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.3.0rc4.dist-info/METADATA,sha256=nsoIeC_TnJAbmM1wNUIO_dpEiFFZR2xr1AiwJzuJnqk,184332
-acryl_datahub-1.3.0rc4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.3.0rc4.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
-acryl_datahub-1.3.0rc4.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.3.0rc4.dist-info/RECORD,,
+acryl_datahub-1.3.0.1rc1.dist-info/METADATA,sha256=7XyrCvdsnuEzzHSrBD4-gyr723Kq-Nj2Jawvh5sOz1M,184432
+acryl_datahub-1.3.0.1rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.3.0.1rc1.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
+acryl_datahub-1.3.0.1rc1.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.3.0.1rc1.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.3.0rc4"
+__version__ = "1.3.0.1rc1"
 
 
 def is_dev_mode() -> bool:

datahub/configuration/common.py

@@ -173,11 +173,11 @@ class ConnectionModel(BaseModel):
     """Represents the config associated with a connection"""
 
     class Config:
-        if PYDANTIC_VERSION_2:  # noqa: SIM108
+        if PYDANTIC_VERSION_2:
             extra = "allow"
         else:
             extra = Extra.allow
-        underscore_attrs_are_private = True
+            underscore_attrs_are_private = True
 
 
 class TransformerSemantics(ConfigEnum):

datahub/ingestion/source/fivetran/config.py

@@ -29,9 +29,7 @@ from datahub.ingestion.source.state.stale_entity_removal_handler import (
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionConfigBase,
 )
-from datahub.ingestion.source.unity.config import (
-    UnityCatalogConnectionConfig,
-)
+from datahub.ingestion.source.unity.connection import UnityCatalogConnectionConfig
 from datahub.utilities.lossy_collections import LossyList
 from datahub.utilities.perf_timer import PerfTimer
 

datahub/ingestion/source/unity/config.py

@@ -2,7 +2,6 @@ import logging
 import os
 from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional, Union
-from urllib.parse import urlparse
 
 import pydantic
 from pydantic import Field
@@ -20,10 +19,8 @@ from datahub.configuration.source_common import (
 )
 from datahub.configuration.validate_field_removal import pydantic_removed_field
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
-from datahub.ingestion.source.ge_data_profiler import DATABRICKS
 from datahub.ingestion.source.ge_profiling_config import GEProfilingConfig
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
-from datahub.ingestion.source.sql.sqlalchemy_uri import make_sqlalchemy_uri
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StatefulStaleMetadataRemovalConfig,
 )
@@ -31,6 +28,7 @@ from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionConfigBase,
     StatefulProfilingConfigMixin,
 )
+from datahub.ingestion.source.unity.connection import UnityCatalogConnectionConfig
 from datahub.ingestion.source.usage.usage_common import BaseUsageConfig
 from datahub.ingestion.source_config.operation_config import (
     OperationConfig,
@@ -132,55 +130,6 @@ class UnityCatalogGEProfilerConfig(UnityCatalogProfilerConfig, GEProfilingConfig
     )
 
 
-class UnityCatalogConnectionConfig(ConfigModel):
-    """
-    Configuration for connecting to Databricks Unity Catalog.
-    Contains only connection-related fields that can be reused across different sources.
-    """
-
-    scheme: str = DATABRICKS
-    token: str = pydantic.Field(description="Databricks personal access token")
-    workspace_url: str = pydantic.Field(
-        description="Databricks workspace url. e.g. https://my-workspace.cloud.databricks.com"
-    )
-    warehouse_id: Optional[str] = pydantic.Field(
-        default=None,
-        description=(
-            "SQL Warehouse id, for running queries. Must be explicitly provided to enable SQL-based features. "
-            "Required for the following features that need SQL access: "
-            "1) Tag extraction (include_tags=True) - queries system.information_schema.tags "
-            "2) Hive Metastore catalog (include_hive_metastore=True) - queries legacy hive_metastore catalog "
-            "3) System table lineage (lineage_data_source=SYSTEM_TABLES) - queries system.access.table_lineage/column_lineage "
-            "4) Data profiling (profiling.enabled=True) - runs SELECT/ANALYZE queries on tables. "
-            "When warehouse_id is missing, these features will be automatically disabled (with warnings) to allow ingestion to continue."
-        ),
-    )
-
-    extra_client_options: Dict[str, Any] = Field(
-        default={},
-        description="Additional options to pass to Databricks SQLAlchemy client.",
-    )
-
-    def __init__(self, **data: Any):
-        super().__init__(**data)
-
-    def get_sql_alchemy_url(self, database: Optional[str] = None) -> str:
-        uri_opts = {"http_path": f"/sql/1.0/warehouses/{self.warehouse_id}"}
-        if database:
-            uri_opts["catalog"] = database
-        return make_sqlalchemy_uri(
-            scheme=self.scheme,
-            username="token",
-            password=self.token,
-            at=urlparse(self.workspace_url).netloc,
-            db=database,
-            uri_opts=uri_opts,
-        )
-
-    def get_options(self) -> dict:
-        return self.extra_client_options
-
-
 class UnityCatalogSourceConfig(
     UnityCatalogConnectionConfig,
     SQLCommonConfig,

datahub/ingestion/source/unity/connection.py

@@ -0,0 +1,61 @@
+"""Databricks Unity Catalog connection configuration."""
+
+from typing import Any, Dict, Optional
+from urllib.parse import urlparse
+
+import pydantic
+from pydantic import Field
+
+from datahub.configuration.common import ConfigModel
+from datahub.ingestion.source.sql.sqlalchemy_uri import make_sqlalchemy_uri
+
+DATABRICKS = "databricks"
+
+
+class UnityCatalogConnectionConfig(ConfigModel):
+    """
+    Configuration for connecting to Databricks Unity Catalog.
+    Contains only connection-related fields that can be reused across different sources.
+    """
+
+    scheme: str = DATABRICKS
+    token: str = pydantic.Field(description="Databricks personal access token")
+    workspace_url: str = pydantic.Field(
+        description="Databricks workspace url. e.g. https://my-workspace.cloud.databricks.com"
+    )
+    warehouse_id: Optional[str] = pydantic.Field(
+        default=None,
+        description=(
+            "SQL Warehouse id, for running queries. Must be explicitly provided to enable SQL-based features. "
+            "Required for the following features that need SQL access: "
+            "1) Tag extraction (include_tags=True) - queries system.information_schema.tags "
+            "2) Hive Metastore catalog (include_hive_metastore=True) - queries legacy hive_metastore catalog "
+            "3) System table lineage (lineage_data_source=SYSTEM_TABLES) - queries system.access.table_lineage/column_lineage "
+            "4) Data profiling (profiling.enabled=True) - runs SELECT/ANALYZE queries on tables. "
+            "When warehouse_id is missing, these features will be automatically disabled (with warnings) to allow ingestion to continue."
+        ),
+    )
+
+    extra_client_options: Dict[str, Any] = Field(
+        default={},
+        description="Additional options to pass to Databricks SQLAlchemy client.",
+    )
+
+    def __init__(self, **data: Any):
+        super().__init__(**data)
+
+    def get_sql_alchemy_url(self, database: Optional[str] = None) -> str:
+        uri_opts = {"http_path": f"/sql/1.0/warehouses/{self.warehouse_id}"}
+        if database:
+            uri_opts["catalog"] = database
+        return make_sqlalchemy_uri(
+            scheme=self.scheme,
+            username="token",
+            password=self.token,
+            at=urlparse(self.workspace_url).netloc,
+            db=database,
+            uri_opts=uri_opts,
+        )
+
+    def get_options(self) -> dict:
+        return self.extra_client_options

datahub/ingestion/source_report/ingestion_stage.py

@@ -4,7 +4,6 @@ from contextlib import AbstractContextManager
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from enum import Enum
-from typing import Tuple
 
 from datahub.utilities.perf_timer import PerfTimer
 from datahub.utilities.stats_collections import TopKDict
@@ -38,9 +37,7 @@ class IngestionStageReport:
     ingestion_high_stage_seconds: dict[IngestionHighStage, float] = field(
         default_factory=lambda: defaultdict(float)
     )
-    ingestion_stage_durations: TopKDict[Tuple[IngestionHighStage, str], float] = field(
-        default_factory=TopKDict
-    )
+    ingestion_stage_durations: TopKDict[str, float] = field(default_factory=TopKDict)
 
     def new_stage(
         self, stage: str, high_stage: IngestionHighStage = IngestionHighStage._UNDEFINED
@@ -81,9 +78,9 @@ class IngestionStageContext(AbstractContextManager):
                 f"Time spent in stage <{self._ingestion_stage}>: {elapsed} seconds",
                 stacklevel=2,
             )
-            self._report.ingestion_stage_durations[
-                (self._high_stage, self._ingestion_stage)
-            ] = elapsed
+            # Store tuple as string to avoid serialization errors
+            key = f"({self._high_stage.value}, {self._ingestion_stage})"
+            self._report.ingestion_stage_durations[key] = elapsed
         else:
             logger.info(
                 f"Time spent in stage <{self._high_stage.value}>: {elapsed} seconds",

datahub/sql_parsing/sqlglot_utils.py

@@ -40,9 +40,6 @@ def _get_dialect_str(platform: str) -> str:
         # let the fuzzy resolution logic handle it.
         # MariaDB is a fork of MySQL, so we reuse the same dialect.
         return "mysql, normalization_strategy = lowercase"
-    # Dremio is based upon drill. Not 100% compatibility
-    elif platform == "dremio":
-        return "drill"
     else:
         return platform