acryl-datahub 1.2.0.11rc5__py3-none-any.whl → 1.3.0rc5__py3-none-any.whl

{acryl_datahub-1.2.0.11rc5.dist-info → acryl_datahub-1.3.0rc5.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.2.0.11rc5.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.3.0rc5.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=Uoww4PlvFNDBfp6ZJQjTb1JZU6f9boAYIIUAiM-zWns,324
+datahub/_version.py,sha256=tQwOz8j8lgLnVKUe2eNZLleYAQzj2kRWOmo8yGoOMOk,321
 datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -334,7 +334,7 @@ datahub/ingestion/source/excel/report.py,sha256=oEkeI8J6is7zB9iz4RqASu_-Q5xl36lA
 datahub/ingestion/source/excel/source.py,sha256=w_vOz4UD7BcXBBDKoo81_6-QFeOPITuXqkfjIMHCQj4,23827
 datahub/ingestion/source/excel/util.py,sha256=YYmadYuCiT-4_MfQM0YSE7wuDcE0k8o2KrlOKM9Z6eI,406
 datahub/ingestion/source/fivetran/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/fivetran/config.py,sha256=vNmnQM3oekr2dOLPria-wjCLmp27bcYypIfoA6xx5k8,10290
+datahub/ingestion/source/fivetran/config.py,sha256=6yriUMtTPMZUHqbZ9gzyFduPVt6CxzirdYSg4k-ziYI,10285
 datahub/ingestion/source/fivetran/data_classes.py,sha256=ecdUJH5BEze0yv-uFpKWPNaNmV1gORDA2XMFk0zhcBw,595
 datahub/ingestion/source/fivetran/fivetran.py,sha256=Up5wbLk7hBk9b0pqcHwW6b0H52UJj90cmLhn0QJeZ4g,14416
 datahub/ingestion/source/fivetran/fivetran_log_api.py,sha256=-ibtfgxFv08P5_X5PVqV4CocxAjRWmY858esQL5OaAQ,13697
@@ -568,7 +568,8 @@ datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=wsVD0SkGUwb-H9
 datahub/ingestion/source/tableau/tableau_validation.py,sha256=Hjbfc1AMIkGgzo5ffWXtNRjrxSxzHvw7-dYZDt4d3WE,1819
 datahub/ingestion/source/unity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/unity/analyze_profiler.py,sha256=2pqkFY30CfN4aHgFZZntjeG0hNhBytZJvXC13VfTc1I,4689
-datahub/ingestion/source/unity/config.py,sha256=A5lkm-koBDOnBSSCTzOvYlsSIxT-xbK3NcJMS6xJMaQ,20914
+datahub/ingestion/source/unity/config.py,sha256=lHvr-PGVcZ0P_2e0RuwmfSRlQRJ81astx4hQZkNrX_k,18713
+datahub/ingestion/source/unity/connection.py,sha256=iCsQhZ1vxzv1qQKTl_sFUZdmBLLIrNdu2X2V8hT7IGI,2441
 datahub/ingestion/source/unity/connection_test.py,sha256=Dwpz4AIc6ZDwq6pWmRCSCuDUgNjPP_bVAVJumgAAS4w,2661
 datahub/ingestion/source/unity/ge_profiler.py,sha256=NBRHZceq-f95iUn7u0h7cgcd9nAc48Aa-lmp_BqE0As,8409
 datahub/ingestion/source/unity/hive_metastore_proxy.py,sha256=IAWWJjaW0si_UF52Se2D7wmdYRY_afUG4QlVmQu6xaw,15351
@@ -1125,8 +1126,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.2.0.11rc5.dist-info/METADATA,sha256=Yb9MOwBUZG-CqJKSTyxscoMetKu6ZLIz6SNrrWEzmaI,184341
-acryl_datahub-1.2.0.11rc5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.2.0.11rc5.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
-acryl_datahub-1.2.0.11rc5.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.2.0.11rc5.dist-info/RECORD,,
+acryl_datahub-1.3.0rc5.dist-info/METADATA,sha256=k7aciCTdmq9GKnJBJOrO9nQZt5Os9gcongq5sM9F330,184426
+acryl_datahub-1.3.0rc5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.3.0rc5.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
+acryl_datahub-1.3.0rc5.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.3.0rc5.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.2.0.11rc5"
+__version__ = "1.3.0rc5"
 
 
 def is_dev_mode() -> bool:

datahub/ingestion/source/fivetran/config.py

@@ -29,9 +29,7 @@ from datahub.ingestion.source.state.stale_entity_removal_handler import (
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionConfigBase,
 )
-from datahub.ingestion.source.unity.config import (
-    UnityCatalogConnectionConfig,
-)
+from datahub.ingestion.source.unity.connection import UnityCatalogConnectionConfig
 from datahub.utilities.lossy_collections import LossyList
 from datahub.utilities.perf_timer import PerfTimer
 

datahub/ingestion/source/unity/config.py

@@ -2,7 +2,6 @@ import logging
 import os
 from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional, Union
-from urllib.parse import urlparse
 
 import pydantic
 from pydantic import Field
@@ -20,10 +19,8 @@ from datahub.configuration.source_common import (
 )
 from datahub.configuration.validate_field_removal import pydantic_removed_field
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
-from datahub.ingestion.source.ge_data_profiler import DATABRICKS
 from datahub.ingestion.source.ge_profiling_config import GEProfilingConfig
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
-from datahub.ingestion.source.sql.sqlalchemy_uri import make_sqlalchemy_uri
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StatefulStaleMetadataRemovalConfig,
 )
@@ -31,6 +28,7 @@ from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionConfigBase,
     StatefulProfilingConfigMixin,
 )
+from datahub.ingestion.source.unity.connection import UnityCatalogConnectionConfig
 from datahub.ingestion.source.usage.usage_common import BaseUsageConfig
 from datahub.ingestion.source_config.operation_config import (
     OperationConfig,
@@ -132,55 +130,6 @@ class UnityCatalogGEProfilerConfig(UnityCatalogProfilerConfig, GEProfilingConfig
     )
 
 
-class UnityCatalogConnectionConfig(ConfigModel):
-    """
-    Configuration for connecting to Databricks Unity Catalog.
-    Contains only connection-related fields that can be reused across different sources.
-    """
-
-    scheme: str = DATABRICKS
-    token: str = pydantic.Field(description="Databricks personal access token")
-    workspace_url: str = pydantic.Field(
-        description="Databricks workspace url. e.g. https://my-workspace.cloud.databricks.com"
-    )
-    warehouse_id: Optional[str] = pydantic.Field(
-        default=None,
-        description=(
-            "SQL Warehouse id, for running queries. Must be explicitly provided to enable SQL-based features. "
-            "Required for the following features that need SQL access: "
-            "1) Tag extraction (include_tags=True) - queries system.information_schema.tags "
-            "2) Hive Metastore catalog (include_hive_metastore=True) - queries legacy hive_metastore catalog "
-            "3) System table lineage (lineage_data_source=SYSTEM_TABLES) - queries system.access.table_lineage/column_lineage "
-            "4) Data profiling (profiling.enabled=True) - runs SELECT/ANALYZE queries on tables. "
-            "When warehouse_id is missing, these features will be automatically disabled (with warnings) to allow ingestion to continue."
-        ),
-    )
-
-    extra_client_options: Dict[str, Any] = Field(
-        default={},
-        description="Additional options to pass to Databricks SQLAlchemy client.",
-    )
-
-    def __init__(self, **data: Any):
-        super().__init__(**data)
-
-    def get_sql_alchemy_url(self, database: Optional[str] = None) -> str:
-        uri_opts = {"http_path": f"/sql/1.0/warehouses/{self.warehouse_id}"}
-        if database:
-            uri_opts["catalog"] = database
-        return make_sqlalchemy_uri(
-            scheme=self.scheme,
-            username="token",
-            password=self.token,
-            at=urlparse(self.workspace_url).netloc,
-            db=database,
-            uri_opts=uri_opts,
-        )
-
-    def get_options(self) -> dict:
-        return self.extra_client_options
-
-
 class UnityCatalogSourceConfig(
     UnityCatalogConnectionConfig,
     SQLCommonConfig,

datahub/ingestion/source/unity/connection.py

@@ -0,0 +1,61 @@
+"""Databricks Unity Catalog connection configuration."""
+
+from typing import Any, Dict, Optional
+from urllib.parse import urlparse
+
+import pydantic
+from pydantic import Field
+
+from datahub.configuration.common import ConfigModel
+from datahub.ingestion.source.sql.sqlalchemy_uri import make_sqlalchemy_uri
+
+DATABRICKS = "databricks"
+
+
+class UnityCatalogConnectionConfig(ConfigModel):
+    """
+    Configuration for connecting to Databricks Unity Catalog.
+    Contains only connection-related fields that can be reused across different sources.
+    """
+
+    scheme: str = DATABRICKS
+    token: str = pydantic.Field(description="Databricks personal access token")
+    workspace_url: str = pydantic.Field(
+        description="Databricks workspace url. e.g. https://my-workspace.cloud.databricks.com"
+    )
+    warehouse_id: Optional[str] = pydantic.Field(
+        default=None,
+        description=(
+            "SQL Warehouse id, for running queries. Must be explicitly provided to enable SQL-based features. "
+            "Required for the following features that need SQL access: "
+            "1) Tag extraction (include_tags=True) - queries system.information_schema.tags "
+            "2) Hive Metastore catalog (include_hive_metastore=True) - queries legacy hive_metastore catalog "
+            "3) System table lineage (lineage_data_source=SYSTEM_TABLES) - queries system.access.table_lineage/column_lineage "
+            "4) Data profiling (profiling.enabled=True) - runs SELECT/ANALYZE queries on tables. "
+            "When warehouse_id is missing, these features will be automatically disabled (with warnings) to allow ingestion to continue."
+        ),
+    )
+
+    extra_client_options: Dict[str, Any] = Field(
+        default={},
+        description="Additional options to pass to Databricks SQLAlchemy client.",
+    )
+
+    def __init__(self, **data: Any):
+        super().__init__(**data)
+
+    def get_sql_alchemy_url(self, database: Optional[str] = None) -> str:
+        uri_opts = {"http_path": f"/sql/1.0/warehouses/{self.warehouse_id}"}
+        if database:
+            uri_opts["catalog"] = database
+        return make_sqlalchemy_uri(
+            scheme=self.scheme,
+            username="token",
+            password=self.token,
+            at=urlparse(self.workspace_url).netloc,
+            db=database,
+            uri_opts=uri_opts,
+        )
+
+    def get_options(self) -> dict:
+        return self.extra_client_options