openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/source/database/databricks/queries.py

@@ -87,28 +87,98 @@ DATABRICKS_GET_COLUMN_TAGS = textwrap.dedent(
 
 DATABRICKS_DDL = "SHOW CREATE TABLE `{table_name}`"
 
-DATABRICKS_GET_TABLE_LINEAGE_FOR_JOB = textwrap.dedent(
+DATABRICKS_GET_TABLE_LINEAGE_FOR_JOB = """
+SELECT 
+    entity_id AS job_id,
+    source_table_full_name,
+    target_table_full_name
+FROM system.access.table_lineage
+WHERE entity_type = 'JOB'
+    AND event_time >= current_date() - INTERVAL {lookback_days} DAYS
+    AND source_table_full_name IS NOT NULL
+    AND target_table_full_name IS NOT NULL
+GROUP BY entity_id, source_table_full_name, target_table_full_name
+"""
+
+DATABRICKS_GET_COLUMN_LINEAGE_FOR_JOB = """
+SELECT
+    entity_id as job_id,
+    source_table_full_name,
+    source_column_name,
+    target_table_full_name,
+    target_column_name
+FROM system.access.column_lineage
+WHERE entity_type = 'JOB'
+    AND event_time >= current_date() - INTERVAL {lookback_days} DAYS
+    AND source_table_full_name IS NOT NULL
+    AND target_table_full_name IS NOT NULL
+    AND source_column_name IS NOT NULL
+    AND target_column_name IS NOT NULL
+GROUP BY
+    entity_id,
+    source_table_full_name,
+    source_column_name,
+    target_table_full_name,
+    target_column_name
+"""
+
+# Test connection queries
+TEST_VIEW_DEFINITIONS = textwrap.dedent(
+    """
+    SELECT
+        TABLE_NAME,
+        TABLE_SCHEMA,
+        VIEW_DEFINITION
+    FROM INFORMATION_SCHEMA.VIEWS
+    WHERE VIEW_DEFINITION IS NOT NULL
+    LIMIT 1
+    """
+)
+
+TEST_CATALOG_TAGS = textwrap.dedent(
+    """
+    SELECT COUNT(*) as count
+    FROM `{database_name}`.information_schema.catalog_tags
+    WHERE 1=0
+    """
+)
+
+TEST_SCHEMA_TAGS = textwrap.dedent(
+    """
+    SELECT COUNT(*) as count
+    FROM `{database_name}`.information_schema.schema_tags
+    WHERE 1=0
+    """
+)
+
+TEST_TABLE_TAGS = textwrap.dedent(
+    """
+    SELECT COUNT(*) as count
+    FROM `{database_name}`.information_schema.table_tags
+    WHERE 1=0
+    """
+)
+
+TEST_COLUMN_TAGS = textwrap.dedent(
+    """
+    SELECT COUNT(*) as count
+    FROM `{database_name}`.information_schema.column_tags
+    WHERE 1=0
+    """
+)
+
+TEST_TABLE_LINEAGE = textwrap.dedent(
     """
-    SELECT DISTINCT
-        entity_id as job_id,
-        entity_type as job_type,
-        source_table_full_name as source_table_full_name,
-        target_table_full_name as target_table_full_name
+    SELECT COUNT(*) as count
     FROM system.access.table_lineage
-    WHERE entity_type ILIKE 'job'
+    WHERE 1=0
     """
 )
 
-DATABRICKS_GET_COLUMN_LINEAGE_FOR_JOB = textwrap.dedent(
+TEST_COLUMN_LINEAGE = textwrap.dedent(
     """
-    SELECT DISTINCT
-        entity_id as job_id,
-        entity_type as job_type,
-        source_table_full_name as source_table_full_name,
-        source_column_name as source_column_name,
-        target_table_full_name as target_table_full_name,
-        target_column_name as target_column_name
+    SELECT COUNT(*) as count
     FROM system.access.column_lineage
-    WHERE entity_type ILIKE 'job'
+    WHERE 1=0
     """
 )

metadata/ingestion/source/database/databricks/service_spec.py

@@ -1,6 +1,9 @@
 from metadata.data_quality.interface.sqlalchemy.databricks.test_suite_interface import (
     DatabricksTestSuiteInterface,
 )
+from metadata.ingestion.source.database.databricks.data_diff.data_diff import (
+    DatabricksTableParameter,
+)
 from metadata.ingestion.source.database.databricks.lineage import (
     DatabricksLineageSource,
 )
@@ -19,4 +22,5 @@ ServiceSpec = DefaultDatabaseSpec(
     profiler_class=DatabricksProfilerInterface,
     test_suite_class=DatabricksTestSuiteInterface,
     sampler_class=DatabricksSamplerInterface,
+    data_diff=DatabricksTableParameter,
 )

metadata/ingestion/source/database/dbt/constants.py

@@ -105,7 +105,6 @@ class SkipResourceTypeEnum(Enum):
 
     ANALYSIS = "analysis"
     TEST = "test"
-    SOURCE = "source"
 
 
 class CompiledQueriesEnum(Enum):

metadata/ingestion/source/database/dbt/metadata.py

@@ -15,7 +15,7 @@ DBT source methods.
 import traceback
 from copy import deepcopy
 from datetime import datetime
-from typing import Any, Iterable, List, Optional
+from typing import Any, Iterable, List, Optional, Union
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest
@@ -346,6 +346,9 @@ class DbtSource(DbtServiceSource):
                         )
                     )
             try:
+                # Deduplicate tags before building FQNs
+                dbt_tags_list = list(set(dbt_tags_list)) if dbt_tags_list else []
+
                 # Create all the tags added
                 dbt_tag_labels = [
                     fqn.build(
@@ -443,8 +446,12 @@ class DbtSource(DbtServiceSource):
                 ),
                 fetch_multiple_entities=True,
             )
+
+            if not table_entities:
+                return None
+
             logger.debug(
-                f"Found table entities from {fqn_search_string}: {table_entities}"
+                f"Found table entities from {fqn_search_string}: {len(table_entities)} entities"
             )
             return (
                 next(iter(filter(None, table_entities)), None)
@@ -455,6 +462,10 @@ class DbtSource(DbtServiceSource):
         try:
             table_entity = search_table(table_fqn)
             if table_entity:
+                logger.debug(
+                    f"Using Table Entity: {table_entity.fullyQualifiedName.root}"
+                    f"with id {table_entity.id}"
+                )
                 return table_entity
 
             if self.source_config.searchAcrossDatabases:
@@ -483,7 +494,9 @@ class DbtSource(DbtServiceSource):
             )
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.warning(f"Failed to get table entity from OpenMetadata: {exc}")
+            logger.warning(
+                f"Failed to get table entity '{table_fqn}' from OpenMetadata: {exc}"
+            )
 
         return None
 
@@ -511,6 +524,7 @@ class DbtSource(DbtServiceSource):
             self.context.get().exposures = {}
             self.context.get().dbt_tests = {}
             self.context.get().run_results_generate_time = None
+
             # Since we'll be processing multiple run_results for a single project
             # we'll only consider the first run_results generated_at time
             if (
@@ -520,6 +534,9 @@ class DbtSource(DbtServiceSource):
                 self.context.get().run_results_generate_time = (
                     dbt_objects.dbt_run_results[0].metadata.generated_at
                 )
+            dbt_project_name = getattr(
+                dbt_objects.dbt_manifest.metadata, "project_name", None
+            )
             for key, manifest_node in manifest_entities.items():
                 try:
                     resource_type = getattr(
@@ -543,7 +560,7 @@ class DbtSource(DbtServiceSource):
 
                     if (
                         dbt_objects.dbt_sources
-                        and resource_type == SkipResourceTypeEnum.SOURCE.value
+                        and resource_type == DbtCommonEnum.SOURCE.value
                     ):
                         self.add_dbt_sources(
                             key,
@@ -616,7 +633,8 @@ class DbtSource(DbtServiceSource):
 
                     if table_entity := self._get_table_entity(table_fqn=table_fqn):
                         logger.debug(
-                            f"Using Table Entity for datamodel: {table_entity}"
+                            f"Using Table Entity for datamodel: {table_entity.fullyQualifiedName.root}"
+                            f"with id {table_entity.id}"
                         )
                         data_model_link = DataModelLink(
                             table_entity=table_entity,
@@ -644,6 +662,7 @@ class DbtSource(DbtServiceSource):
                                     catalog_node=catalog_node,
                                 ),
                                 tags=dbt_table_tags_list or [],
+                                dbtSourceProject=dbt_project_name,
                             ),
                         )
                         yield Either(right=data_model_link)
@@ -708,20 +727,6 @@ class DbtSource(DbtServiceSource):
                     )
                     continue
 
-        if dbt_node.resource_type == SkipResourceTypeEnum.SOURCE.value:
-            parent_fqn = fqn.build(
-                self.metadata,
-                entity_type=Table,
-                service_name=self.config.serviceName,
-                database_name=get_corrected_name(dbt_node.database),
-                schema_name=get_corrected_name(dbt_node.schema_),
-                table_name=dbt_node.name,
-            )
-
-            # check if the parent table exists in OM before adding it to the upstream list
-            if self._get_table_entity(table_fqn=parent_fqn):
-                upstream_nodes.append(parent_fqn)
-
         return upstream_nodes
 
     def parse_data_model_columns(
@@ -914,6 +919,7 @@ class DbtSource(DbtServiceSource):
                 query_fqn = fqn._build(  # pylint: disable=protected-access
                     *source_elements[-3:]
                 )
+                query_fqn = ".".join([f'"{i}"' for i in query_fqn.split(".")])
                 query = (
                     f"create table {query_fqn} as {data_model_link.datamodel.sql.root}"
                 )
@@ -924,7 +930,7 @@ class DbtSource(DbtServiceSource):
                 lineages = get_lineage_by_query(
                     self.metadata,
                     query=query,
-                    service_name=source_elements[0],
+                    service_names=source_elements[0],
                     database_name=source_elements[1],
                     schema_name=source_elements[2],
                     dialect=dialect,
@@ -948,6 +954,9 @@ class DbtSource(DbtServiceSource):
     def create_dbt_exposures_lineage(
         self, exposure_spec: dict
     ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Method to process dbt exposure lineage
+        """
         to_entity = exposure_spec[DbtCommonEnum.EXPOSURE]
         upstream = exposure_spec[DbtCommonEnum.UPSTREAM]
         manifest_node = exposure_spec[DbtCommonEnum.MANIFEST_NODE]
@@ -1020,6 +1029,23 @@ class DbtSource(DbtServiceSource):
                     or []
                 )
 
+            if dbt_meta_info.openmetadata and dbt_meta_info.openmetadata.tags:
+                for tag_fqn in dbt_meta_info.openmetadata.tags:
+                    # Parse classification.tag format
+                    tag_parts = tag_fqn.split(fqn.FQN_SEPARATOR)
+                    if len(tag_parts) >= 2:
+                        classification_name = tag_parts[0]
+                        tag_name = fqn.FQN_SEPARATOR.join(tag_parts[1:])
+                        dbt_table_tags_list.extend(
+                            get_tag_labels(
+                                metadata=self.metadata,
+                                tags=[tag_name],
+                                classification_name=classification_name,
+                                include_tags=True,
+                            )
+                            or []
+                        )
+
         except Exception as exc:  # pylint: disable=broad-except
             logger.debug(traceback.format_exc())
             logger.warning(f"Failed to process meta dbt Tags and GlossaryTerms: {exc}")
@@ -1319,7 +1345,7 @@ class DbtSource(DbtServiceSource):
                         )
                     except APIError as err:
                         if err.code != 409:
-                            raise APIError(err) from err
+                            raise err
 
         except Exception as err:  # pylint: disable=broad-except
             logger.debug(traceback.format_exc())

metadata/ingestion/source/database/dbt/models.py

@@ -40,6 +40,7 @@ class DbtFilteredModel(BaseModel):
 class DbtMetaGlossaryTier(BaseModel):
     tier: Optional[str] = None
     glossary: Optional[List[str]] = None
+    tags: Optional[List[str]] = None
 
 
 class DbtMeta(BaseModel):

metadata/ingestion/source/database/exasol/connection.py

@@ -19,6 +19,7 @@ from metadata.ingestion.connections.builders import (
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.exasol.queries import EXASOL_TEST_GET_QUERIES
 from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
@@ -89,10 +90,15 @@ def test_connection(
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
+
+    queries = {
+        "GetQueries": EXASOL_TEST_GET_QUERIES,
+    }
     return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        queries=queries,
         timeout_seconds=timeout_seconds,
     )

metadata/ingestion/source/database/exasol/lineage.py

@@ -0,0 +1,24 @@
+from metadata.ingestion.lineage.models import Dialect
+from metadata.ingestion.source.database.exasol.queries import EXASOL_SQL_STATEMENT
+from metadata.ingestion.source.database.exasol.query_parser import (
+    ExasolQueryParserSource,
+)
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class ExasolLineageSource(ExasolQueryParserSource, LineageSource):
+    """
+    Exasol class for Lineage
+    """
+
+    dialect = Dialect.EXASOL
+    sql_stmt = EXASOL_SQL_STATEMENT
+    filters = """
+        AND (
+            s.command_name IN ('MERGE', 'UPDATE', 'CREATE TABLE AS', 'CREATE VIEW')
+            OR (s.command_name = 'INSERT' AND LOWER(s.sql_text) LIKE '%insert%into%select%from%')
+    )
+    """

metadata/ingestion/source/database/exasol/metadata.py

@@ -1,5 +1,7 @@
 from typing import Optional, cast
 
+from sqlalchemy.engine.reflection import Inspector
+
 from metadata.generated.schema.entity.services.connections.database.exasolConnection import (
     ExasolConnection,
 )
@@ -9,6 +11,10 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
+from metadata.utils.sqlalchemy_utils import get_all_table_ddls, get_table_ddl
+
+Inspector.get_all_table_ddls = get_all_table_ddls
+Inspector.get_table_ddl = get_table_ddl
 
 
 class ExasolSource(CommonDbSourceService):

metadata/ingestion/source/database/exasol/queries.py

@@ -1,3 +1,25 @@
+import textwrap
+
+EXASOL_SQL_STATEMENT = textwrap.dedent(
+    """
+    SELECT
+      s.sql_text "query_text",
+      s.command_name "query_type",
+      se.user_name "user_name",
+      s.start_time "start_time",
+      s.stop_time "end_time",
+      s.duration "duration"
+    FROM EXA_DBA_AUDIT_SQL s
+    JOIN EXA_DBA_AUDIT_SESSIONS se
+    ON s.SESSION_ID = se.SESSION_ID
+    WHERE s.sql_text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' 
+    AND s.sql_text NOT LIKE '/* {{"app": "dbt", %%}} */%%'
+    AND start_time between TO_TIMESTAMP('{start_time}') and TO_TIMESTAMP('{end_time}')
+    {filters}
+    LIMIT {result_limit}
+    """
+)
+
 EXASOL_TEST_GET_QUERIES = """
     SELECT
       s.sql_text,

metadata/ingestion/source/database/exasol/query_parser.py

@@ -0,0 +1,33 @@
+from abc import ABC
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.exasolConnection import (
+    ExasolConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.query_parser_source import QueryParserSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class ExasolQueryParserSource(QueryParserSource, ABC):
+    """
+    Exasol base for Usage and Lineage
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: ExasolConnection = config.serviceConnection.root.config
+        if not isinstance(connection, ExasolConnection):
+            raise InvalidSourceException(
+                f"Expected ExasolConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/exasol/service_spec.py

@@ -1,4 +1,8 @@
+from metadata.ingestion.source.database.exasol.lineage import ExasolLineageSource
 from metadata.ingestion.source.database.exasol.metadata import ExasolSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(metadata_source_class=ExasolSource)
+ServiceSpec = DefaultDatabaseSpec(
+    metadata_source_class=ExasolSource,
+    lineage_source_class=ExasolLineageSource,
+)

metadata/ingestion/source/database/glue/metadata.py

@@ -406,7 +406,7 @@ class GlueSource(ExternalTableLineageMixin, DatabaseServiceSource):
                 for glue_col in glue_columns:
                     col_name = glue_col["Name"]
                     col_type = glue_col["Type"]
-                    col_comment = glue_col.get("Comment", "")
+                    col_comment = glue_col.get("Comment")
                     col_parameters = glue_col.get("Parameters", {})
 
                     # Check if this is a non-current Iceberg column
@@ -427,7 +427,7 @@ class GlueSource(ExternalTableLineageMixin, DatabaseServiceSource):
                 for glue_col in partition_keys:
                     col_name = glue_col["Name"]
                     col_type = glue_col["Type"]
-                    col_comment = glue_col.get("Comment", "")
+                    col_comment = glue_col.get("Comment")
                     col_parameters = glue_col.get("Parameters", {})
 
                     # Check if this is a non-current Iceberg column

metadata/ingestion/source/database/hive/connection.py

@@ -48,11 +48,20 @@ from metadata.ingestion.connections.test_connections import (
     test_connection_db_schema_sources,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.hive.custom_hive_connection import (
+    CustomHiveConnection,
+)
 from metadata.utils.constants import THREE_MIN
+from metadata.utils.ssl_manager import check_ssl_and_init
 
 HIVE_POSTGRES_SCHEME = "hive+postgres"
 HIVE_MYSQL_SCHEME = "hive+mysql"
 
+# Monkey-patch the pyhive.hive module to use our custom connection
+import pyhive.hive
+
+pyhive.hive.Connection = CustomHiveConnection
+
 
 def get_connection_url(connection: HiveConnection) -> str:
     """
@@ -113,6 +122,34 @@ def get_connection(connection: HiveConnection) -> Engine:
             "kerberos_service_name"
         ] = connection.kerberosServiceName
 
+    # Handle SSL using SSL manager (following established patterns)
+    ssl_manager = check_ssl_and_init(connection)
+    if ssl_manager:
+        connection = ssl_manager.setup_ssl(connection)
+        # Store SSL manager for cleanup
+        connection._ssl_manager = ssl_manager
+
+    # Add SSL configuration to connection arguments if SSL is enabled
+    if hasattr(connection, "useSSL") and connection.useSSL:
+        if not connection.connectionArguments:
+            connection.connectionArguments = init_empty_connection_arguments()
+        connection.connectionArguments.root["use_ssl"] = True
+
+        # Add SSL certificate configuration if available
+        if hasattr(connection, "sslConfig") and connection.sslConfig:
+            if connection.sslConfig.root.sslCertificate:
+                connection.connectionArguments.root[
+                    "ssl_certfile"
+                ] = connection.sslConfig.root.sslCertificate
+            if connection.sslConfig.root.sslKey:
+                connection.connectionArguments.root[
+                    "ssl_keyfile"
+                ] = connection.sslConfig.root.sslKey
+            if connection.sslConfig.root.caCertificate:
+                connection.connectionArguments.root[
+                    "ssl_ca_certs"
+                ] = connection.sslConfig.root.caCertificate
+
     return create_generic_db_connection(
         connection=connection,
         get_connection_url_fn=get_connection_url,
@@ -192,8 +229,10 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
-    if service_connection.metastoreConnection and isinstance(
-        service_connection.metastoreConnection, dict
+    if (
+        service_connection.metastoreConnection
+        and isinstance(service_connection.metastoreConnection, dict)
+        and len(service_connection.metastoreConnection) > 0
     ):
         try:
             service_connection.metastoreConnection = MysqlConnection.model_validate(