openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.9.2.0__py3-none-any.whl

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -436,21 +436,21 @@ class PowerbiSource(DashboardServiceSource):
         measures = []
         for measure in table.measures or []:
             try:
-                measure_type = (
-                    DataType.MEASURE_HIDDEN
-                    if measure.isHidden
-                    else DataType.MEASURE_VISIBLE
-                )
+                measure_type = DataType.MEASURE_VISIBLE
+                if measure.isHidden:
+                    measure_type = DataType.MEASURE_HIDDEN
+                expression_text = f"Expression : {measure.expression}"
                 description_text = (
-                    f"{measure.description}\n\nExpression : {measure.expression}"
+                    f"Description : {measure.description}"
                     if measure.description
-                    else f"Expression : {measure.expression}"
+                    else ""
                 )
+                description_field_text = f"{expression_text}\n\n{description_text}"
                 parsed_measure = PowerBiMeasureModel(
                     dataType=measure_type,
                     dataTypeDisplay=measure_type,
                     name=measure.name,
-                    description=description_text,
+                    description=description_field_text,
                 )
                 measures.append(Column(**parsed_measure.model_dump()))
             except Exception as err:
@@ -866,7 +866,7 @@ class PowerbiSource(DashboardServiceSource):
 
         try:
             table_info = self._parse_table_info_from_source_exp(table, datamodel_entity)
-            table_name = table.name or table_info.get("table")
+            table_name = table_info.get("table") or table.name
             schema_name = table_info.get("schema")
             database_name = table_info.get("database")
             if (

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -135,7 +135,7 @@ class PowerBiMeasures(BaseModel):
     name: str
     expression: str
     description: Optional[str] = None
-    isHidden: bool
+    isHidden: Optional[bool] = False
 
 
 class PowerBITableSource(BaseModel):

metadata/ingestion/source/database/dbt/metadata.py

@@ -924,7 +924,7 @@ class DbtSource(DbtServiceSource):
                 lineages = get_lineage_by_query(
                     self.metadata,
                     query=query,
-                    service_name=source_elements[0],
+                    service_names=source_elements[0],
                     database_name=source_elements[1],
                     schema_name=source_elements[2],
                     dialect=dialect,

metadata/ingestion/source/database/exasol/connection.py

@@ -19,6 +19,7 @@ from metadata.ingestion.connections.builders import (
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.exasol.queries import EXASOL_TEST_GET_QUERIES
 from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
@@ -89,10 +90,15 @@ def test_connection(
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
+
+    queries = {
+        "GetQueries": EXASOL_TEST_GET_QUERIES,
+    }
     return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        queries=queries,
         timeout_seconds=timeout_seconds,
     )

metadata/ingestion/source/database/exasol/lineage.py

@@ -0,0 +1,24 @@
+from metadata.ingestion.lineage.models import Dialect
+from metadata.ingestion.source.database.exasol.queries import EXASOL_SQL_STATEMENT
+from metadata.ingestion.source.database.exasol.query_parser import (
+    ExasolQueryParserSource,
+)
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class ExasolLineageSource(ExasolQueryParserSource, LineageSource):
+    """
+    Exasol class for Lineage
+    """
+
+    dialect = Dialect.EXASOL
+    sql_stmt = EXASOL_SQL_STATEMENT
+    filters = """
+        AND (
+            s.command_name IN ('MERGE', 'UPDATE', 'CREATE TABLE AS', 'CREATE VIEW')
+            OR (s.command_name = 'INSERT' AND LOWER(s.sql_text) LIKE '%insert%into%select%from%')
+    )
+    """

metadata/ingestion/source/database/exasol/metadata.py

@@ -1,5 +1,7 @@
 from typing import Optional, cast
 
+from sqlalchemy.engine.reflection import Inspector
+
 from metadata.generated.schema.entity.services.connections.database.exasolConnection import (
     ExasolConnection,
 )
@@ -9,6 +11,10 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
+from metadata.utils.sqlalchemy_utils import get_all_table_ddls, get_table_ddl
+
+Inspector.get_all_table_ddls = get_all_table_ddls
+Inspector.get_table_ddl = get_table_ddl
 
 
 class ExasolSource(CommonDbSourceService):

metadata/ingestion/source/database/exasol/queries.py

@@ -1,3 +1,25 @@
+import textwrap
+
+EXASOL_SQL_STATEMENT = textwrap.dedent(
+    """
+    SELECT
+      s.sql_text "query_text",
+      s.command_name "query_type",
+      se.user_name "user_name",
+      s.start_time "start_time",
+      s.stop_time "end_time",
+      s.duration "duration"
+    FROM EXA_DBA_AUDIT_SQL s
+    JOIN EXA_DBA_AUDIT_SESSIONS se
+    ON s.SESSION_ID = se.SESSION_ID
+    WHERE s.sql_text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' 
+    AND s.sql_text NOT LIKE '/* {{"app": "dbt", %%}} */%%'
+    AND start_time between TO_TIMESTAMP('{start_time}') and TO_TIMESTAMP('{end_time}')
+    {filters}
+    LIMIT {result_limit}
+    """
+)
+
 EXASOL_TEST_GET_QUERIES = """
     SELECT
       s.sql_text,

metadata/ingestion/source/database/exasol/query_parser.py

@@ -0,0 +1,33 @@
+from abc import ABC
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.exasolConnection import (
+    ExasolConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.query_parser_source import QueryParserSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class ExasolQueryParserSource(QueryParserSource, ABC):
+    """
+    Exasol base for Usage and Lineage
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: ExasolConnection = config.serviceConnection.root.config
+        if not isinstance(connection, ExasolConnection):
+            raise InvalidSourceException(
+                f"Expected ExasolConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/exasol/service_spec.py

@@ -1,4 +1,8 @@
+from metadata.ingestion.source.database.exasol.lineage import ExasolLineageSource
 from metadata.ingestion.source.database.exasol.metadata import ExasolSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(metadata_source_class=ExasolSource)
+ServiceSpec = DefaultDatabaseSpec(
+    metadata_source_class=ExasolSource,
+    lineage_source_class=ExasolLineageSource,
+)

metadata/ingestion/source/database/lineage_source.py

@@ -262,10 +262,18 @@ class LineageSource(QueryParserSource, ABC):
 
         for table_query in table_queries or []:
             if not self._query_already_processed(table_query):
+                # Prepare service names for lineage processing
+                service_names = [table_query.serviceName]
+                if (
+                    self.source_config.processCrossDatabaseLineage
+                    and self.source_config.crossDatabaseServiceNames
+                ):
+                    service_names.extend(self.source_config.crossDatabaseServiceNames)
+
                 lineages: Iterable[Either[AddLineageRequest]] = get_lineage_by_query(
                     self.metadata,
                     query=table_query.query,
-                    service_name=table_query.serviceName,
+                    service_names=service_names,
                     database_name=table_query.databaseName,
                     schema_name=table_query.databaseSchema,
                     dialect=self.dialect,
@@ -334,10 +342,18 @@ class LineageSource(QueryParserSource, ABC):
                         "View Filtered Out",
                     )
                     continue
+                # Prepare service names for view lineage processing
+                service_names = [self.config.serviceName]
+                if (
+                    self.source_config.processCrossDatabaseLineage
+                    and self.source_config.crossDatabaseServiceNames
+                ):
+                    service_names.extend(self.source_config.crossDatabaseServiceNames)
+
                 for lineage in get_view_lineage(
                     view=view,
                     metadata=self.metadata,
-                    service_name=self.config.serviceName,
+                    service_names=service_names,
                     connection_type=self.service_connection.type.value,
                     timeout_seconds=self.source_config.parsingTimeoutLimit,
                 ):

metadata/ingestion/source/database/saphana/cdata_parser.py

@@ -18,9 +18,10 @@ import xml.etree.ElementTree as ET
 from collections import defaultdict
 from enum import Enum
 from functools import lru_cache
-from typing import Dict, Iterable, List, NewType, Optional, Set
+from typing import Dict, Iterable, List, NewType, Optional, Set, Tuple
 
 from pydantic import Field, computed_field
+from sqlalchemy.engine import Engine
 from typing_extensions import Annotated
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
@@ -44,6 +45,7 @@ from metadata.ingestion.source.database.saphana.models import (
     SYS_BIC_SCHEMA_NAME,
     ViewType,
 )
+from metadata.ingestion.source.database.saphana.queries import SAPHANA_SCHEMA_MAPPING
 from metadata.utils import fqn
 from metadata.utils.constants import ENTITY_REFERENCE_TYPE_MAP
 from metadata.utils.dispatch import enum_register
@@ -158,6 +160,7 @@ class DataSource(BaseModel):
     def get_entity(
         self,
         metadata: OpenMetadata,
+        engine: Engine,
         service_name: str,
     ) -> Table:
         """Build the Entity Reference for this DataSource"""
@@ -168,13 +171,14 @@ class DataSource(BaseModel):
             )
 
         if self.source_type == ViewType.DATA_BASE_TABLE:
+            schema_name = _get_mapped_schema(engine=engine, schema_name=self.location)
             # The source is a table, so the location is the schema
             fqn_ = fqn.build(
                 metadata=metadata,
                 entity_type=Table,
                 service_name=service_name,
                 database_name=None,  # TODO: Can we assume HXE?
-                schema_name=self.location,
+                schema_name=schema_name,
                 table_name=self.name,
             )
         else:
@@ -243,17 +247,67 @@ class ParsedLineage(BaseModel):
         return id(self)
 
     def to_request(
-        self, metadata: OpenMetadata, service_name: str, to_entity: Table
+        self,
+        metadata: OpenMetadata,
+        engine: Engine,
+        service_name: str,
+        to_entity: Table,
     ) -> Iterable[Either[AddLineageRequest]]:
         """Given the target entity, build the AddLineageRequest based on the sources in `self`"""
         for source in self.sources:
             try:
                 source_table = source.get_entity(
-                    metadata=metadata, service_name=service_name
+                    metadata=metadata, engine=engine, service_name=service_name
                 )
                 if not source_table:
                     logger.warning(f"Can't find table for source [{source}]")
                     continue
+
+                column_lineage = []
+                for mapping in self.mappings:
+                    if mapping.data_source != source:
+                        continue
+
+                    from_columns = []
+                    for source_col in mapping.sources:
+                        from_column_fqn = get_column_fqn(
+                            table_entity=source_table,
+                            column=source_col,
+                        )
+                        if not from_column_fqn:
+                            logger.warning(
+                                f"Can't find source column [{source_col}] in [{source_table}]"
+                            )
+                            continue
+
+                        from_columns.append(
+                            FullyQualifiedEntityName(
+                                from_column_fqn,
+                            )
+                        )
+
+                    to_column_fqn = get_column_fqn(
+                        table_entity=to_entity,
+                        column=mapping.target,
+                    )
+                    if not to_column_fqn:
+                        logger.warning(
+                            f"Can't find target column [{mapping.target}] in [{to_entity}]."
+                            f" For source columns: {from_columns}"
+                        )
+                        continue
+
+                    to_column = FullyQualifiedEntityName(
+                        to_column_fqn,
+                    )
+                    column_lineage.append(
+                        ColumnLineage(
+                            fromColumns=from_columns,
+                            toColumn=to_column,
+                            function=mapping.formula,
+                        )
+                    )
+
                 yield Either(
                     right=AddLineageRequest(
                         edge=EntitiesEdge(
@@ -267,30 +321,7 @@ class ParsedLineage(BaseModel):
                             ),
                             lineageDetails=LineageDetails(
                                 source=Source.ViewLineage,
-                                columnsLineage=[
-                                    ColumnLineage(
-                                        fromColumns=[
-                                            FullyQualifiedEntityName(
-                                                get_column_fqn(
-                                                    table_entity=source_table,
-                                                    column=source_col,
-                                                )
-                                            )
-                                            for source_col in mapping.sources
-                                        ],
-                                        toColumn=FullyQualifiedEntityName(
-                                            get_column_fqn(
-                                                table_entity=to_entity,
-                                                column=mapping.target,
-                                            )
-                                        ),
-                                        function=mapping.formula
-                                        if mapping.formula
-                                        else None,
-                                    )
-                                    for mapping in self.mappings
-                                    if mapping.data_source == source
-                                ],
+                                columnsLineage=column_lineage,
                             ),
                         )
                     )
@@ -339,6 +370,39 @@ def _get_column_datasources(
     }
 
 
+def _get_column_datasources_with_names(
+    entry: ET.Element, datasource_map: Optional[DataSourceMap] = None
+) -> List[Tuple[DataSource, str]]:
+    """
+    Get the DataSource and the actual source column name after traversal.
+    Returns a list of tuples (DataSource, column_name).
+    """
+    if (
+        datasource_map
+        and entry.get(CDATAKeys.COLUMN_OBJECT_NAME.value) in datasource_map
+    ):
+        # Traverse to get the actual sources and column names
+        ds_col_pairs = _traverse_ds_with_columns(
+            current_column=entry.get(CDATAKeys.COLUMN_NAME.value),
+            ds_origin_list=[],
+            current_ds=datasource_map[entry.get(CDATAKeys.COLUMN_OBJECT_NAME.value)],
+            datasource_map=datasource_map,
+        )
+        return ds_col_pairs
+
+    # If we don't have any logical sources, use the column name as-is
+    return [
+        (
+            DataSource(
+                name=entry.get(CDATAKeys.COLUMN_OBJECT_NAME.value),
+                location=entry.get(CDATAKeys.SCHEMA_NAME.value),
+                source_type=ViewType.DATA_BASE_TABLE,
+            ),
+            entry.get(CDATAKeys.COLUMN_NAME.value),
+        )
+    ]
+
+
 def _traverse_ds(
     current_column: str,
     ds_origin_list: List[DataSource],
@@ -355,7 +419,9 @@ def _traverse_ds(
 
     else:
         # Based on our current column, find the parents from the mappings in the current_ds
-        current_ds_mapping: DataSourceMapping = current_ds.mapping.get(current_column)
+        current_ds_mapping: Optional[DataSourceMapping] = current_ds.mapping.get(
+            current_column
+        )
 
         if current_ds_mapping:
             for parent in current_ds_mapping.parents:
@@ -381,6 +447,54 @@ def _traverse_ds(
     return ds_origin_list
 
 
+def _traverse_ds_with_columns(
+    current_column: str,
+    ds_origin_list: List[Tuple[DataSource, str]],
+    current_ds: DataSource,
+    datasource_map: Optional[DataSourceMap],
+) -> List[Tuple[DataSource, str]]:
+    """
+    Traverse the ds dict jumping from target -> source columns and getting the right parent.
+    We keep inspecting current datasources and will append to the origin list the ones
+    that are not LOGICAL, along with the final column name.
+    Returns a list of tuples (DataSource, column_name).
+    """
+    if current_ds.source_type != ViewType.LOGICAL:
+        # This is a final datasource, append it with the current column name
+        ds_origin_list.append((current_ds, current_column))
+
+    else:
+        # Based on our current column, find the parents from the mappings in the current_ds
+        current_ds_mapping: Optional[DataSourceMapping] = current_ds.mapping.get(
+            current_column
+        )
+
+        if current_ds_mapping:
+            for parent in current_ds_mapping.parents:
+                parent_ds = datasource_map.get(parent.parent)
+                if not parent_ds:
+                    raise CDATAParsingError(
+                        f"Can't find parent [{parent.parent}] for column [{current_column}]"
+                    )
+
+                # Traverse from the source column in the parent mapping
+                # Note: parent.source is the column name in the parent datasource
+                _traverse_ds_with_columns(
+                    current_column=parent.source,
+                    ds_origin_list=ds_origin_list,
+                    current_ds=parent_ds,
+                    datasource_map=datasource_map,
+                )
+        else:
+            # Current column not in mapping. This can happen for calculated view attributes
+            logger.info(
+                f"Can't find mapping for column [{current_column}] in [{current_ds}]. "
+                f"We still have to implement `calculatedViewAttributes`."
+            )
+
+    return ds_origin_list
+
+
 def _read_attributes(
     tree: ET.Element, ns: dict, datasource_map: Optional[DataSourceMap] = None
 ) -> ParsedLineage:
@@ -392,17 +506,20 @@ def _read_attributes(
 
     for attribute in attribute_list.findall(CDATAKeys.ATTRIBUTE.value, ns):
         key_mapping = attribute.find(CDATAKeys.KEY_MAPPING.value, ns)
-        data_sources = _get_column_datasources(
+
+        # Get the actual source datasources and their column names
+        data_sources_with_columns = _get_column_datasources_with_names(
             entry=key_mapping, datasource_map=datasource_map
         )
+
         attr_lineage = ParsedLineage(
             mappings=[
                 ColumnMapping(
-                    data_source=ds,
-                    sources=[key_mapping.get(CDATAKeys.COLUMN_NAME.value)],
+                    data_source=ds_info[0],  # The datasource
+                    sources=[ds_info[1]],  # The actual source column name
                     target=attribute.get(CDATAKeys.ID.value),
                 )
-                for ds in data_sources
+                for ds_info in data_sources_with_columns
             ]
         )
         lineage += attr_lineage
@@ -456,17 +573,20 @@ def _read_base_measures(
 
     for measure in base_measures.findall(CDATAKeys.MEASURE.value, ns):
         measure_mapping = measure.find(CDATAKeys.MEASURE_MAPPING.value, ns)
-        data_sources = _get_column_datasources(
+
+        # Get the actual source datasources and their column names
+        data_sources_with_columns = _get_column_datasources_with_names(
             entry=measure_mapping, datasource_map=datasource_map
         )
+
         measure_lineage = ParsedLineage(
             mappings=[
                 ColumnMapping(
-                    data_source=ds,
-                    sources=[measure_mapping.get(CDATAKeys.COLUMN_NAME.value)],
+                    data_source=ds_info[0],  # The datasource
+                    sources=[ds_info[1]],  # The actual source column name
                     target=measure.get(CDATAKeys.ID.value),
                 )
-                for ds in data_sources
+                for ds_info in data_sources_with_columns
             ]
         )
         lineage += measure_lineage
@@ -519,7 +639,12 @@ def _(cdata: str) -> ParsedLineage:
     tree = ET.fromstring(cdata)
     measure_group = tree.find(CDATAKeys.PRIVATE_MEASURE_GROUP.value, ns)
     # TODO: Handle lineage from calculatedMeasures, restrictedMeasures and sharedDimensions
-    return _read_attributes(measure_group, ns)
+    attribute_lineage = _read_attributes(measure_group, ns)
+    base_measure_lineage = _read_base_measures(
+        tree=measure_group, ns=ns, datasource_map=None
+    )
+
+    return attribute_lineage + base_measure_lineage
 
 
 @parse_registry.add(ViewType.ATTRIBUTE_VIEW.value)
@@ -643,10 +768,17 @@ def _parse_cv_data_sources(tree: ET.Element, ns: dict) -> DataSourceMap:
 
     for cv in calculation_views.findall(CDATAKeys.CALCULATION_VIEW.value, ns):
         mappings = _build_mappings(calculation_view=cv, ns=ns)
+        # Build mapping dict, keeping only the first occurrence of each target
+        # (subsequent ones are typically for join conditions)
+        mapping_dict = {}
+        for mapping in mappings:
+            if mapping.target not in mapping_dict:
+                mapping_dict[mapping.target] = mapping
+
         datasource_map[cv.get(CDATAKeys.ID.value)] = DataSource(
             name=cv.get(CDATAKeys.ID.value),
             location=None,
-            mapping={mapping.target: mapping for mapping in mappings},
+            mapping=mapping_dict,
             source_type=ViewType.LOGICAL,
         )
 
@@ -690,28 +822,48 @@ def _build_mappings(calculation_view: ET.Element, ns: dict) -> List[DataSourceMa
 def _build_input_mappings(
     calculation_view: ET.Element, ns: dict
 ) -> List[DataSourceMapping]:
-    """Map input nodes"""
+    """
+    Map input nodes preserving the exact target-to-source relationships.
+
+    IMPORTANT: Each target column should map to exactly one source.
+    When there are multiple inputs with the same source column name,
+    they map to different target columns (e.g., PRICE vs PRICE_1).
+    """
     mappings = []
     for input_node in calculation_view.findall(CDATAKeys.INPUT.value, ns):
+        input_node_name = input_node.get(CDATAKeys.NODE.value).replace("#", "")
+
         for mapping in input_node.findall(CDATAKeys.MAPPING.value, ns):
-            if mapping.get(CDATAKeys.SOURCE.value) and mapping.get(
-                CDATAKeys.TARGET.value
-            ):
+            source_col = mapping.get(CDATAKeys.SOURCE.value)
+            target_col = mapping.get(CDATAKeys.TARGET.value)
+
+            if source_col and target_col:
+                # Each target column gets its own mapping entry
+                # We don't group here because each target maps to a specific source
                 mappings.append(
                     DataSourceMapping(
-                        target=mapping.get(CDATAKeys.TARGET.value),
+                        target=target_col,
                         parents=[
                             ParentSource(
-                                source=mapping.get(CDATAKeys.SOURCE.value),
-                                parent=input_node.get(CDATAKeys.NODE.value).replace(
-                                    "#", ""
-                                ),
+                                source=source_col,
+                                parent=input_node_name,
                             )
                         ],
                     )
                 )
 
-    return _group_mappings(mappings)
+    # For Union views, we need to group because multiple inputs can map to the same target
+    # For Join views, we should NOT group because each target has a unique source
+    calculation_view_type = calculation_view.get(
+        "{http://www.w3.org/2001/XMLSchema-instance}type"
+    )
+
+    if calculation_view_type and "UnionView" in calculation_view_type:
+        return _group_mappings(mappings)
+    else:
+        # For Join, Projection, Aggregation views - each target has exactly one source
+        # We still return the list but don't group
+        return mappings
 
 
 def _build_cv_attributes(
@@ -770,3 +922,22 @@ def _group_mappings(mappings: List[DataSourceMapping]) -> List[DataSourceMapping
     ]
 
     return grouped_data
+
+
+@lru_cache(maxsize=256)
+def _get_mapped_schema(
+    engine: Engine,
+    schema_name: str,
+) -> str:
+    """
+    Get the physical schema for a given authoring schema
+    If schema is not mapped, then consider it as the physical schema
+    """
+    with engine.connect() as conn:
+        result = conn.execute(
+            SAPHANA_SCHEMA_MAPPING.format(authoring_schema=schema_name)
+        )
+        row = result.fetchone()
+        if row is not None:
+            return row[0]
+    return schema_name