openmetadata-ingestion 1.6.8.0__py3-none-any.whl → 1.7.0.0rc2__py3-none-any.whl

metadata/ingestion/source/dashboard/superset/db_source.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -25,7 +25,6 @@ from metadata.generated.schema.api.data.createDashboardDataModel import (
 )
 from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.data.dashboardDataModel import DataModelType
-from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
     MysqlConnection,
 )
@@ -58,6 +57,7 @@ from metadata.ingestion.source.dashboard.superset.queries import (
 )
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_datamodel
+from metadata.utils.fqn import build_es_fqn_search_string
 from metadata.utils.helpers import (
     clean_uri,
     get_database_name_for_lineage,
@@ -95,17 +95,15 @@ class SupersetDBSource(SupersetSourceMixin):
             logger.debug(traceback.format_exc())
             logger.warning(f"Failed to fetch chart list due to - {err}]")
 
-    def get_column_list(self, table_name: str) -> Iterable[FetchChart]:
+    def get_column_list(self, table_id: Optional[int]) -> Iterable[FetchChart]:
         try:
-            if table_name:
-                col_list = self.engine.execute(
-                    FETCH_COLUMN, table_name=table_name.lower()
-                )
+            if table_id:
+                col_list = self.engine.execute(FETCH_COLUMN, table_id=table_id)
                 return [FetchColumn(**col) for col in col_list]
         except Exception as err:
             logger.debug(traceback.format_exc())
             logger.warning(
-                f"Failed to fetch column name list for table: [{table_name} due to - {err}]"
+                f"Failed to fetch column name list for table: [{table_id} due to - {err}]"
             )
         return []
 
@@ -162,10 +160,10 @@ class SupersetDBSource(SupersetSourceMixin):
             )
 
     def _get_datasource_fqn_for_lineage(
-        self, chart_json: FetchChart, db_service_entity: DatabaseService
+        self, chart_json: FetchChart, db_service_name: Optional[str]
     ):
         return (
-            self._get_datasource_fqn(db_service_entity, chart_json)
+            self._get_datasource_fqn(db_service_name, chart_json)
             if chart_json.table_name
             else None
         )
@@ -214,23 +212,27 @@ class SupersetDBSource(SupersetSourceMixin):
         if sqa_str:
             sqa_url = make_url(sqa_str)
             default_db_name = sqa_url.database if sqa_url else None
+
         return get_database_name_for_lineage(db_service_entity, default_db_name)
 
     def _get_datasource_fqn(
-        self, db_service_entity: DatabaseService, chart_json: FetchChart
+        self, db_service_name: Optional[str], chart_json: FetchChart
     ) -> Optional[str]:
         try:
-            dataset_fqn = fqn.build(
-                self.metadata,
-                entity_type=Table,
-                table_name=chart_json.table_name,
-                database_name=self._get_database_name(
+            database_name = None
+            if db_service_name:
+                db_service_entity = self.metadata.get_by_name(
+                    entity=DatabaseService, fqn=db_service_name
+                )
+                database_name = self._get_database_name(
                     chart_json.sqlalchemy_uri, db_service_entity
-                ),
+                )
+            return build_es_fqn_search_string(
+                database_name=database_name,
                 schema_name=chart_json.table_schema,
-                service_name=db_service_entity.name.root,
+                service_name=db_service_name or "*",
+                table_name=chart_json.table_name,
             )
-            return dataset_fqn
         except Exception as err:
             logger.debug(traceback.format_exc())
             logger.warning(
@@ -255,7 +257,7 @@ class SupersetDBSource(SupersetSourceMixin):
                     self.status.filter(
                         chart_json.table_name, "Data model filtered out."
                     )
-                col_names = self.get_column_list(chart_json.table_name)
+                col_names = self.get_column_list(chart_json.table_id)
                 try:
                     data_model_request = CreateDashboardDataModelRequest(
                         name=EntityName(str(chart_json.datasource_id)),
@@ -285,5 +287,5 @@ class SupersetDBSource(SupersetSourceMixin):
         Returns:
             List of columns as str to generate column lineage
         """
-        col_list = self.get_column_list(chart_json.table_name)
+        col_list = self.get_column_list(chart_json.table_id)
         return [col.column_name for col in col_list]

metadata/ingestion/source/dashboard/superset/metadata.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

metadata/ingestion/source/dashboard/superset/mixin.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -13,7 +13,10 @@ Superset mixin module
 """
 import json
 import traceback
-from typing import Iterable, List, Optional, Union
+from typing import Dict, Iterable, List, Optional, Tuple, Union
+
+from collate_sqllineage.core.models import Column as LineageColumn
+from collate_sqllineage.core.models import Table as LineageTable
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel
@@ -27,16 +30,18 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
 from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
-from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.entityLineage import ColumnLineage
 from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.lineage.parser import LineageParser
+from metadata.ingestion.lineage.sql_lineage import get_column_fqn
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
 from metadata.ingestion.source.dashboard.superset.models import (
@@ -47,6 +52,9 @@ from metadata.ingestion.source.dashboard.superset.models import (
     FetchDashboard,
     SupersetDatasource,
 )
+from metadata.ingestion.source.dashboard.superset.utils import (
+    get_dashboard_data_model_column_fqn,
+)
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.utils import fqn
 from metadata.utils.logger import ingestion_logger
@@ -146,63 +154,199 @@ class SupersetSourceMixin(DashboardServiceSource):
             )
         return []
 
+    def _is_table_to_table_lineage(self, columns: tuple, table: LineageTable) -> bool:
+        from_column: LineageColumn = columns[0]
+        to_column: LineageColumn = columns[-1]
+
+        if not isinstance(from_column.parent, LineageTable):
+            return False
+
+        if not isinstance(to_column.parent, LineageTable):
+            return False
+
+        if from_column.parent.schema.raw_name != table.schema.raw_name:
+            return False
+
+        if from_column.parent.raw_name != table.raw_name:
+            return False
+
+        return True
+
+    def _append_value_to_dict_list(
+        self, input_dict: Dict[str, List[str]], dict_key: str, list_value: str
+    ) -> None:
+        if input_dict.get(dict_key):
+            input_dict[dict_key].append(list_value)
+        else:
+            input_dict[dict_key] = [list_value]
+
+    def _get_table_schema(self, table: LineageTable, chart: FetchChart) -> str:
+        if table.schema.raw_name == table.schema.unknown:
+            return chart.table_schema
+
+        return table.schema.raw_name
+
+    def _create_column_lineage_mapping(
+        self, parser: LineageParser, table: LineageTable, chart: FetchChart
+    ) -> Dict[str, List[str]]:
+        result = {}
+        table_to_table_lineage = [
+            _columns
+            for _columns in parser.column_lineage
+            if self._is_table_to_table_lineage(_columns, table)
+        ]
+
+        for columns in table_to_table_lineage:
+            from_column_name = columns[0].raw_name
+            to_column_name = columns[-1].raw_name
+
+            if from_column_name != "*" and to_column_name != "*":
+                self._append_value_to_dict_list(
+                    result, to_column_name, from_column_name
+                )
+
+            if from_column_name == "*" and to_column_name == "*":
+                for col_name in self._get_columns_list_for_lineage(chart):
+                    self._append_value_to_dict_list(result, col_name, col_name)
+
+        return result
+
+    def _parse_lineage_from_dataset_sql(
+        self, chart_json: FetchChart
+    ) -> List[Tuple[FetchChart, Dict[str, List[str]]]]:
+        # Every SQL query in tables is a SQL statement SELECTING data.
+        # To get lineage we 'simulate' INSERT INTO query into dummy table.
+        result = []
+        parser = LineageParser(f"INSERT INTO dummy_table {chart_json.sql}")
+
+        for table in parser.source_tables:
+            table_name = table.raw_name
+            table_schema = self._get_table_schema(table, chart_json)
+
+            column_mapping: Dict[str, List[str]] = self._create_column_lineage_mapping(
+                parser, table, chart_json
+            )
+
+            result.append(
+                (
+                    FetchChart(
+                        table_name=table_name,
+                        schema=table_schema,
+                        sqlalchemy_uri=chart_json.sqlalchemy_uri,
+                    ),
+                    column_mapping,
+                )
+            )
+
+        return result
+
+    def _enrich_raw_input_tables(
+        self,
+        from_entities: List[Tuple[FetchChart, Dict[str, List[str]]]],
+        to_entity: DashboardDataModel,
+        db_service_name: Optional[str],
+    ):
+        result = []
+
+        for from_entity in from_entities:
+            input_table, _column_lineage = from_entity
+            datasource_fqn = self._get_datasource_fqn_for_lineage(
+                input_table, db_service_name
+            )
+            from_entity = self.metadata.search_in_any_service(
+                entity_type=Table,
+                fqn_search_string=datasource_fqn,
+            )
+            if not from_entity:
+                continue
+            column_lineage: List[ColumnLineage] = []
+            for to_column, from_columns in _column_lineage.items():
+                _from_columns = [
+                    get_column_fqn(from_entity, from_column)
+                    for from_column in from_columns
+                    if get_column_fqn(from_entity, from_column)
+                ]
+
+                _to_column = get_dashboard_data_model_column_fqn(to_entity, to_column)
+
+                if _from_columns and _to_column:
+                    column_lineage.append(
+                        ColumnLineage(
+                            fromColumns=_from_columns,
+                            toColumn=_to_column,
+                        )
+                    )
+
+            result.append((from_entity, column_lineage))
+
+        return result
+
+    def _get_input_tables(self, chart: FetchChart):
+        if getattr(chart, "sql", None):
+            result = self._parse_lineage_from_dataset_sql(chart)
+        else:
+            result = [
+                (chart, {c: [c] for c in self._get_columns_list_for_lineage(chart)})
+            ]
+
+        return result
+
+    def _get_dashboard_data_model_entity(
+        self, chart: FetchChart
+    ) -> Optional[DashboardDataModel]:
+        datamodel_fqn = fqn.build(
+            self.metadata,
+            entity_type=DashboardDataModel,
+            service_name=self.config.serviceName,
+            data_model_name=str(chart.datasource_id),
+        )
+        return self.metadata.get_by_name(
+            entity=DashboardDataModel,
+            fqn=datamodel_fqn,
+        )
+
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: Union[FetchDashboard, DashboardResult],
-        db_service_name: DatabaseService,
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between datamodel and table
         """
-        db_service_entity = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=db_service_name
-        )
-        if db_service_entity:
-            for chart_id in self._get_charts_of_dashboard(dashboard_details):
-                chart_json = self.all_charts.get(chart_id)
-                if chart_json:
-                    try:
-                        datasource_fqn = self._get_datasource_fqn_for_lineage(
-                            chart_json, db_service_entity
-                        )
-                        if not datasource_fqn:
-                            continue
-                        from_entity = self.metadata.get_by_name(
-                            entity=Table,
-                            fqn=datasource_fqn,
-                        )
-                        datamodel_fqn = fqn.build(
-                            self.metadata,
-                            entity_type=DashboardDataModel,
-                            service_name=self.config.serviceName,
-                            data_model_name=str(chart_json.datasource_id),
-                        )
-                        to_entity = self.metadata.get_by_name(
-                            entity=DashboardDataModel,
-                            fqn=datamodel_fqn,
-                        )
+        for chart_json in filter(
+            None,
+            [
+                self.all_charts.get(chart_id)
+                for chart_id in self._get_charts_of_dashboard(dashboard_details)
+            ],
+        ):
+            try:
+                to_entity = self._get_dashboard_data_model_entity(chart_json)
 
-                        columns_list = self._get_columns_list_for_lineage(chart_json)
-                        column_lineage = self._get_column_lineage(
-                            from_entity, to_entity, columns_list
-                        )
-                        if from_entity and to_entity:
-                            yield self._get_add_lineage_request(
-                                to_entity=to_entity,
-                                from_entity=from_entity,
-                                column_lineage=column_lineage,
-                            )
-                    except Exception as exc:
-                        yield Either(
-                            left=StackTraceError(
-                                name=db_service_name,
-                                error=(
-                                    "Error to yield dashboard lineage details for DB "
-                                    f"service name [{db_service_name}]: {exc}"
-                                ),
-                                stackTrace=traceback.format_exc(),
-                            )
+                if to_entity:
+                    _input_tables = self._get_input_tables(chart_json)
+                    input_tables = self._enrich_raw_input_tables(
+                        _input_tables, to_entity, db_service_name
+                    )
+                    for input_table in input_tables:
+                        from_entity_table, column_lineage = input_table
+
+                        yield self._get_add_lineage_request(
+                            to_entity=to_entity,
+                            from_entity=from_entity_table,
+                            column_lineage=column_lineage,
                         )
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name="Dashboard Lineage Details",
+                        error=(
+                            "Error to yield dashboard lineage details for DB "
+                            f"service name [{db_service_name}]: {exc}"
+                        ),
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
 
     def _get_datamodel(
         self, datamodel: Union[SupersetDatasource, FetchChart]
@@ -238,6 +382,18 @@ class SupersetSourceMixin(DashboardServiceSource):
             return DataType(col_parse["arrayDataType"])
         return None
 
+    def parse_row_data_type(self, col_parse: dict) -> List[Column]:
+        """
+        Set children to single UNKNOWN column for Trino row columns
+        to prevent validation error requiring non empty list of children.
+        """
+        if col_parse["dataType"] == "ROW" and not col_parse.get("children"):
+            return [Column(name="unknown", dataType=DataType.UNKNOWN)]
+
+        if col_parse.get("children"):
+            return col_parse["children"]
+        return []
+
     def get_column_info(
         self, data_source: List[Union[DataSourceResult, FetchColumn]]
     ) -> Optional[List[Column]]:
@@ -259,9 +415,7 @@ class SupersetSourceMixin(DashboardServiceSource):
                         dataTypeDisplay=field.type,
                         dataType=col_parse["dataType"],
                         arrayDataType=self.parse_array_data_type(col_parse),
-                        children=list(col_parse["children"])
-                        if col_parse.get("children")
-                        else None,
+                        children=self.parse_row_data_type(col_parse),
                         name=str(field.id),
                         displayName=field.column_name,
                         description=field.description,

metadata/ingestion/source/dashboard/superset/models.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -144,17 +144,20 @@ class FetchChart(BaseModel):
     id: Optional[int] = None
     slice_name: Optional[str] = None
     description: Optional[str] = None
+    table_id: Optional[int] = None
     table_name: Optional[str] = None
     table_schema: Optional[str] = Field(None, alias="schema")
     database_name: Optional[str] = None
     sqlalchemy_uri: Optional[str] = None
     viz_type: Optional[str] = None
     datasource_id: Optional[int] = None
+    sql: Optional[str] = None
 
 
 class FetchColumn(BaseModel):
     id: Optional[int] = None
     type: Optional[str] = None
     column_name: Optional[str] = None
+    table_id: Optional[int] = None
     table_name: Optional[str] = None
     description: Optional[str] = None

metadata/ingestion/source/dashboard/superset/queries.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,8 +20,10 @@ select
 	s.description,
     	s.datasource_id,
     	s.viz_type,
+	t.id AS table_id,
 	t.table_name,
 	t.schema,
+	t.sql,
 	db.database_name,
     db.sqlalchemy_uri
 from 
@@ -91,9 +93,10 @@ LIMIT 1
 FETCH_COLUMN = """
 select 
 	tc.id, 
-    	table_name ,
-    	column_name, 
-    	type,
+    	tc.table_name ,
+    	tc.column_name, 
+		tc.table_id, 
+    	tc.type,
     	tc.description 
 from 
 	table_columns  tc  
@@ -102,5 +105,5 @@ inner join
 on 
 	t.id=tc.table_id  
 where 
-	table_name=%(table_name)s
+	tc.table_id=%(table_id)s
 """

metadata/ingestion/source/dashboard/superset/utils.py

@@ -0,0 +1,36 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Superset utils module
+"""
+
+from typing import Optional
+
+from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel
+
+
+def get_dashboard_data_model_column_fqn(
+    dashboard_data_model_entity: DashboardDataModel, column: str
+) -> Optional[str]:
+    """
+    Get fqn of column if exist in dashboard data model entity.
+
+    This is Superset implementation specific as table name is stored within displayName (table name contains
+    numerical id), which is not consistent with implementations of dashboard data model columns of
+    other dashboard sources.
+    """
+    if not dashboard_data_model_entity:
+        return None
+    for dashboard_data_model_column in dashboard_data_model_entity.columns:
+        if column.lower() == dashboard_data_model_column.displayName.lower():
+            return dashboard_data_model_column.fullyQualifiedName.root
+
+    return None

metadata/ingestion/source/dashboard/tableau/__init__.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.