openmetadata-ingestion 1.8.0.0__py3-none-any.whl → 1.8.2.0__py3-none-any.whl

metadata/ingestion/source/dashboard/quicksight/metadata.py

@@ -170,9 +170,11 @@ class QuicksightSource(DashboardServiceSource):
             name=EntityName(dashboard_details.DashboardId),
             sourceUrl=SourceUrl(self.dashboard_url),
             displayName=dashboard_details.Name,
-            description=Markdown(dashboard_details.Version.Description)
-            if dashboard_details.Version and dashboard_details.Version.Description
-            else None,
+            description=(
+                Markdown(dashboard_details.Version.Description)
+                if dashboard_details.Version and dashboard_details.Version.Description
+                else None
+            ),
             charts=[
                 FullyQualifiedEntityName(
                     fqn.build(
@@ -254,11 +256,17 @@ class QuicksightSource(DashboardServiceSource):
         data_model_entity,
         data_source_resp: DataSourceModel,
         dashboard_details: DashboardDetail,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
     ) -> Iterable[Either[AddLineageRequest]]:
         """yield lineage from table(parsed form query source) <-> dashboard"""
         db_service_entity = None
-        if db_service_name:
+        (
+            db_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+        if db_service_prefix:
             db_service_entity = self.metadata.get_by_name(
                 entity=DatabaseService, fqn=db_service_name
             )
@@ -276,26 +284,58 @@ class QuicksightSource(DashboardServiceSource):
         try:
             lineage_parser = LineageParser(
                 sql_query,
-                ConnectionTypeDialectMapper.dialect_of(
-                    db_service_entity.serviceType.value
-                )
-                if db_service_entity
-                else Dialect.ANSI,
+                (
+                    ConnectionTypeDialectMapper.dialect_of(
+                        db_service_entity.serviceType.value
+                    )
+                    if db_service_entity
+                    else Dialect.ANSI
+                ),
             )
             lineage_details = LineageDetails(
                 source=LineageSource.DashboardLineage, sqlQuery=sql_query
             )
             for db_name in source_database_names:
+                if (
+                    prefix_database_name
+                    and db_name
+                    and prefix_database_name.lower() != str(db_name).lower()
+                ):
+                    logger.debug(
+                        f"Database {db_name} does not match prefix {prefix_database_name}"
+                    )
+                    continue
                 for table in lineage_parser.source_tables:
                     database_schema_name, table = fqn.split(str(table))[-2:]
                     database_schema_name = self.check_database_schema_name(
                         database_schema_name
                     )
+
+                    if (
+                        prefix_schema_name
+                        and database_schema_name
+                        and prefix_schema_name.lower() != database_schema_name.lower()
+                    ):
+                        logger.debug(
+                            f"Schema {database_schema_name} does not match prefix {prefix_schema_name}"
+                        )
+                        continue
+
+                    if (
+                        prefix_table_name
+                        and table
+                        and prefix_table_name.lower() != table.lower()
+                    ):
+                        logger.debug(
+                            f"Table {table} does not match prefix {prefix_table_name}"
+                        )
+                        continue
+
                     fqn_search_string = build_es_fqn_search_string(
-                        database_name=db_name,
-                        schema_name=database_schema_name,
+                        database_name=prefix_database_name or db_name,
+                        schema_name=prefix_schema_name or database_schema_name,
                         service_name=db_service_name or "*",
-                        table_name=table,
+                        table_name=prefix_table_name or table,
                     )
                     from_entities = self.metadata.search_in_any_service(
                         entity_type=Table,
@@ -391,20 +431,58 @@ class QuicksightSource(DashboardServiceSource):
         data_model_entity,
         data_source_resp: DataSourceModel,
         dashboard_details: DashboardDetail,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
     ) -> Iterable[Either[AddLineageRequest]]:
         """yield lineage from table <-> dashboard"""
         try:
+            (
+                db_service_name,
+                prefix_database_name,
+                prefix_schema_name,
+                prefix_table_name,
+            ) = self.parse_db_service_prefix(db_service_prefix)
             schema_name = data_source_resp.data_source_resp.schema_name
             table_name = data_source_resp.data_source_resp.table_name
+
+            if (
+                prefix_schema_name
+                and schema_name
+                and prefix_schema_name.lower() != schema_name.lower()
+            ):
+                logger.debug(
+                    f"Schema {schema_name} does not match prefix {prefix_schema_name}"
+                )
+                return
+
+            if (
+                prefix_table_name
+                and table_name
+                and prefix_table_name.lower() != table_name.lower()
+            ):
+                logger.debug(
+                    f"Table {table_name} does not match prefix {prefix_table_name}"
+                )
+                return
+
             if data_source_resp and data_source_resp.DataSourceParameters:
                 data_source_dict = data_source_resp.DataSourceParameters
                 for db in data_source_dict.keys() or []:
+                    database_name = data_source_dict[db].get("Database")
+                    if (
+                        prefix_database_name
+                        and database_name
+                        and prefix_database_name.lower() != database_name.lower()
+                    ):
+                        logger.debug(
+                            f"Database {database_name} does not match prefix {prefix_database_name}"
+                        )
+                        continue
+
                     fqn_search_string = build_es_fqn_search_string(
-                        database_name=data_source_dict[db].get("Database"),
-                        schema_name=schema_name,
+                        database_name=prefix_database_name or database_name,
+                        schema_name=prefix_schema_name or schema_name,
                         service_name=db_service_name or "*",
-                        table_name=table_name,
+                        table_name=prefix_table_name or table_name,
                     )
                     from_entity = self.metadata.search_in_any_service(
                         entity_type=Table,
@@ -446,11 +524,12 @@ class QuicksightSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(  # pylint: disable=too-many-locals
         self,
         dashboard_details: DashboardDetail,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
         """
+        db_service_name, *_ = self.parse_db_service_prefix(db_service_prefix)
         for datamodel in self.data_models or []:
             try:
                 data_model_entity = self._get_datamodel(
@@ -463,7 +542,7 @@ class QuicksightSource(DashboardServiceSource):
                         data_model_entity,
                         datamodel.DataSource,
                         dashboard_details,
-                        db_service_name,
+                        db_service_prefix,
                     )
                 elif isinstance(
                     datamodel.DataSource.data_source_resp, DataSourceRespS3
@@ -476,7 +555,7 @@ class QuicksightSource(DashboardServiceSource):
                         data_model_entity,
                         datamodel.DataSource,
                         dashboard_details,
-                        db_service_name,
+                        db_service_prefix,
                     )
             except Exception as exc:  # pylint: disable=broad-except
                 yield Either(

metadata/ingestion/source/dashboard/redash/metadata.py

@@ -200,13 +200,20 @@ class RedashSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(  # pylint: disable=too-many-locals
         self,
         dashboard_details: dict,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
         In redash we do not get table, database_schema or database name but we do get query
         the lineage is being generated based on the query
         """
+        (
+            prefix_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         to_fqn = fqn.build(
             self.metadata,
             entity_type=LineageDashboard,
@@ -233,11 +240,48 @@ class RedashSource(DashboardServiceSource):
                         )
                         if not database_schema_table.get("table"):
                             continue
+
+                        if (
+                            prefix_table_name
+                            and prefix_table_name.lower()
+                            != database_schema_table.get("table").lower()
+                        ):
+                            logger.debug(
+                                f"Table {database_schema_table.get('table')} does not match prefix {prefix_table_name}"
+                            )
+                            continue
+
+                        if (
+                            prefix_schema_name
+                            and database_schema_name
+                            and prefix_schema_name.lower()
+                            != database_schema_name.lower()
+                        ):
+                            logger.debug(
+                                f"Schema {database_schema_name} does not match prefix {prefix_schema_name}"
+                            )
+                            continue
+
+                        if (
+                            prefix_database_name
+                            and database_schema_table.get("database")
+                            and prefix_database_name.lower()
+                            != database_schema_table.get("database").lower()
+                        ):
+                            logger.debug(
+                                f"Database {database_schema_table.get('database')} does not match prefix {prefix_database_name}"
+                            )
+                            continue
+
                         fqn_search_string = build_es_fqn_search_string(
-                            database_name=database_schema_table.get("database"),
-                            schema_name=database_schema_name,
-                            service_name=db_service_name or "*",
-                            table_name=database_schema_table.get("table"),
+                            database_name=(
+                                prefix_database_name
+                                or database_schema_table.get("database")
+                            ),
+                            schema_name=(prefix_schema_name or database_schema_name),
+                            service_name=prefix_service_name,
+                            table_name=prefix_table_name
+                            or database_schema_table.get("table"),
                         )
                         from_entity = self.metadata.search_in_any_service(
                             entity_type=Table,
@@ -253,7 +297,7 @@ class RedashSource(DashboardServiceSource):
                         name="Lineage",
                         error=(
                             "Error to yield dashboard lineage details for DB "
-                            f"service name [{db_service_name}]: {exc}"
+                            f"service name [{prefix_service_name}]: {exc}"
                         ),
                         stackTrace=traceback.format_exc(),
                     )
@@ -274,6 +318,7 @@ class RedashSource(DashboardServiceSource):
                 ):
                     self.status.filter(chart_display_name, "Chart Pattern not allowed")
                     continue
+
                 yield Either(
                     right=CreateChartRequest(
                         name=EntityName(str(widgets["id"])),

metadata/ingestion/source/dashboard/sigma/metadata.py

@@ -115,9 +115,11 @@ class SigmaSource(DashboardServiceSource):
             dashboard_request = CreateDashboardRequest(
                 name=EntityName(str(dashboard_details.workbookId)),
                 displayName=dashboard_details.name,
-                description=Markdown(dashboard_details.description)
-                if dashboard_details.description
-                else None,
+                description=(
+                    Markdown(dashboard_details.description)
+                    if dashboard_details.description
+                    else None
+                ),
                 charts=[
                     FullyQualifiedEntityName(
                         fqn.build(
@@ -165,9 +167,11 @@ class SigmaSource(DashboardServiceSource):
                             self.context.get().dashboard_service
                         ),
                         sourceUrl=SourceUrl(dashboard_details.url),
-                        description=Markdown(dashboard_details.description)
-                        if dashboard_details.description
-                        else None,
+                        description=(
+                            Markdown(dashboard_details.description)
+                            if dashboard_details.description
+                            else None
+                        ),
                     )
                 )
             except Exception as exc:
@@ -197,18 +201,61 @@ class SigmaSource(DashboardServiceSource):
         return None
 
     def _get_table_entity_from_node(
-        self, node: NodeDetails, db_service_name: Optional[str]
+        self, node: NodeDetails, db_service_prefix: Optional[str] = None
     ) -> Optional[Table]:
         """
         Get the table entity for lineage
         """
+        (
+            prefix_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix or "*")
+
         if node.node_schema:
+            schema_parts = node.node_schema.split(".")
+            schema_name = schema_parts[-1]
+            database_name = schema_parts[0] if len(schema_parts) > 1 else None
+            table_name = node.name
+
+            # Validate prefix filters
+            if (
+                prefix_table_name
+                and table_name
+                and prefix_table_name.lower() != table_name.lower()
+            ):
+                logger.debug(
+                    f"Table {table_name} does not match prefix {prefix_table_name}"
+                )
+                return None
+
+            if (
+                prefix_schema_name
+                and schema_name
+                and prefix_schema_name.lower() != schema_name.lower()
+            ):
+                logger.debug(
+                    f"Schema {schema_name} does not match prefix {prefix_schema_name}"
+                )
+                return None
+
+            if (
+                prefix_database_name
+                and database_name
+                and prefix_database_name.lower() != database_name.lower()
+            ):
+                logger.debug(
+                    f"Database {database_name} does not match prefix {prefix_database_name}"
+                )
+                return None
+
             try:
                 fqn_search_string = build_es_fqn_search_string(
-                    database_name=None,
-                    schema_name=node.node_schema,
-                    service_name=db_service_name or "*",
-                    table_name=node.name,
+                    service_name=prefix_service_name,
+                    database_name=prefix_database_name or database_name,
+                    schema_name=prefix_schema_name or schema_name,
+                    table_name=prefix_table_name or table_name,
                 )
                 return self.metadata.search_in_any_service(
                     entity_type=Table,
@@ -217,12 +264,13 @@ class SigmaSource(DashboardServiceSource):
             except Exception as exc:
                 logger.debug(traceback.format_exc())
                 logger.warning(f"Error occured while finding table fqn: {exc}")
+
         return None
 
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: WorkbookDetails,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ):
         """
         yield dashboard lineage
@@ -239,7 +287,7 @@ class SigmaSource(DashboardServiceSource):
                     )
                     for node in nodes:
                         table_entity = self._get_table_entity_from_node(
-                            node, db_service_name
+                            node, db_service_prefix
                         )
                         if table_entity and data_model.columns:
                             columns_list = data_model.columns
@@ -257,7 +305,7 @@ class SigmaSource(DashboardServiceSource):
                         name=f"{dashboard_details.name} Lineage",
                         error=(
                             "Error to yield dashboard lineage details for DB "
-                            f"service name [{db_service_name}]: {exc}"
+                            f"service prefix [{db_service_prefix}]: {exc}"
                         ),
                         stackTrace=traceback.format_exc(),
                     )

metadata/ingestion/source/dashboard/superset/api_source.py

@@ -130,10 +130,10 @@ class SupersetAPISource(SupersetSourceMixin):
             )
 
     def _get_datasource_fqn_for_lineage(
-        self, chart_json: ChartResult, db_service_name: Optional[str]
+        self, chart_json: ChartResult, db_service_prefix: Optional[str]
     ):
         return (
-            self._get_datasource_fqn(chart_json.datasource_id, db_service_name)
+            self._get_datasource_fqn(chart_json.datasource_id, db_service_prefix)
             if chart_json.datasource_id
             else None
         )
@@ -153,9 +153,11 @@ class SupersetAPISource(SupersetSourceMixin):
                 chart = CreateChartRequest(
                     name=EntityName(str(chart_json.id)),
                     displayName=chart_json.slice_name,
-                    description=Markdown(chart_json.description)
-                    if chart_json.description
-                    else None,
+                    description=(
+                        Markdown(chart_json.description)
+                        if chart_json.description
+                        else None
+                    ),
                     chartType=get_standard_chart_type(chart_json.viz_type),
                     sourceUrl=SourceUrl(
                         f"{clean_uri(self.service_connection.hostPort)}{chart_json.url}"
@@ -173,13 +175,19 @@ class SupersetAPISource(SupersetSourceMixin):
                 )
 
     def _get_datasource_fqn(
-        self, datasource_id: str, db_service_name: Optional[str]
+        self, datasource_id: str, db_service_prefix: Optional[str]
     ) -> Optional[str]:
+        (
+            db_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
         try:
             datasource_json = self.client.fetch_datasource(datasource_id)
             if datasource_json:
                 database_name = None
-                if db_service_name:
+                if db_service_prefix:
                     database_json = self.client.fetch_database(
                         datasource_json.result.database.id
                     )
@@ -194,11 +202,45 @@ class SupersetAPISource(SupersetSourceMixin):
                     database_name = get_database_name_for_lineage(
                         db_service_entity, default_database_name
                     )
+
+                    if (
+                        prefix_database_name
+                        and database_name
+                        and prefix_database_name.lower() != database_name.lower()
+                    ):
+                        logger.debug(
+                            f"Database {database_name} does not match prefix {prefix_database_name}"
+                        )
+                        return None
+
+                    if (
+                        prefix_schema_name
+                        and datasource_json.result.table_schema
+                        and prefix_schema_name.lower()
+                        != datasource_json.result.table_schema.lower()
+                    ):
+                        logger.debug(
+                            f"Schema {datasource_json.result.table_schema} does not match prefix {prefix_schema_name}"
+                        )
+                        return None
+
+                    if (
+                        prefix_table_name
+                        and datasource_json.result.table_name
+                        and prefix_table_name.lower()
+                        != datasource_json.result.table_name.lower()
+                    ):
+                        logger.debug(
+                            f"Table {datasource_json.result.table_name} does not match prefix {prefix_table_name}"
+                        )
+                        return None
+
                 return build_es_fqn_search_string(
-                    database_name=database_name,
-                    schema_name=datasource_json.result.table_schema,
+                    database_name=prefix_database_name or database_name,
+                    schema_name=prefix_schema_name
+                    or datasource_json.result.table_schema,
                     service_name=db_service_name or "*",
-                    table_name=datasource_json.result.table_name,
+                    table_name=prefix_table_name or datasource_json.result.table_name,
                 )
         except Exception as err:
             logger.debug(traceback.format_exc())

metadata/ingestion/source/dashboard/superset/db_source.py

@@ -160,10 +160,10 @@ class SupersetDBSource(SupersetSourceMixin):
             )
 
     def _get_datasource_fqn_for_lineage(
-        self, chart_json: FetchChart, db_service_name: Optional[str]
+        self, chart_json: FetchChart, db_service_prefix: Optional[str]
     ):
         return (
-            self._get_datasource_fqn(db_service_name, chart_json)
+            self._get_datasource_fqn(db_service_prefix, chart_json)
             if chart_json.table_name
             else None
         )
@@ -186,9 +186,11 @@ class SupersetDBSource(SupersetSourceMixin):
                 chart = CreateChartRequest(
                     name=EntityName(str(chart_json.id)),
                     displayName=chart_json.slice_name,
-                    description=Markdown(chart_json.description)
-                    if chart_json.description
-                    else None,
+                    description=(
+                        Markdown(chart_json.description)
+                        if chart_json.description
+                        else None
+                    ),
                     chartType=get_standard_chart_type(chart_json.viz_type),
                     sourceUrl=SourceUrl(
                         f"{clean_uri(self.service_connection.hostPort)}/explore/?slice_id={chart_json.id}"
@@ -216,9 +218,16 @@ class SupersetDBSource(SupersetSourceMixin):
         return get_database_name_for_lineage(db_service_entity, default_db_name)
 
     def _get_datasource_fqn(
-        self, db_service_name: Optional[str], chart_json: FetchChart
+        self, db_service_prefix: Optional[str], chart_json: FetchChart
     ) -> Optional[str]:
         try:
+            (
+                db_service_name,
+                prefix_database_name,
+                prefix_schema_name,
+                prefix_table_name,
+            ) = self.parse_db_service_prefix(db_service_prefix)
+
             database_name = None
             if db_service_name:
                 db_service_entity = self.metadata.get_by_name(
@@ -227,11 +236,42 @@ class SupersetDBSource(SupersetSourceMixin):
                 database_name = self._get_database_name(
                     chart_json.sqlalchemy_uri, db_service_entity
                 )
+
+            if (
+                prefix_database_name
+                and database_name
+                and prefix_database_name.lower() != database_name.lower()
+            ):
+                logger.debug(
+                    f"Database {database_name} does not match prefix {prefix_database_name}"
+                )
+                return None
+
+            if (
+                prefix_schema_name
+                and chart_json.table_schema
+                and prefix_schema_name.lower() != chart_json.table_schema.lower()
+            ):
+                logger.debug(
+                    f"Schema {chart_json.table_schema} does not match prefix {prefix_schema_name}"
+                )
+                return None
+
+            if (
+                prefix_table_name
+                and chart_json.table_name
+                and prefix_table_name.lower() != chart_json.table_name.lower()
+            ):
+                logger.debug(
+                    f"Table {chart_json.table_name} does not match prefix {prefix_table_name}"
+                )
+                return None
+
             return build_es_fqn_search_string(
-                database_name=database_name,
-                schema_name=chart_json.table_schema,
+                database_name=prefix_database_name or database_name,
+                schema_name=prefix_schema_name or chart_json.table_schema,
                 service_name=db_service_name or "*",
-                table_name=chart_json.table_name,
+                table_name=prefix_table_name or chart_json.table_name,
             )
         except Exception as err:
             logger.debug(traceback.format_exc())

metadata/ingestion/source/dashboard/superset/mixin.py

@@ -244,14 +244,14 @@ class SupersetSourceMixin(DashboardServiceSource):
         self,
         from_entities: List[Tuple[FetchChart, Dict[str, List[str]]]],
         to_entity: DashboardDataModel,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
     ):
         result = []
 
         for from_entity in from_entities:
             input_table, _column_lineage = from_entity
             datasource_fqn = self._get_datasource_fqn_for_lineage(
-                input_table, db_service_name
+                input_table, db_service_prefix
             )
             from_entity = self.metadata.search_in_any_service(
                 entity_type=Table,
@@ -308,7 +308,7 @@ class SupersetSourceMixin(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: Union[FetchDashboard, DashboardResult],
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between datamodel and table
@@ -326,7 +326,7 @@ class SupersetSourceMixin(DashboardServiceSource):
                 if to_entity:
                     _input_tables = self._get_input_tables(chart_json)
                     input_tables = self._enrich_raw_input_tables(
-                        _input_tables, to_entity, db_service_name
+                        _input_tables, to_entity, db_service_prefix
                     )
                     for input_table in input_tables:
                         from_entity_table, column_lineage = input_table
@@ -342,7 +342,7 @@ class SupersetSourceMixin(DashboardServiceSource):
                         name="Dashboard Lineage Details",
                         error=(
                             "Error to yield dashboard lineage details for DB "
-                            f"service name [{db_service_name}]: {exc}"
+                            f"service prefix [{db_service_prefix}]: {exc}"
                         ),
                         stackTrace=traceback.format_exc(),
                     )