openmetadata-ingestion 1.8.1.0__py3-none-any.whl → 1.8.2.0__py3-none-any.whl

metadata/ingestion/source/dashboard/superset/db_source.py

@@ -160,10 +160,10 @@ class SupersetDBSource(SupersetSourceMixin):
             )
 
     def _get_datasource_fqn_for_lineage(
-        self, chart_json: FetchChart, db_service_name: Optional[str]
+        self, chart_json: FetchChart, db_service_prefix: Optional[str]
     ):
         return (
-            self._get_datasource_fqn(db_service_name, chart_json)
+            self._get_datasource_fqn(db_service_prefix, chart_json)
             if chart_json.table_name
             else None
         )
@@ -186,9 +186,11 @@ class SupersetDBSource(SupersetSourceMixin):
                 chart = CreateChartRequest(
                     name=EntityName(str(chart_json.id)),
                     displayName=chart_json.slice_name,
-                    description=Markdown(chart_json.description)
-                    if chart_json.description
-                    else None,
+                    description=(
+                        Markdown(chart_json.description)
+                        if chart_json.description
+                        else None
+                    ),
                     chartType=get_standard_chart_type(chart_json.viz_type),
                     sourceUrl=SourceUrl(
                         f"{clean_uri(self.service_connection.hostPort)}/explore/?slice_id={chart_json.id}"
@@ -216,9 +218,16 @@ class SupersetDBSource(SupersetSourceMixin):
         return get_database_name_for_lineage(db_service_entity, default_db_name)
 
     def _get_datasource_fqn(
-        self, db_service_name: Optional[str], chart_json: FetchChart
+        self, db_service_prefix: Optional[str], chart_json: FetchChart
     ) -> Optional[str]:
         try:
+            (
+                db_service_name,
+                prefix_database_name,
+                prefix_schema_name,
+                prefix_table_name,
+            ) = self.parse_db_service_prefix(db_service_prefix)
+
             database_name = None
             if db_service_name:
                 db_service_entity = self.metadata.get_by_name(
@@ -227,11 +236,42 @@ class SupersetDBSource(SupersetSourceMixin):
                 database_name = self._get_database_name(
                     chart_json.sqlalchemy_uri, db_service_entity
                 )
+
+            if (
+                prefix_database_name
+                and database_name
+                and prefix_database_name.lower() != database_name.lower()
+            ):
+                logger.debug(
+                    f"Database {database_name} does not match prefix {prefix_database_name}"
+                )
+                return None
+
+            if (
+                prefix_schema_name
+                and chart_json.table_schema
+                and prefix_schema_name.lower() != chart_json.table_schema.lower()
+            ):
+                logger.debug(
+                    f"Schema {chart_json.table_schema} does not match prefix {prefix_schema_name}"
+                )
+                return None
+
+            if (
+                prefix_table_name
+                and chart_json.table_name
+                and prefix_table_name.lower() != chart_json.table_name.lower()
+            ):
+                logger.debug(
+                    f"Table {chart_json.table_name} does not match prefix {prefix_table_name}"
+                )
+                return None
+
             return build_es_fqn_search_string(
-                database_name=database_name,
-                schema_name=chart_json.table_schema,
+                database_name=prefix_database_name or database_name,
+                schema_name=prefix_schema_name or chart_json.table_schema,
                 service_name=db_service_name or "*",
-                table_name=chart_json.table_name,
+                table_name=prefix_table_name or chart_json.table_name,
             )
         except Exception as err:
             logger.debug(traceback.format_exc())

metadata/ingestion/source/dashboard/superset/mixin.py

@@ -244,14 +244,14 @@ class SupersetSourceMixin(DashboardServiceSource):
         self,
         from_entities: List[Tuple[FetchChart, Dict[str, List[str]]]],
         to_entity: DashboardDataModel,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
     ):
         result = []
 
         for from_entity in from_entities:
             input_table, _column_lineage = from_entity
             datasource_fqn = self._get_datasource_fqn_for_lineage(
-                input_table, db_service_name
+                input_table, db_service_prefix
             )
             from_entity = self.metadata.search_in_any_service(
                 entity_type=Table,
@@ -308,7 +308,7 @@ class SupersetSourceMixin(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: Union[FetchDashboard, DashboardResult],
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between datamodel and table
@@ -326,7 +326,7 @@ class SupersetSourceMixin(DashboardServiceSource):
                 if to_entity:
                     _input_tables = self._get_input_tables(chart_json)
                     input_tables = self._enrich_raw_input_tables(
-                        _input_tables, to_entity, db_service_name
+                        _input_tables, to_entity, db_service_prefix
                     )
                     for input_table in input_tables:
                         from_entity_table, column_lineage = input_table
@@ -342,7 +342,7 @@ class SupersetSourceMixin(DashboardServiceSource):
                         name="Dashboard Lineage Details",
                         error=(
                             "Error to yield dashboard lineage details for DB "
-                            f"service name [{db_service_name}]: {exc}"
+                            f"service prefix [{db_service_prefix}]: {exc}"
                         ),
                         stackTrace=traceback.format_exc(),
                     )

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -242,9 +242,9 @@ class TableauSource(DashboardServiceSource):
             data_model_request = CreateDashboardDataModelRequest(
                 name=EntityName(data_model.id),
                 displayName=data_model_name,
-                description=Markdown(data_model.description)
-                if data_model.description
-                else None,
+                description=(
+                    Markdown(data_model.description) if data_model.description else None
+                ),
                 service=FullyQualifiedEntityName(self.context.get().dashboard_service),
                 dataModelType=data_model_type.value,
                 serviceType=DashboardServiceType.Tableau.value,
@@ -311,9 +311,11 @@ class TableauSource(DashboardServiceSource):
             dashboard_request = CreateDashboardRequest(
                 name=EntityName(dashboard_details.id),
                 displayName=dashboard_details.name,
-                description=Markdown(dashboard_details.description)
-                if dashboard_details.description
-                else None,
+                description=(
+                    Markdown(dashboard_details.description)
+                    if dashboard_details.description
+                    else None
+                ),
                 project=self.get_project_name(dashboard_details=dashboard_details),
                 charts=[
                     FullyQualifiedEntityName(
@@ -458,7 +460,7 @@ class TableauSource(DashboardServiceSource):
         self,
         upstream_data_model: DataSource,
         datamodel: DataSource,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
         upstream_data_model_entity: DashboardDataModel,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
@@ -472,7 +474,7 @@ class TableauSource(DashboardServiceSource):
                 if column is not None
             }
             for table in datamodel.upstreamTables or []:
-                om_tables = self._get_database_tables(db_service_name, table)
+                om_tables = self._get_database_tables(db_service_prefix, table)
                 for om_table_and_query in om_tables or []:
                     column_lineage = self._get_column_lineage(
                         table,
@@ -572,11 +574,17 @@ class TableauSource(DashboardServiceSource):
         self,
         datamodel: DataSource,
         data_model_entity: DashboardDataModel,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
     ) -> Iterable[Either[AddLineageRequest]]:
         """ "
         Method to create lineage between tables<->published datasource<->embedded datasource
         """
+        (
+            db_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
         for upstream_data_model in datamodel.upstreamDatasources or []:
             try:
                 upstream_data_model_entity = self._get_datamodel(
@@ -596,7 +604,7 @@ class TableauSource(DashboardServiceSource):
                     yield from self._get_table_datamodel_lineage(
                         upstream_data_model=upstream_data_model,
                         datamodel=datamodel,
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         upstream_data_model_entity=upstream_data_model_entity,
                     )
 
@@ -613,11 +621,13 @@ class TableauSource(DashboardServiceSource):
                                 )
                             lineage_parser = LineageParser(
                                 query,
-                                ConnectionTypeDialectMapper.dialect_of(
-                                    db_service_entity.serviceType.value
-                                )
-                                if db_service_entity
-                                else Dialect.ANSI,
+                                (
+                                    ConnectionTypeDialectMapper.dialect_of(
+                                        db_service_entity.serviceType.value
+                                    )
+                                    if db_service_entity
+                                    else Dialect.ANSI
+                                ),
                             )
                             for source_table in lineage_parser.source_tables or []:
                                 database_schema_table = fqn.split_table_name(
@@ -637,11 +647,44 @@ class TableauSource(DashboardServiceSource):
                                     database_schema_table.get("database_schema")
                                 )
                                 table_name = database_schema_table.get("table")
+
+                                if (
+                                    prefix_database_name
+                                    and database_name
+                                    and prefix_database_name.lower()
+                                    != database_name.lower()
+                                ):
+                                    logger.debug(
+                                        f"Database {database_name} does not match prefix {prefix_database_name}"
+                                    )
+                                    continue
+
+                                if (
+                                    prefix_schema_name
+                                    and schema_name
+                                    and prefix_schema_name.lower()
+                                    != schema_name.lower()
+                                ):
+                                    logger.debug(
+                                        f"Schema {schema_name} does not match prefix {prefix_schema_name}"
+                                    )
+                                    continue
+
+                                if (
+                                    prefix_table_name
+                                    and table_name
+                                    and prefix_table_name.lower() != table_name.lower()
+                                ):
+                                    logger.debug(
+                                        f"Table {table_name} does not match prefix {prefix_table_name}"
+                                    )
+                                    continue
+
                                 fqn_search_string = build_es_fqn_search_string(
-                                    database_name=database_name,
-                                    schema_name=schema_name,
+                                    database_name=prefix_database_name or database_name,
+                                    schema_name=prefix_schema_name or schema_name,
                                     service_name=db_service_name or "*",
-                                    table_name=table_name,
+                                    table_name=prefix_table_name or table_name,
                                 )
                                 from_entities = self.metadata.search_in_any_service(
                                     entity_type=Table,
@@ -675,7 +718,7 @@ class TableauSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: TableauDashboard,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         This method creates the lineage between tables and datamodels
@@ -697,7 +740,7 @@ class TableauSource(DashboardServiceSource):
                         yield from self._get_datamodel_table_lineage(
                             datamodel=datamodel,
                             data_model_entity=data_model_entity,
-                            db_service_name=db_service_name,
+                            db_service_prefix=db_service_prefix,
                         )
                     else:
                         # else we'll create lineage only using Embedded Datasources in below format
@@ -705,7 +748,7 @@ class TableauSource(DashboardServiceSource):
                         yield from self._get_table_datamodel_lineage(
                             upstream_data_model=datamodel,
                             datamodel=datamodel,
-                            db_service_name=db_service_name,
+                            db_service_prefix=db_service_prefix,
                             upstream_data_model_entity=data_model_entity,
                         )
 
@@ -715,7 +758,7 @@ class TableauSource(DashboardServiceSource):
                         name="Lineage",
                         error=(
                             "Error to yield dashboard lineage details for DB "
-                            f"service name [{db_service_name}]: {err}"
+                            f"service prefix [{db_service_prefix}]: {err}"
                         ),
                         stackTrace=traceback.format_exc(),
                     )
@@ -785,18 +828,26 @@ class TableauSource(DashboardServiceSource):
         self.metadata.close()
 
     def _get_table_entities_from_api(
-        self, db_service_name: Optional[str], table: UpstreamTable
+        self, db_service_prefix: Optional[str], table: UpstreamTable
     ) -> Optional[List[TableAndQuery]]:
         """
         In case we get the table details from the Graphql APIs we process them
         """
         try:
+            (
+                db_service_name,
+                prefix_database_name,
+                prefix_schema_name,
+                prefix_table_name,
+            ) = self.parse_db_service_prefix(db_service_prefix)
+
             database_schema_table = fqn.split_table_name(table.name)
             database_name = (
                 table.database.name
                 if table.database and table.database.name
                 else database_schema_table.get("database")
             )
+
             if db_service_name:
                 db_service_entity = self.metadata.get_by_name(
                     entity=DatabaseService, fqn=db_service_name
@@ -812,11 +863,38 @@ class TableauSource(DashboardServiceSource):
                 else database_schema_table.get("database_schema")
             )
             table_name = database_schema_table.get("table")
+
+            if (
+                prefix_database_name
+                and database_name
+                and prefix_database_name.lower() != database_name.lower()
+            ):
+                logger.debug(
+                    f"Database {database_name} does not match prefix {prefix_database_name}"
+                )
+                return None
+            if (
+                prefix_schema_name
+                and schema_name
+                and prefix_schema_name.lower() != schema_name.lower()
+            ):
+                logger.debug(
+                    f"Schema {schema_name} does not match prefix {prefix_schema_name}"
+                )
+            if (
+                prefix_table_name
+                and table_name
+                and prefix_table_name.lower() != table_name.lower()
+            ):
+                logger.debug(
+                    f"Table {table_name} does not match prefix {prefix_table_name}"
+                )
+
             fqn_search_string = build_es_fqn_search_string(
-                database_name=database_name,
-                schema_name=schema_name,
+                database_name=prefix_database_name or database_name,
+                schema_name=prefix_schema_name or schema_name,
                 service_name=db_service_name or "*",
-                table_name=table_name,
+                table_name=prefix_table_name or table_name,
             )
             table_entity = self.metadata.search_in_any_service(
                 entity_type=Table,
@@ -861,13 +939,20 @@ class TableauSource(DashboardServiceSource):
         return None
 
     def _get_table_entities_from_query(
-        self, db_service_name: Optional[str], table: UpstreamTable
+        self, db_service_prefix: Optional[str], table: UpstreamTable
     ) -> Optional[List[TableAndQuery]]:
         """
         In case we get the table details from the Graphql APIs we process them
         """
         tables_list = []
         try:
+            (
+                db_service_name,
+                prefix_database_name,
+                prefix_schema_name,
+                prefix_table_name,
+            ) = self.parse_db_service_prefix(db_service_prefix)
+
             for custom_sql_table in table.referencedByQueries or []:
                 db_service_entity = None
                 if db_service_name:
@@ -876,11 +961,13 @@ class TableauSource(DashboardServiceSource):
                     )
                 lineage_parser = LineageParser(
                     custom_sql_table.query,
-                    ConnectionTypeDialectMapper.dialect_of(
-                        db_service_entity.serviceType.value
-                    )
-                    if db_service_entity
-                    else Dialect.ANSI,
+                    (
+                        ConnectionTypeDialectMapper.dialect_of(
+                            db_service_entity.serviceType.value
+                        )
+                        if db_service_entity
+                        else Dialect.ANSI
+                    ),
                 )
                 for source_table in lineage_parser.source_tables or []:
                     database_schema_table = fqn.split_table_name(str(source_table))
@@ -897,11 +984,40 @@ class TableauSource(DashboardServiceSource):
                         database_schema_table.get("database_schema")
                     )
                     table_name = database_schema_table.get("table")
+
+                    if (
+                        prefix_database_name
+                        and database_name
+                        and prefix_database_name.lower() != database_name.lower()
+                    ):
+                        logger.debug(
+                            f"Database {database_name} does not match prefix {prefix_database_name}"
+                        )
+                        continue
+                    if (
+                        prefix_schema_name
+                        and schema_name
+                        and prefix_schema_name.lower() != schema_name.lower()
+                    ):
+                        logger.debug(
+                            f"Schema {schema_name} does not match prefix {prefix_schema_name}"
+                        )
+                        continue
+                    if (
+                        prefix_table_name
+                        and table_name
+                        and prefix_table_name.lower() != table_name.lower()
+                    ):
+                        logger.debug(
+                            f"Table {table_name} does not match prefix {prefix_table_name}"
+                        )
+                        continue
+
                     fqn_search_string = build_es_fqn_search_string(
-                        database_name=database_name,
-                        schema_name=schema_name,
+                        database_name=prefix_database_name or database_name,
+                        schema_name=prefix_schema_name or schema_name,
                         service_name=db_service_name or "*",
-                        table_name=table_name,
+                        table_name=prefix_table_name or table_name,
                     )
                     from_entities = self.metadata.search_in_any_service(
                         entity_type=Table,
@@ -928,7 +1044,7 @@ class TableauSource(DashboardServiceSource):
         return tables_list or []
 
     def _get_database_tables(
-        self, db_service_name: Optional[str], table: UpstreamTable
+        self, db_service_prefix: Optional[str], table: UpstreamTable
     ) -> Optional[List[TableAndQuery]]:
         """
         Get the table entities for lineage
@@ -936,12 +1052,12 @@ class TableauSource(DashboardServiceSource):
         # If we get the table details from the Graphql APIs we process them directly
         if table.name:
             return self._get_table_entities_from_api(
-                db_service_name=db_service_name, table=table
+                db_service_prefix=db_service_prefix, table=table
             )
         # Else we get the table details from the SQL queries and process them using SQL lineage parser
         if table.referencedByQueries:
             return self._get_table_entities_from_query(
-                db_service_name=db_service_name, table=table
+                db_service_prefix=db_service_prefix, table=table
             )
         return None
 
@@ -971,9 +1087,11 @@ class TableauSource(DashboardServiceSource):
             try:
                 if column:
                     parsed_column = {
-                        "dataTypeDisplay": column.remoteType
-                        if column.remoteType
-                        else DataType.UNKNOWN.value,
+                        "dataTypeDisplay": (
+                            column.remoteType
+                            if column.remoteType
+                            else DataType.UNKNOWN.value
+                        ),
                         "dataType": ColumnTypeParser.get_column_type(
                             column.remoteType if column.remoteType else None
                         ),

metadata/ingestion/source/database/bigquery/service_spec.py

@@ -1,9 +1,9 @@
 from metadata.ingestion.source.database.bigquery.lineage import BigqueryLineageSource
 from metadata.ingestion.source.database.bigquery.metadata import BigquerySource
-from metadata.ingestion.source.database.bigquery.profiler.profiler import (
-    BigQueryProfiler,
-)
 from metadata.ingestion.source.database.bigquery.usage import BigqueryUsageSource
+from metadata.profiler.interface.sqlalchemy.bigquery.profiler_interface import (
+    BigQueryProfilerInterface,
+)
 from metadata.sampler.sqlalchemy.bigquery.sampler import BigQuerySampler
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
@@ -11,6 +11,6 @@ ServiceSpec = DefaultDatabaseSpec(
     metadata_source_class=BigquerySource,
     lineage_source_class=BigqueryLineageSource,
     usage_source_class=BigqueryUsageSource,
-    profiler_class=BigQueryProfiler,
+    profiler_class=BigQueryProfilerInterface,
     sampler_class=BigQuerySampler,
 )

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -12,6 +12,7 @@
 Hosts the singledispatch to get DBT files
 """
 import json
+import os
 import traceback
 from collections import defaultdict
 from functools import singledispatch
@@ -78,11 +79,7 @@ def _(config: DbtLocalConfig):
     try:
         blob_grouped_by_directory = defaultdict(list)
 
-        subdirectory = (
-            config.dbtManifestFilePath.rsplit("/", 1)[0]
-            if "/" in config.dbtManifestFilePath
-            else ""
-        )
+        subdirectory = os.path.dirname(config.dbtManifestFilePath)
         blob_grouped_by_directory[subdirectory] = [
             config.dbtManifestFilePath,
             config.dbtCatalogFilePath,
@@ -247,8 +244,8 @@ def get_blobs_grouped_by_dir(blobs: List[str]) -> Dict[str, List[str]]:
     """
     blob_grouped_by_directory = defaultdict(list)
     for blob in blobs:
-        subdirectory = blob.rsplit("/", 1)[0] if "/" in blob else ""
-        blob_file_name = blob.rsplit("/", 1)[1] if "/" in blob else blob
+        subdirectory = os.path.dirname(blob)
+        blob_file_name = os.path.basename(blob)
         # We'll be processing multiple run_result files from a single dir
         # Grouping them together to process them in a single go
         if (
@@ -283,7 +280,7 @@ def download_dbt_files(
             for blob in blobs:
                 if blob:
                     reader = get_reader(config_source=config, client=client)
-                    blob_file_name = blob.rsplit("/", 1)[1] if "/" in blob else blob
+                    blob_file_name = os.path.basename(blob)
                     if DBT_MANIFEST_FILE_NAME == blob_file_name.lower():
                         logger.debug(f"{DBT_MANIFEST_FILE_NAME} found in {key}")
                         dbt_manifest = reader.read(path=blob, **kwargs)

metadata/ingestion/source/database/lineage_source.py

@@ -49,6 +49,7 @@ from metadata.ingestion.source.database.query_parser_source import QueryParserSo
 from metadata.ingestion.source.models import TableView
 from metadata.utils import fqn
 from metadata.utils.db_utils import get_view_lineage
+from metadata.utils.filters import filter_by_table
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -313,6 +314,15 @@ class LineageSource(QueryParserSource, ABC):
     ) -> Iterable[Either[AddLineageRequest]]:
         try:
             for view in views:
+                if filter_by_table(
+                    self.source_config.tableFilterPattern,
+                    view.table_name,
+                ):
+                    self.status.filter(
+                        view.table_name,
+                        "View Filtered Out",
+                    )
+                    continue
                 for lineage in get_view_lineage(
                     view=view,
                     metadata=self.metadata,

metadata/ingestion/source/database/redshift/queries.py

@@ -15,9 +15,6 @@ SQL Queries used during ingestion
 import textwrap
 from typing import List
 
-from sqlalchemy import text
-from sqlalchemy.orm.session import Session
-
 from metadata.utils.profiler_utils import QueryResult
 from metadata.utils.time_utils import datetime_to_timestamp
 
@@ -418,38 +415,6 @@ STL_QUERY = """
 """
 
 
-def get_query_results(
-    session: Session,
-    query,
-    operation,
-) -> List[QueryResult]:
-    """get query results either from cache or from the database
-
-    Args:
-        session (Session): session
-        query (_type_): query
-        operation (_type_): operation
-
-    Returns:
-        List[QueryResult]:
-    """
-    cursor = session.execute(text(query))
-    results = [
-        QueryResult(
-            database_name=row.database,
-            schema_name=row.schema,
-            table_name=row.table,
-            query_text=None,
-            query_type=operation,
-            start_time=row.starttime,
-            rows=row.rows,
-        )
-        for row in cursor
-    ]
-
-    return results
-
-
 def get_metric_result(ddls: List[QueryResult], table_name: str) -> List:
     """Given query results, retur the metric result
 

metadata/ingestion/source/database/redshift/service_spec.py

@@ -1,14 +1,14 @@
 from metadata.ingestion.source.database.redshift.lineage import RedshiftLineageSource
 from metadata.ingestion.source.database.redshift.metadata import RedshiftSource
-from metadata.ingestion.source.database.redshift.profiler.profiler import (
-    RedshiftProfiler,
-)
 from metadata.ingestion.source.database.redshift.usage import RedshiftUsageSource
+from metadata.profiler.interface.sqlalchemy.redshift.profiler_interface import (
+    RedshiftProfilerInterface,
+)
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
 ServiceSpec = DefaultDatabaseSpec(
     metadata_source_class=RedshiftSource,
     lineage_source_class=RedshiftLineageSource,
     usage_source_class=RedshiftUsageSource,
-    profiler_class=RedshiftProfiler,
+    profiler_class=RedshiftProfilerInterface,
 )