openmetadata-ingestion 1.4.0.0rc1__py3-none-any.whl → 1.4.0.0rc2__py3-none-any.whl

metadata/ingestion/processor/query_parser.py

@@ -17,9 +17,6 @@ import traceback
 from typing import Optional
 
 from metadata.config.common import ConfigModel
-from metadata.generated.schema.entity.services.ingestionPipelines.status import (
-    StackTraceError,
-)
 from metadata.generated.schema.type.basic import DateTime
 from metadata.generated.schema.type.queryParserData import ParsedData, QueryParserData
 from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery
@@ -104,28 +101,35 @@ class QueryParserProcessor(Processor):
         return cls(config, metadata, connection_type)
 
     def _run(self, record: TableQueries) -> Optional[Either[QueryParserData]]:
-        if record and record.queries:
-            data = []
-            for table_query in record.queries:
-                try:
-                    parsed_sql = parse_sql_statement(
-                        table_query,
-                        ConnectionTypeDialectMapper.dialect_of(self.connection_type),
-                    )
-                    if parsed_sql:
-                        data.append(parsed_sql)
-                except Exception as exc:
-                    return Either(
-                        left=StackTraceError(
-                            name="Query",
-                            error=f"Error processing query [{table_query.query}]: {exc}",
-                            stackTrace=traceback.format_exc(),
-                        )
-                    )
-
-            return Either(right=QueryParserData(parsedData=data))
-
-        return None
+        if record is None or record.queries is None:
+            return None
+
+        data = []
+        success_cnt = 0
+        failed_cnt = 0
+        total_cnt = len(record.queries)
+
+        for table_query in record.queries:
+            try:
+                parsed_sql = parse_sql_statement(
+                    table_query,
+                    ConnectionTypeDialectMapper.dialect_of(self.connection_type),
+                )
+                if parsed_sql:
+                    data.append(parsed_sql)
+                success_cnt += 1
+            except Exception as exc:
+                failed_cnt += 1
+                logger.debug(traceback.format_exc())
+                logger.warning(f"Error processing query [{table_query.query}]: {exc}")
+            cur_total_cnt = success_cnt + failed_cnt
+            if cur_total_cnt % 1000 == 0 or cur_total_cnt == total_cnt:
+                logger.info(
+                    f"Total query count:{cur_total_cnt} / {total_cnt}."
+                    f" Current success count: {success_cnt}."
+                    f" Current failed count: {failed_cnt}."
+                )
+        return Either(right=QueryParserData(parsedData=data))
 
     def close(self):
         """Nothing to close"""

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -514,7 +514,7 @@ class PowerbiSource(DashboardServiceSource):
                 datamodel_fqn = fqn.build(
                     self.metadata,
                     entity_type=DashboardDataModel,
-                    service_name=self.config.serviceName,
+                    service_name=self.context.get().dashboard_service,
                     data_model_name=dataset.id,
                 )
                 datamodel_entity = self.metadata.get_by_name(
@@ -552,6 +552,25 @@ class PowerbiSource(DashboardServiceSource):
                 )
             )
 
+    @staticmethod
+    def _get_data_model_column_fqn(
+        data_model_entity: DashboardDataModel, column: str
+    ) -> Optional[str]:
+        """
+        Get fqn of column if exist in data model entity or its child columns
+        """
+        try:
+            if not data_model_entity:
+                return None
+            for tbl_column in data_model_entity.columns:
+                for child_column in tbl_column.children or []:
+                    if column.lower() == child_column.name.__root__.lower():
+                        return child_column.fullyQualifiedName.__root__
+            return None
+        except Exception as exc:
+            logger.debug(f"Error to get data_model_column_fqn {exc}")
+            logger.debug(traceback.format_exc())
+
     def _get_table_and_datamodel_lineage(
         self,
         db_service_name: str,
@@ -576,8 +595,14 @@ class PowerbiSource(DashboardServiceSource):
             )
 
             if table_entity and datamodel_entity:
+                columns_list = [column.name for column in table.columns]
+                column_lineage = self._get_column_lineage(
+                    table_entity, datamodel_entity, columns_list
+                )
                 return self._get_add_lineage_request(
-                    to_entity=datamodel_entity, from_entity=table_entity
+                    to_entity=datamodel_entity,
+                    from_entity=table_entity,
+                    column_lineage=column_lineage,
                 )
         except Exception as exc:  # pylint: disable=broad-except
             return Either(

metadata/ingestion/source/database/athena/client.py

@@ -74,7 +74,7 @@ class AthenaLakeFormationClient:
             return LFTags(**response)
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.warning(
+            logger.debug(
                 f"Unable to get lf tags for table resource [{table_name}] due to: {exc}"
             )
         return LFTags()

metadata/ingestion/source/database/athena/lineage.py

@@ -15,6 +15,7 @@ from typing import Iterable, Optional
 
 from metadata.generated.schema.type.tableQuery import TableQuery
 from metadata.ingestion.source.database.athena.query_parser import (
+    QUERY_SUCCESS_STATUS,
     AthenaQueryParserSource,
 )
 from metadata.ingestion.source.database.lineage_source import LineageSource
@@ -37,6 +38,7 @@ class AthenaLineageSource(AthenaQueryParserSource, LineageSource):
                 if (
                     query.Status.SubmissionDateTime.date() >= self.start.date()
                     and self.is_not_dbt_or_om_query(query.Query)
+                    and query.Status.State.upper() == QUERY_SUCCESS_STATUS
                 ):
                     yield TableQuery(
                         query=query.Query,

metadata/ingestion/source/database/athena/models.py

@@ -23,7 +23,7 @@ class QueryExecutionIdsResponse(BaseModel):
 
 
 class Status(BaseModel):
-    State: Optional[str]
+    State: Optional[str] = "FAILED"  # Default value
     SubmissionDateTime: Optional[datetime]
 
 

metadata/ingestion/source/database/athena/query_parser.py

@@ -41,6 +41,8 @@ ATHENA_QUERY_PAGINATOR_LIMIT = 50
 
 ATHENA_ENABLED_WORK_GROUP_STATE = "ENABLED"
 
+QUERY_SUCCESS_STATUS = "SUCCEEDED"
+
 
 class AthenaQueryParserSource(QueryParserSource, ABC):
     """

metadata/ingestion/source/database/athena/usage.py

@@ -15,6 +15,7 @@ from typing import Iterable
 
 from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery
 from metadata.ingestion.source.database.athena.query_parser import (
+    QUERY_SUCCESS_STATUS,
     AthenaQueryParserSource,
 )
 from metadata.ingestion.source.database.usage_source import UsageSource
@@ -56,6 +57,7 @@ class AthenaUsageSource(AthenaQueryParserSource, UsageSource):
                 for query in query_list.QueryExecutions
                 if query.Status
                 and query.Query
+                and query.Status.State.upper() == QUERY_SUCCESS_STATUS
                 and query.Status.SubmissionDateTime.date() >= self.start.date()
                 and self.is_not_dbt_or_om_query(query.Query)
             ]

metadata/ingestion/source/database/bigquery/metadata.py

@@ -99,7 +99,11 @@ from metadata.utils.credentials import GOOGLE_CREDENTIALS
 from metadata.utils.filters import filter_by_database, filter_by_schema
 from metadata.utils.helpers import get_start_and_end
 from metadata.utils.logger import ingestion_logger
-from metadata.utils.sqlalchemy_utils import is_complex_type
+from metadata.utils.sqlalchemy_utils import (
+    get_all_table_ddls,
+    get_table_ddl,
+    is_complex_type,
+)
 from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_label
 from metadata.utils.tag_utils import get_tag_labels as fetch_tag_labels_om
 
@@ -210,6 +214,9 @@ BigQueryDialect._build_formatted_table_id = (  # pylint: disable=protected-acces
 BigQueryDialect.get_pk_constraint = get_pk_constraint
 BigQueryDialect.get_foreign_keys = get_foreign_keys
 
+Inspector.get_all_table_ddls = get_all_table_ddls
+Inspector.get_table_ddl = get_table_ddl
+
 
 class BigquerySource(
     LifeCycleQueryMixin, StoredProcedureMixin, CommonDbSourceService, MultiDBSource
@@ -601,21 +608,33 @@ class BigquerySource(
                         f"Error trying to connect to database {project_id}: {exc}"
                     )
 
-    def get_view_definition(
+    def get_schema_definition(
         self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
     ) -> Optional[str]:
-        if table_type == TableType.View:
-            try:
+        """
+        Get the DDL statement or View Definition for a table
+        """
+        try:
+            if table_type == TableType.View:
                 view_definition = inspector.get_view_definition(
                     fqn._build(self.context.get().database, schema_name, table_name)
                 )
                 view_definition = (
-                    "" if view_definition is None else str(view_definition)
+                    f"CREATE VIEW {schema_name}.{table_name} AS {str(view_definition)}"
+                    if view_definition is not None
+                    else None
                 )
-            except NotImplementedError:
-                logger.warning("View definition not implemented")
-                view_definition = ""
-            return f"CREATE VIEW {schema_name}.{table_name} AS {view_definition}"
+                return view_definition
+
+            schema_definition = inspector.get_table_ddl(
+                self.connection, table_name, schema_name
+            )
+            schema_definition = (
+                str(schema_definition) if schema_definition is not None else None
+            )
+            return schema_definition
+        except NotImplementedError:
+            logger.warning("Schema definition not implemented")
         return None
 
     def get_table_partition_details(

metadata/ingestion/source/database/clickhouse/metadata.py

@@ -45,7 +45,9 @@ from metadata.ingestion.source.database.common_db_source import (
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import (
     get_all_table_comments,
+    get_all_table_ddls,
     get_all_view_definitions,
+    get_table_ddl,
 )
 
 logger = ingestion_logger()
@@ -95,6 +97,8 @@ ClickHouseDialect._get_column_info = (  # pylint: disable=protected-access
 )
 Inspector.get_mview_names = get_mview_names
 ClickHouseDialect.get_mview_names = get_mview_names_dialect
+Inspector.get_all_table_ddls = get_all_table_ddls
+Inspector.get_table_ddl = get_table_ddl
 
 
 class ClickhouseSource(CommonDbSourceService):
@@ -142,25 +146,29 @@ class ClickhouseSource(CommonDbSourceService):
 
         return regular_tables + material_tables + view_tables
 
-    def get_view_definition(
+    def get_schema_definition(
         self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
     ) -> Optional[str]:
-        if table_type in {TableType.View, TableType.MaterializedView}:
-            definition_fn = inspector.get_view_definition
-            try:
-                view_definition = definition_fn(table_name, schema_name)
-                view_definition = (
-                    "" if view_definition is None else str(view_definition)
+        """
+        Get the DDL statement or View Definition for a table
+        """
+        try:
+            if table_type in {TableType.View, TableType.MaterializedView}:
+                definition_fn = inspector.get_view_definition
+                schema_definition = definition_fn(table_name, schema_name)
+            else:
+                schema_definition = inspector.get_table_ddl(
+                    self.connection, table_name, schema_name
                 )
-                return view_definition
+            schema_definition = (
+                str(schema_definition) if schema_definition is not None else None
+            )
+            return schema_definition
 
-            except NotImplementedError:
-                logger.warning("View definition not implemented")
+        except NotImplementedError:
+            logger.warning("Schema definition not implemented")
 
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.warning(
-                    f"Failed to fetch view definition for {table_name}: {exc}"
-                )
-            return None
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch schema definition for {table_name}: {exc}")
         return None

metadata/ingestion/source/database/common_db_source.py

@@ -359,26 +359,32 @@ class CommonDbSourceService(
             logger.debug(traceback.format_exc())
 
     @calculate_execution_time()
-    def get_view_definition(
+    def get_schema_definition(
         self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
     ) -> Optional[str]:
-        if table_type in (TableType.View, TableType.MaterializedView):
-            try:
-                view_definition = inspector.get_view_definition(table_name, schema_name)
-                view_definition = (
-                    "" if view_definition is None else str(view_definition)
+        """
+        Get the DDL statement or View Definition for a table
+        """
+        try:
+            if table_type in (TableType.View, TableType.MaterializedView):
+                schema_definition = inspector.get_view_definition(
+                    table_name, schema_name
+                )
+            else:
+                schema_definition = inspector.get_table_ddl(
+                    self.connection, table_name, schema_name
                 )
-                return view_definition
+            schema_definition = (
+                str(schema_definition) if schema_definition is not None else None
+            )
+            return schema_definition
 
-            except NotImplementedError:
-                logger.warning("View definition not implemented")
+        except NotImplementedError:
+            logger.warning("Schema definition not implemented")
 
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.warning(
-                    f"Failed to fetch view definition for {table_name}: {exc}"
-                )
-            return None
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch schema definition for {table_name}: {exc}")
         return None
 
     def is_partition(  # pylint: disable=unused-argument
@@ -449,7 +455,7 @@ class CommonDbSourceService(
                 inspector=self.inspector,
             )
 
-            view_definition = self.get_view_definition(
+            schema_definition = self.get_schema_definition(
                 table_type=table_type,
                 table_name=table_name,
                 schema_name=schema_name,
@@ -468,7 +474,7 @@ class CommonDbSourceService(
                 ),
                 columns=columns,
                 tableConstraints=table_constraints,
-                viewDefinition=view_definition,
+                schemaDefinition=schema_definition,
                 databaseSchema=fqn.build(
                     metadata=self.metadata,
                     entity_type=DatabaseSchema,
@@ -501,13 +507,13 @@ class CommonDbSourceService(
             self.register_record(table_request=table_request)
 
             # Flag view as visited
-            if table_type == TableType.View or view_definition:
+            if table_type == TableType.View or schema_definition:
                 table_view = TableView.parse_obj(
                     {
                         "table_name": table_name,
                         "schema_name": schema_name,
                         "db_name": self.context.get().database,
-                        "view_definition": view_definition,
+                        "view_definition": schema_definition,
                     }
                 )
                 self.context.get_global().table_views.append(table_view)

metadata/ingestion/source/database/database_service.py

@@ -521,7 +521,9 @@ class DatabaseServiceSource(
         Method to process the table owners
         """
         try:
-            if self.source_config.includeOwners:
+            if self.source_config.includeOwners and hasattr(
+                self.inspector, "get_table_owner"
+            ):
                 owner_name = self.inspector.get_table_owner(
                     connection=self.connection,  # pylint: disable=no-member.fetchall()
                     table_name=table_name,

metadata/ingestion/source/database/databricks/metadata.py

@@ -60,7 +60,9 @@ from metadata.utils.constants import DEFAULT_DATABASE
 from metadata.utils.filters import filter_by_database
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import (
+    get_all_table_ddls,
     get_all_view_definitions,
+    get_table_ddl,
     get_view_definition_wrapper,
 )
 from metadata.utils.tag_utils import get_ometa_tag_and_classification
@@ -129,7 +131,6 @@ def get_columns(self, connection, table_name, schema=None, **kw):
     value should match what is provided in the 'source.config.database' field in the
     Databricks ingest config file.
     """
-    db_name = kw["db_name"] if "db_name" in kw else None
 
     rows = _get_column_rows(self, connection, table_name, schema)
     result = []
@@ -157,20 +158,13 @@ def get_columns(self, connection, table_name, schema=None, **kw):
             "system_data_type": raw_col_type,
         }
         if col_type in {"array", "struct", "map"}:
-            if db_name and schema:
-                rows = dict(
-                    connection.execute(
-                        f"DESCRIBE {db_name}.{schema}.{table_name} {col_name}"
-                    ).fetchall()
-                )
-            else:
-                rows = dict(
-                    connection.execute(
-                        f"DESCRIBE {schema}.{table_name} {col_name}"
-                        if schema
-                        else f"DESCRIBE {table_name} {col_name}"
-                    ).fetchall()
-                )
+            rows = dict(
+                connection.execute(
+                    f"DESCRIBE {schema}.{table_name} {col_name}"
+                    if schema
+                    else f"DESCRIBE {table_name} {col_name}"
+                ).fetchall()
+            )
 
             col_info["system_data_type"] = rows["data_type"]
             col_info["is_complex"] = True
@@ -259,6 +253,8 @@ DatabricksDialect.get_schema_names = get_schema_names
 DatabricksDialect.get_view_definition = get_view_definition
 DatabricksDialect.get_all_view_definitions = get_all_view_definitions
 reflection.Inspector.get_schema_names = get_schema_names_reflection
+reflection.Inspector.get_all_table_ddls = get_all_table_ddls
+reflection.Inspector.get_table_ddl = get_table_ddl
 
 
 class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDBSource):
@@ -596,9 +592,7 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
         try:
             cursor = self.connection.execute(
                 DATABRICKS_GET_TABLE_COMMENTS.format(
-                    schema_name=schema_name,
-                    table_name=table_name,
-                    catalog_name=self.context.get().database,
+                    schema_name=schema_name, table_name=table_name
                 )
             )
             for result in list(cursor):

metadata/ingestion/source/database/databricks/queries.py

@@ -24,10 +24,7 @@ DATABRICKS_VIEW_DEFINITIONS = textwrap.dedent(
     """
 )
 
-
-DATABRICKS_GET_TABLE_COMMENTS = (
-    "DESCRIBE TABLE EXTENDED {catalog_name}.{schema_name}.{table_name}"
-)
+DATABRICKS_GET_TABLE_COMMENTS = "DESCRIBE TABLE EXTENDED {schema_name}.{table_name}"
 
 DATABRICKS_GET_CATALOGS = "SHOW CATALOGS"
 

metadata/ingestion/source/database/dbt/metadata.py

@@ -28,7 +28,6 @@ from metadata.generated.schema.entity.data.table import (
     ModelType,
     Table,
 )
-from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
@@ -132,13 +131,7 @@ class DbtSource(DbtServiceSource):
         """
         By default for DBT nothing is required to be prepared
         """
-        database_service = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=self.config.serviceName
-        )
-        if not database_service:
-            raise InvalidServiceException(
-                f"Service with name {self.config.serviceName} not found"
-            )
+        pass
 
     def get_dbt_owner(
         self, manifest_node: dict, catalog_node: Optional[dict]
@@ -413,7 +406,7 @@ class DbtSource(DbtServiceSource):
                     table_fqn = fqn.build(
                         self.metadata,
                         entity_type=Table,
-                        service_name=self.config.serviceName,
+                        service_name="*",
                         database_name=get_corrected_name(manifest_node.database),
                         schema_name=get_corrected_name(manifest_node.schema_),
                         table_name=model_name,
@@ -507,7 +500,7 @@ class DbtSource(DbtServiceSource):
                         parent_fqn = fqn.build(
                             self.metadata,
                             entity_type=Table,
-                            service_name=self.config.serviceName,
+                            service_name="*",
                             database_name=get_corrected_name(parent_node.database),
                             schema_name=get_corrected_name(parent_node.schema_),
                             table_name=table_name,
@@ -586,9 +579,10 @@ class DbtSource(DbtServiceSource):
                 columns.append(
                     Column(
                         name=column_name,
-                        description=manifest_column.description
-                        if manifest_column.description
-                        else column_description,
+                        # If the catalog description is present, use it, else use the manifest description
+                        description=column_description
+                        if column_description
+                        else manifest_column.description,
                         dataType=ColumnTypeParser.get_column_type(
                             catalog_column.type
                             if catalog_column
@@ -948,7 +942,7 @@ class DbtSource(DbtServiceSource):
                     test_case_fqn = fqn.build(
                         self.metadata,
                         entity_type=TestCase,
-                        service_name=self.config.serviceName,
+                        service_name=source_elements[0],
                         database_name=source_elements[1],
                         schema_name=source_elements[2],
                         table_name=source_elements[3],

metadata/ingestion/source/database/deltalake/connection.py

@@ -65,13 +65,13 @@ def get_connection(connection: DeltaLakeConnection) -> SparkSession:
             )
         if connection.metastoreConnection.jdbcDriverClassPath:
             builder.config(
-                "sparks.driver.extraClassPath",
+                "spark.driver.extraClassPath",
                 connection.metastoreConnection.jdbcDriverClassPath,
             )
         if connection.metastoreConnection.username:
             builder.config(
                 "spark.hadoop.javax.jdo.option.ConnectionUserName",
-                connection.metastoreConnection.metastoreDb,
+                connection.metastoreConnection.username,
             )
         if connection.metastoreConnection.password:
             builder.config(

metadata/ingestion/source/database/deltalake/metadata.py

@@ -284,7 +284,7 @@ class DeltalakeSource(DatabaseServiceSource):
                     database_name=self.context.get().database,
                     schema_name=schema_name,
                 ),
-                viewDefinition=view_definition,
+                schemaDefinition=view_definition,
             )
 
             yield Either(right=table_request)

metadata/ingestion/source/database/hive/metadata.py

@@ -12,9 +12,11 @@
 Hive source methods.
 """
 
+import traceback
 from typing import Optional, Tuple
 
 from pyhive.sqlalchemy_hive import HiveDialect
+from sqlalchemy.engine.reflection import Inspector
 
 from metadata.generated.schema.entity.services.connections.database.hiveConnection import (
     HiveConnection,
@@ -96,3 +98,24 @@ class HiveSource(CommonDbSourceService):
             )
         self._connection_map = {}  # Lazy init as well
         self._inspector_map = {}
+
+    def get_schema_definition(  # pylint: disable=unused-argument
+        self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
+    ) -> Optional[str]:
+        """
+        Get the DDL statement or View Definition for a table
+        """
+        try:
+            schema_definition = inspector.get_view_definition(table_name, schema_name)
+            schema_definition = (
+                str(schema_definition) if schema_definition is not None else None
+            )
+            return schema_definition
+
+        except NotImplementedError:
+            logger.warning("Schema definition not implemented")
+
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch schema definition for {table_name}: {exc}")
+        return None

metadata/ingestion/source/database/mssql/metadata.py

@@ -13,6 +13,7 @@ import traceback
 from typing import Dict, Iterable, List, Optional
 
 from sqlalchemy.dialects.mssql.base import MSDialect, ischema_names
+from sqlalchemy.engine.reflection import Inspector
 
 from metadata.generated.schema.api.data.createStoredProcedure import (
     CreateStoredProcedureRequest,
@@ -65,7 +66,9 @@ from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqa_utils import update_mssql_ischema_names
 from metadata.utils.sqlalchemy_utils import (
     get_all_table_comments,
+    get_all_table_ddls,
     get_all_view_definitions,
+    get_table_ddl,
 )
 
 logger = ingestion_logger()
@@ -87,6 +90,9 @@ MSDialect.get_foreign_keys = get_foreign_keys
 MSDialect.get_table_names = get_table_names
 MSDialect.get_view_names = get_view_names
 
+Inspector.get_all_table_ddls = get_all_table_ddls
+Inspector.get_table_ddl = get_table_ddl
+
 
 class MssqlSource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource):
     """