openmetadata-ingestion 1.7.0.0rc2__py3-none-any.whl → 1.7.1.0__py3-none-any.whl

metadata/ingestion/source/database/dbt/metadata.py

@@ -150,29 +150,37 @@ class DbtSource(DbtServiceSource):
         Returns dbt owner
         """
         try:
-            owner = None
+            owner_ref = None
             dbt_owner = None
             if catalog_node:
                 dbt_owner = catalog_node.metadata.owner
             if manifest_node:
                 dbt_owner = manifest_node.meta.get(DbtCommonEnum.OWNER.value)
-            if dbt_owner:
-                owner = self.metadata.get_reference_by_name(
+            if dbt_owner and isinstance(dbt_owner, str):
+                owner_ref = self.metadata.get_reference_by_name(
                     name=dbt_owner, is_owner=True
+                ) or self.metadata.get_reference_by_email(email=dbt_owner)
+                if owner_ref:
+                    return owner_ref
+                logger.warning(
+                    "Unable to ingest owner from DBT since no user or"
+                    f" team was found with name {dbt_owner}"
                 )
-
-                if owner:
-                    return owner
-
-                # If owner is not found, try to find the owner in OMD using email
-                owner = self.metadata.get_reference_by_email(email=dbt_owner)
-
-                if not owner:
-                    logger.warning(
-                        "Unable to ingest owner from DBT since no user or"
-                        f" team was found with name {dbt_owner}"
-                    )
-            return owner
+            elif dbt_owner and isinstance(dbt_owner, list):
+                owner_list = EntityReferenceList(root=[])
+                for owner_name in dbt_owner:
+                    owner_ref = self.metadata.get_reference_by_name(
+                        name=owner_name, is_owner=True
+                    ) or self.metadata.get_reference_by_email(email=owner_name)
+                    if owner_ref:
+                        owner_list.root.extend(owner_ref.root)
+                    else:
+                        logger.warning(
+                            "Unable to ingest owner from DBT since no user or"
+                            f" team was found with name {owner_name}"
+                        )
+                if owner_list.root:
+                    return owner_list
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(f"Unable to ingest owner from DBT due to: {exc}")
@@ -741,7 +749,12 @@ class DbtSource(DbtServiceSource):
                                     type="table",
                                 ),
                                 lineageDetails=LineageDetails(
-                                    source=LineageSource.DbtLineage
+                                    source=LineageSource.DbtLineage,
+                                    sqlQuery=SqlQuery(
+                                        data_model_link.datamodel.sql.root
+                                    )
+                                    if data_model_link.datamodel.sql
+                                    else None,
                                 ),
                             )
                         )

metadata/ingestion/source/database/hive/metadata.py

@@ -18,6 +18,7 @@ from typing import Optional, Tuple
 from pyhive.sqlalchemy_hive import HiveDialect
 from sqlalchemy.engine.reflection import Inspector
 
+from metadata.generated.schema.entity.data.table import TableType
 from metadata.generated.schema.entity.services.connections.database.hiveConnection import (
     HiveConnection,
 )
@@ -106,7 +107,13 @@ class HiveSource(CommonDbSourceService):
         Get the DDL statement or View Definition for a table
         """
         try:
-            schema_definition = inspector.get_view_definition(table_name, schema_name)
+            if self.source_config.includeDDL or table_type in (
+                TableType.View,
+                TableType.MaterializedView,
+            ):
+                schema_definition = inspector.get_view_definition(
+                    table_name, schema_name
+                )
             schema_definition = (
                 str(schema_definition).strip()
                 if schema_definition is not None

metadata/ingestion/source/database/life_cycle_query_mixin.py

@@ -128,6 +128,15 @@ class LifeCycleQueryMixin:
                 )
             )
 
+    def get_life_cycle_query(self):
+        """
+        Get the life cycle query
+        """
+        return self.life_cycle_query.format(
+            database_name=self.context.get().database,
+            schema_name=self.context.get().database_schema,
+        )
+
     def yield_life_cycle_data(self, _) -> Iterable[Either[OMetaLifeCycleData]]:
         """
         Get the life cycle data of the table

metadata/ingestion/source/database/mysql/connection.py

@@ -38,7 +38,10 @@ from metadata.ingestion.connections.test_connections import (
     test_connection_db_schema_sources,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.database.mysql.queries import MYSQL_TEST_GET_QUERIES
+from metadata.ingestion.source.database.mysql.queries import (
+    MYSQL_TEST_GET_QUERIES,
+    MYSQL_TEST_GET_QUERIES_SLOW_LOGS,
+)
 from metadata.utils.constants import THREE_MIN
 
 
@@ -50,7 +53,10 @@ def get_connection(connection: MysqlConnection) -> Engine:
         azure_client = AzureClient(connection.authType.azureConfig).create_client()
         if not connection.authType.azureConfig.scopes:
             raise ValueError(
-                "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default"
+                "Azure Scopes are missing, please refer https://learn.microsoft.com/"
+                "en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-micr"
+                "osoft-entra-access-token and fetch the resource associated with it,"
+                " for e.g. https://ossrdbms-aad.database.windows.net/.default"
             )
         access_token_obj = azure_client.get_token(
             *connection.authType.azureConfig.scopes.split(",")
@@ -75,7 +81,9 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
     queries = {
-        "GetQueries": MYSQL_TEST_GET_QUERIES,
+        "GetQueries": MYSQL_TEST_GET_QUERIES
+        if not service_connection.useSlowLogs
+        else MYSQL_TEST_GET_QUERIES_SLOW_LOGS,
     }
     return test_connection_db_schema_sources(
         metadata=metadata,

metadata/ingestion/source/database/mysql/lineage.py

@@ -20,9 +20,9 @@ class MysqlLineageSource(MysqlQueryParserSource, LineageSource):
     sql_stmt = MYSQL_SQL_STATEMENT
     filters = """
         AND (
-            lower(argument) LIKE '%%create%%table%%select%%'
-            OR lower(argument) LIKE '%%insert%%into%%select%%'
-            OR lower(argument) LIKE '%%update%%'
-            OR lower(argument) LIKE '%%merge%%'
+            lower({sql_column}) LIKE '%%create%%table%%select%%'
+            OR lower({sql_column}) LIKE '%%insert%%into%%select%%'
+            OR lower({sql_column}) LIKE '%%update%%'
+            OR lower({sql_column}) LIKE '%%merge%%'
         )
     """

metadata/ingestion/source/database/mysql/queries.py

@@ -36,8 +36,37 @@ LIMIT {result_limit};
 """
 )
 
+
+MYSQL_SQL_STATEMENT_SLOW_LOGS = textwrap.dedent(
+    """
+SELECT 
+	NULL `database_name`,
+	sql_text `query_text`,
+	start_time `start_time`,
+    NULL `end_time`,
+	NULL  `duration`,
+	NULL `schema_name`,
+    NULL `query_type`,
+    NULL `user_name`,
+    NULL `aborted`
+FROM mysql.slow_log
+WHERE start_time between '{start_time}' and '{end_time}'
+    AND sql_text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
+    AND sql_text NOT LIKE '/* {{"app": "dbt", %%}} */%%'
+    {filters}
+ORDER BY start_time desc
+LIMIT {result_limit};
+"""
+)
+
 MYSQL_TEST_GET_QUERIES = textwrap.dedent(
     """
 SELECT `argument` from mysql.general_log limit 1;
 """
 )
+
+MYSQL_TEST_GET_QUERIES_SLOW_LOGS = textwrap.dedent(
+    """
+SELECT `sql_text` from mysql.slow_log limit 1;
+"""
+)

metadata/ingestion/source/database/mysql/query_parser.py

@@ -12,6 +12,7 @@
 Mysql query parser module
 """
 from abc import ABC
+from datetime import datetime
 from typing import Optional
 
 from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
@@ -22,6 +23,10 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 )
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.mysql.queries import (
+    MYSQL_SQL_STATEMENT,
+    MYSQL_SQL_STATEMENT_SLOW_LOGS,
+)
 from metadata.ingestion.source.database.query_parser_source import QueryParserSource
 
 
@@ -44,3 +49,29 @@ class MysqlQueryParserSource(QueryParserSource, ABC):
                 f"Expected MysqlConnection, but got {connection}"
             )
         return cls(config, metadata)
+
+    def get_sql_statement(self, start_time: datetime, end_time: datetime) -> str:
+        """
+        returns sql statement to fetch query logs.
+
+        Override if we have specific parameters
+        """
+        if self.service_connection.useSlowLogs:
+            self.sql_stmt = MYSQL_SQL_STATEMENT_SLOW_LOGS
+        else:
+            self.sql_stmt = MYSQL_SQL_STATEMENT
+        return self.sql_stmt.format(
+            start_time=start_time,
+            end_time=end_time,
+            filters=self.get_filters(),
+            result_limit=self.source_config.resultLimit,
+        )
+
+    def get_filters(self) -> str:
+        if self.service_connection.useSlowLogs:
+            sql_column = "sql_text"
+        else:
+            sql_column = "argument"
+        if self.source_config.filterCondition:
+            return f"{self.filters.format(sql_column=sql_column)} AND {self.source_config.filterCondition}"
+        return self.filters.format(sql_column=sql_column)

metadata/ingestion/source/database/oracle/metadata.py

@@ -57,7 +57,6 @@ from metadata.ingestion.source.database.oracle.utils import (
     _get_col_type,
     _get_constraint_data,
     get_columns,
-    get_mview_definition,
     get_mview_names,
     get_mview_names_dialect,
     get_table_comment,
@@ -94,7 +93,6 @@ OracleDialect.get_all_view_definitions = get_all_view_definitions
 OracleDialect.get_all_table_comments = get_all_table_comments
 OracleDialect.get_table_names = get_table_names
 Inspector.get_mview_names = get_mview_names
-Inspector.get_mview_definition = get_mview_definition
 OracleDialect.get_mview_names = get_mview_names_dialect
 Inspector.get_view_names = get_view_names
 OracleDialect.get_view_names = get_view_names_dialect
@@ -146,43 +144,6 @@ class OracleSource(CommonDbSourceService):
 
         return regular_tables + material_tables
 
-    def get_schema_definition(
-        self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
-    ) -> Optional[str]:
-        """
-        Get the DDL statement or View Definition for a table
-        """
-        try:
-            if table_type not in {TableType.View, TableType.MaterializedView}:
-                schema_definition = inspector.get_table_ddl(
-                    self.connection, table_name, schema_name
-                )
-                return (
-                    str(schema_definition).strip()
-                    if schema_definition is not None
-                    else None
-                )
-
-            definition_fn = inspector.get_view_definition
-            if table_type == TableType.MaterializedView:
-                definition_fn = inspector.get_mview_definition
-
-            schema_definition = definition_fn(table_name, schema_name)
-
-            return (
-                str(schema_definition).strip()
-                if schema_definition is not None
-                else None
-            )
-
-        except NotImplementedError:
-            logger.warning("Schema definition not implemented")
-
-        except Exception as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(f"Failed to fetch Schema definition for {table_name}: {exc}")
-        return None
-
     def process_result(self, data: FetchObjectList):
         """Process data as per our stored procedure format"""
         result_dict = {}

metadata/ingestion/source/database/oracle/queries.py

@@ -31,14 +31,14 @@ ORACLE_ALL_VIEW_DEFINITIONS = textwrap.dedent(
 SELECT
 LOWER(view_name) AS "view_name",
 LOWER(owner) AS "schema",
-DBMS_METADATA.GET_DDL('VIEW', view_name, owner) AS view_def
+text AS view_def
 FROM DBA_VIEWS
 WHERE owner NOT IN ('SYSTEM', 'SYS')
 UNION ALL
 SELECT
 LOWER(mview_name) AS "view_name",
 LOWER(owner) AS "schema",
-DBMS_METADATA.GET_DDL('MATERIALIZED_VIEW', mview_name, owner) AS view_def
+query AS view_def
 FROM DBA_MVIEWS
 WHERE owner NOT IN ('SYSTEM', 'SYS')
 """

metadata/ingestion/source/database/oracle/utils.py

@@ -264,20 +264,6 @@ def get_mview_names_dialect(self, connection, schema=None, **kw):
     return [self.normalize_name(row[0]) for row in cursor]
 
 
-def get_mview_definition(self, mview_name, schema=None):
-    """Return definition for `mview_name`.
-
-    :param schema: Optional, retrieve names from a non-default schema.
-        For special quoting, use :class:`.quoted_name`.
-
-    """
-
-    with self._operation_context() as conn:
-        return self.dialect.get_view_definition(
-            conn, mview_name, schema, info_cache=self.info_cache
-        )
-
-
 @reflection.cache
 def _get_constraint_data(self, connection, table_name, schema=None, dblink="", **kw):
 

metadata/ingestion/source/database/postgres/metadata.py

@@ -71,6 +71,7 @@ from metadata.ingestion.source.database.postgres.utils import (
     get_columns,
     get_etable_owner,
     get_foreign_keys,
+    get_schema_names,
     get_table_comment,
     get_table_owner,
     get_view_definition,
@@ -115,6 +116,7 @@ Inspector.get_table_ddl = get_table_ddl
 Inspector.get_table_owner = get_etable_owner
 
 PGDialect.get_foreign_keys = get_foreign_keys
+PGDialect.get_schema_names = get_schema_names
 
 
 class PostgresSource(CommonDbSourceService, MultiDBSource):
@@ -281,7 +283,7 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
         for row in results:
             try:
                 stored_procedure = PostgresStoredProcedure.model_validate(
-                    dict(row._mapping)
+                    dict(row._mapping)  # pylint: disable=protected-access
                 )
                 yield stored_procedure
             except Exception as exc:

metadata/ingestion/source/database/postgres/queries.py

@@ -196,6 +196,13 @@ POSTGRES_GET_SERVER_VERSION = """
 show server_version_num
 """
 
+# pylint: disable=anomalous-backslash-in-string
+POSTGRES_GET_SCHEMA_NAMES = """
+SELECT nspname FROM pg_namespace
+    WHERE nspname NOT LIKE 'pg\_%'
+    ORDER BY nspname
+"""
+
 POSTGRES_FETCH_FK = """
     SELECT r.conname,
         pg_catalog.pg_get_constraintdef(r.oid, true) as condef,

metadata/ingestion/source/database/postgres/utils.py

@@ -26,6 +26,7 @@ from sqlalchemy.sql import sqltypes
 from metadata.ingestion.source.database.postgres.queries import (
     POSTGRES_COL_IDENTITY,
     POSTGRES_FETCH_FK,
+    POSTGRES_GET_SCHEMA_NAMES,
     POSTGRES_GET_SERVER_VERSION,
     POSTGRES_SQL_COLUMNS,
     POSTGRES_TABLE_COMMENTS,
@@ -43,10 +44,8 @@ logger = utils_logger()
 
 OLD_POSTGRES_VERSION = "130000"
 
-
-def get_etable_owner(
-    self, connection, table_name=None, schema=None
-):  # pylint: disable=unused-argument
+# pylint: disable=unused-argument,too-many-arguments,invalid-name,too-many-locals
+def get_etable_owner(self, connection, table_name=None, schema=None):
     """Return all owners.
 
     :param schema: Optional, retrieve names from a non-default schema.
@@ -67,6 +66,16 @@ def get_etable_owner(
 def get_foreign_keys(
     self, connection, table_name, schema=None, postgresql_ignore_search_path=False, **kw
 ):
+    """
+    Args:
+        connection (_type_): _description_
+        table_name (_type_): _description_
+        schema (_type_, optional): _description_. Defaults to None.
+        postgresql_ignore_search_path (bool, optional): _description_. Defaults to False.
+
+    Returns:
+        _type_: _description_
+    """
     preparer = self.identifier_preparer
     table_oid = self.get_table_oid(
         connection, table_name, schema, info_cache=kw.get("info_cache")
@@ -87,7 +96,7 @@ def get_foreign_keys(
     t = sql.text(POSTGRES_FETCH_FK).columns(
         conname=sqltypes.Unicode, condef=sqltypes.Unicode, con_db_name=sqltypes.Unicode
     )
-    c = connection.execute(t, dict(table=table_oid))
+    c = connection.execute(t, {"table": table_oid})
     fkeys = []
     for conname, condef, conschema, con_db_name in c.fetchall():
         m = re.search(FK_REGEX, condef).groups()
@@ -109,7 +118,7 @@ def get_foreign_keys(
         ) = m
 
         if deferrable is not None:
-            deferrable = True if deferrable == "DEFERRABLE" else False
+            deferrable = deferrable == "DEFERRABLE"
         constrained_columns = tuple(re.split(r"\s*,\s*", constrained_columns))
         constrained_columns = [
             preparer._unquote_identifier(x) for x in constrained_columns
@@ -161,9 +170,7 @@ def get_foreign_keys(
 
 
 @reflection.cache
-def get_table_owner(
-    self, connection, table_name, schema=None, **kw
-):  # pylint: disable=unused-argument
+def get_table_owner(self, connection, table_name, schema=None, **kw):
     return get_table_owner_wrapper(
         self,
         connection=connection,
@@ -174,9 +181,7 @@ def get_table_owner(
 
 
 @reflection.cache
-def get_table_comment(
-    self, connection, table_name, schema=None, **kw
-):  # pylint: disable=unused-argument
+def get_table_comment(self, connection, table_name, schema=None, **kw):
     return get_table_comment_wrapper(
         self,
         connection,
@@ -187,9 +192,7 @@ def get_table_comment(
 
 
 @reflection.cache
-def get_columns(  # pylint: disable=too-many-locals
-    self, connection, table_name, schema=None, **kw
-):
+def get_columns(self, connection, table_name, schema=None, **kw):
     """
     Overriding the dialect method to add raw_data_type in response
     """
@@ -368,7 +371,7 @@ def _handle_array_type(attype):
     )
 
 
-# pylint: disable=too-many-statements,too-many-branches,too-many-locals,too-many-arguments
+# pylint: disable=too-many-statements,too-many-branches
 def get_column_info(
     self,
     name,
@@ -477,9 +480,7 @@ def get_column_info(
 
 
 @reflection.cache
-def get_view_definition(
-    self, connection, table_name, schema=None, **kw
-):  # pylint: disable=unused-argument
+def get_view_definition(self, connection, table_name, schema=None, **kw):
     return get_view_definition_wrapper(
         self,
         connection,
@@ -515,3 +516,11 @@ def get_postgres_time_column_name(engine) -> str:
     ):
         time_column_name = "total_time"
     return time_column_name
+
+
+@reflection.cache
+def get_schema_names(self, connection, **kw):
+    result = connection.execute(
+        sql.text(POSTGRES_GET_SCHEMA_NAMES).columns(nspname=sqltypes.Unicode)
+    )
+    return [name for name, in result]

metadata/ingestion/source/database/snowflake/data_diff/data_diff.py

@@ -0,0 +1,37 @@
+"""Snowflake spec for data diff"""
+
+from typing import Optional, cast
+
+from metadata.data_quality.validations.models import TableParameter
+from metadata.data_quality.validations.runtime_param_setter.base_diff_params_setter import (
+    BaseTableParameter,
+)
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.ingestion.source.database.snowflake.metadata import SnowflakeConnection
+
+
+class SnowflakeTableParameter(BaseTableParameter):
+    """SnowflakeTableParameter class for setting runtime parameters for data diff"""
+
+    def get(
+        self,
+        service: DatabaseService,
+        entity: Table,
+        key_columns,
+        extra_columns,
+        case_sensitive_columns,
+        service_url: Optional[str],
+    ) -> TableParameter:
+        table_param: TableParameter = super().get(
+            service,
+            entity,
+            key_columns,
+            extra_columns,
+            case_sensitive_columns,
+            service_url,
+        )
+        connection_config = cast(SnowflakeConnection, service.connection.config)
+        table_param.privateKey = connection_config.privateKey
+        table_param.passPhrase = connection_config.snowflakePrivatekeyPassphrase
+        return table_param