openmetadata-ingestion 1.5.0.0rc1__py3-none-any.whl → 1.5.0.0rc2__py3-none-any.whl

metadata/ingestion/source/database/mssql/usage.py

@@ -11,8 +11,21 @@
 """
 MSSQL usage module
 """
+from datetime import datetime
+
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.mssql.constants import (
+    DEFAULT_DATETIME_FORMAT,
+    MSSQL_DATEFORMAT_DATETIME_MAP,
+)
 from metadata.ingestion.source.database.mssql.queries import MSSQL_SQL_STATEMENT
 from metadata.ingestion.source.database.mssql.query_parser import MssqlQueryParserSource
+from metadata.ingestion.source.database.mssql.utils import (
+    get_sqlalchemy_engine_dateformat,
+)
 from metadata.ingestion.source.database.usage_source import UsageSource
 
 
@@ -24,3 +37,32 @@ class MssqlUsageSource(MssqlQueryParserSource, UsageSource):
         AND lower(t.text) NOT LIKE '%%create%%function%%'
         AND lower(t.text) NOT LIKE '%%declare%%'
         """
+
+    def __init__(
+        self,
+        config: WorkflowSource,
+        metadata: OpenMetadata,
+        get_engine: bool = True,
+    ):
+        super().__init__(config, metadata, get_engine)
+
+        self.dt_format = DEFAULT_DATETIME_FORMAT
+
+        if self.engine:
+            server_date_format = get_sqlalchemy_engine_dateformat(self.engine)
+            self.dt_format = MSSQL_DATEFORMAT_DATETIME_MAP.get(
+                server_date_format, DEFAULT_DATETIME_FORMAT
+            )
+
+    def get_sql_statement(self, start_time: datetime, end_time: datetime) -> str:
+        """
+        returns sql statement to fetch query logs.
+
+        Override if we have specific parameters
+        """
+        return self.sql_stmt.format(
+            start_time=start_time.strftime(self.dt_format),
+            end_time=end_time.strftime(self.dt_format),
+            filters=self.get_filters(),
+            result_limit=self.source_config.resultLimit,
+        )

metadata/ingestion/source/database/oracle/metadata.py

@@ -54,6 +54,7 @@ from metadata.ingestion.source.database.oracle.queries import (
 )
 from metadata.ingestion.source.database.oracle.utils import (
     _get_col_type,
+    _get_constraint_data,
     get_columns,
     get_mview_definition,
     get_mview_names,
@@ -61,6 +62,8 @@ from metadata.ingestion.source.database.oracle.utils import (
     get_table_comment,
     get_table_names,
     get_view_definition,
+    get_view_names,
+    get_view_names_dialect,
 )
 from metadata.ingestion.source.database.stored_procedures_mixin import (
     QueryByProcedure,
@@ -97,10 +100,14 @@ OracleDialect.get_table_names = get_table_names
 Inspector.get_mview_names = get_mview_names
 Inspector.get_mview_definition = get_mview_definition
 OracleDialect.get_mview_names = get_mview_names_dialect
+Inspector.get_view_names = get_view_names
+OracleDialect.get_view_names = get_view_names_dialect
 
 Inspector.get_all_table_ddls = get_all_table_ddls
 Inspector.get_table_ddl = get_table_ddl
 
+OracleDialect._get_constraint_data = _get_constraint_data
+
 
 class OracleSource(StoredProcedureMixin, CommonDbSourceService):
     """

metadata/ingestion/source/database/oracle/queries.py

@@ -20,7 +20,7 @@ SELECT
 	comments table_comment,
 	LOWER(table_name) "table_name",
 	LOWER(owner) "schema" 	
-FROM ALL_TAB_COMMENTS
+FROM DBA_TAB_COMMENTS
 where comments is not null and owner not in ('SYSTEM', 'SYS')
 """
 )
@@ -32,33 +32,39 @@ SELECT
 LOWER(view_name) AS "view_name",
 LOWER(owner) AS "schema",
 DBMS_METADATA.GET_DDL('VIEW', view_name, owner) AS view_def
-FROM ALL_VIEWS
+FROM DBA_VIEWS
 WHERE owner NOT IN ('SYSTEM', 'SYS')
 UNION ALL
 SELECT
 LOWER(mview_name) AS "view_name",
 LOWER(owner) AS "schema",
 DBMS_METADATA.GET_DDL('MATERIALIZED_VIEW', mview_name, owner) AS view_def
-FROM ALL_MVIEWS
+FROM DBA_MVIEWS
 WHERE owner NOT IN ('SYSTEM', 'SYS')
 """
 )
 
+GET_VIEW_NAMES = textwrap.dedent(
+    """
+SELECT view_name FROM DBA_VIEWS WHERE owner = :owner
+"""
+)
+
 GET_MATERIALIZED_VIEW_NAMES = textwrap.dedent(
     """
-SELECT mview_name FROM ALL_MVIEWS WHERE owner = :owner
+SELECT mview_name FROM DBA_MVIEWS WHERE owner = :owner
 """
 )
 
 ORACLE_GET_TABLE_NAMES = textwrap.dedent(
     """
-SELECT table_name FROM ALL_TABLES WHERE 
+SELECT table_name FROM DBA_TABLES WHERE 
 {tablespace}
 OWNER = :owner  
 AND IOT_NAME IS NULL 
 AND DURATION IS NULL
 AND TABLE_NAME NOT IN 
-(SELECT mview_name FROM ALL_MVIEWS WHERE owner = :owner)
+(SELECT mview_name FROM DBA_MVIEWS WHERE owner = :owner)
 """
 )
 
@@ -67,7 +73,7 @@ ORACLE_IDENTITY_TYPE = textwrap.dedent(
 col.default_on_null,
 (
 	SELECT id.generation_type || ',' || id.IDENTITY_OPTIONS
-	FROM ALL_TAB_IDENTITY_COLS{dblink} id
+	FROM DBA_TAB_IDENTITY_COLS{dblink} id
 	WHERE col.table_name = id.table_name
 	AND col.column_name = id.column_name
 	AND col.owner = id.owner
@@ -83,12 +89,12 @@ SELECT
     LINE,
     TEXT
 FROM
-    ALL_SOURCE
+    DBA_SOURCE
 WHERE
     type = 'PROCEDURE' and owner = '{schema}'
 """
 )
-CHECK_ACCESS_TO_ALL = "SELECT table_name FROM ALL_TABLES where ROWNUM < 2"
+CHECK_ACCESS_TO_ALL = "SELECT table_name FROM DBA_TABLES where ROWNUM < 2"
 ORACLE_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent(
     """
 WITH SP_HISTORY AS (SELECT
@@ -153,8 +159,8 @@ ORACLE_GET_COLUMNS = textwrap.dedent(
             com.comments,
             col.virtual_column,
             {identity_cols}
-        FROM ALL_TAB_COLS{dblink} col
-        LEFT JOIN ALL_COL_COMMENTS{dblink} com
+        FROM DBA_TAB_COLS{dblink} col
+        LEFT JOIN DBA_COL_COMMENTS{dblink} com
         ON col.table_name = com.table_name
         AND col.column_name = com.column_name
         AND col.owner = com.owner
@@ -163,6 +169,34 @@ ORACLE_GET_COLUMNS = textwrap.dedent(
     """
 )
 
+ORACLE_ALL_CONSTRAINTS = textwrap.dedent(
+    """
+        SELECT
+            ac.constraint_name,
+            ac.constraint_type,
+            loc.column_name AS local_column,
+            rem.table_name AS remote_table,
+            rem.column_name AS remote_column,
+            rem.owner AS remote_owner,
+            loc.position as loc_pos,
+            rem.position as rem_pos,
+            ac.search_condition,
+            ac.delete_rule
+        FROM DBA_CONSTRAINTS{dblink} ac,
+            DBA_CONS_COLUMNS{dblink} loc,
+            DBA_CONS_COLUMNS{dblink} rem
+        WHERE ac.table_name = CAST(:table_name AS VARCHAR2(128))
+            AND ac.constraint_type IN ('R','P', 'U', 'C')
+            AND ac.owner = CAST(:owner AS VARCHAR2(128))
+            AND ac.owner = loc.owner
+            AND ac.constraint_name = loc.constraint_name
+            AND ac.r_owner = rem.owner(+)
+            AND ac.r_constraint_name = rem.constraint_name(+)
+            AND (rem.position IS NULL or loc.position=rem.position)
+        ORDER BY ac.constraint_name, loc.position
+    """
+)
+
 ORACLE_QUERY_HISTORY_STATEMENT = textwrap.dedent(
     """
 SELECT 

metadata/ingestion/source/database/oracle/utils.py

@@ -21,6 +21,8 @@ from sqlalchemy.sql import sqltypes
 
 from metadata.ingestion.source.database.oracle.queries import (
     GET_MATERIALIZED_VIEW_NAMES,
+    GET_VIEW_NAMES,
+    ORACLE_ALL_CONSTRAINTS,
     ORACLE_ALL_TABLE_COMMENTS,
     ORACLE_ALL_VIEW_DEFINITIONS,
     ORACLE_GET_COLUMNS,
@@ -222,6 +224,26 @@ def get_table_names(self, connection, schema=None, **kw):
     return [row[0] for row in cursor]
 
 
+def get_view_names(self, schema=None):
+    """Return all materialized view names in `schema`.
+
+    :param schema: Optional, retrieve names from a non-default schema.
+        For special quoting, use :class:`.quoted_name`.
+
+    """
+
+    with self._operation_context() as conn:
+        return self.dialect.get_view_names(conn, schema, info_cache=self.info_cache)
+
+
+@reflection.cache
+def get_view_names_dialect(self, connection, schema=None, **kw):
+    schema = self.denormalize_name(schema or self.default_schema_name)
+    sql_query = sql.text(GET_VIEW_NAMES)
+    cursor = connection.execute(sql_query, {"owner": self.denormalize_name(schema)})
+    return [self.normalize_name(row[0]) for row in cursor]
+
+
 def get_mview_names(self, schema=None):
     """Return all materialized view names in `schema`.
 
@@ -254,3 +276,14 @@ def get_mview_definition(self, mview_name, schema=None):
         return self.dialect.get_view_definition(
             conn, mview_name, schema, info_cache=self.info_cache
         )
+
+
+@reflection.cache
+def _get_constraint_data(self, connection, table_name, schema=None, dblink="", **kw):
+
+    params = {"table_name": table_name, "owner": schema}
+    text = ORACLE_ALL_CONSTRAINTS.format(dblink=dblink)
+
+    rp = connection.execute(sql.text(text), params)
+    constraint_data = rp.fetchall()
+    return constraint_data

metadata/ingestion/source/database/postgres/usage.py

@@ -73,3 +73,9 @@ class PostgresUsageSource(PostgresQueryParserSource, UsageSource):
                 )
             logger.error(f"Source usage processing error - {err}")
             logger.debug(traceback.format_exc())
+
+    def get_filters(self) -> str:
+        if filter_condition := self.source_config.filterCondition:
+            filter_condition = filter_condition.replace("%", "%%")
+            return f"{self.filters} AND s.{filter_condition}"
+        return self.filters

metadata/ingestion/source/database/unitycatalog/client.py

@@ -14,6 +14,8 @@ Client to interact with databricks apis
 import json
 import traceback
 
+from requests import HTTPError
+
 from metadata.ingestion.source.database.databricks.client import (
     API_TIMEOUT,
     DatabricksClient,
@@ -27,6 +29,7 @@ from metadata.utils.logger import ingestion_logger
 logger = ingestion_logger()
 TABLE_LINEAGE_PATH = "/lineage-tracking/table-lineage/get"
 COLUMN_LINEAGE_PATH = "/lineage-tracking/column-lineage/get"
+TABLES_PATH = "/unity-catalog/tables"
 
 
 class UnityCatalogClient(DatabricksClient):
@@ -85,3 +88,21 @@ class UnityCatalogClient(DatabricksClient):
             logger.error(exc)
 
         return LineageColumnStreams()
+
+    def get_owner_info(self, full_table_name: str) -> str:
+        """
+        get owner info from tables API
+        """
+        try:
+            response = self.client.get(
+                f"{self.base_url}{TABLES_PATH}/{full_table_name}",
+                headers=self.headers,
+                timeout=API_TIMEOUT,
+            )
+            if response.status_code != 200:
+                raise HTTPError(response.text)
+            return response.json().get("owner")
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.error(exc)
+        return

metadata/ingestion/source/database/unitycatalog/metadata.py

@@ -49,7 +49,12 @@ from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+)
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -61,6 +66,7 @@ from metadata.ingestion.source.database.external_table_lineage_mixin import (
 )
 from metadata.ingestion.source.database.multi_db_source import MultiDBSource
 from metadata.ingestion.source.database.stored_procedures_mixin import QueryByProcedure
+from metadata.ingestion.source.database.unitycatalog.client import UnityCatalogClient
 from metadata.ingestion.source.database.unitycatalog.connection import get_connection
 from metadata.ingestion.source.database.unitycatalog.models import (
     ColumnJson,
@@ -99,6 +105,7 @@ class UnitycatalogSource(
         )
         self.external_location_map = {}
         self.client = get_connection(self.service_connection)
+        self.api_client = UnityCatalogClient(self.service_connection)
         self.connection_obj = self.client
         self.table_constraints = []
         self.context.storage_location = None
@@ -147,9 +154,11 @@ class UnitycatalogSource(
                     )
                     if filter_by_database(
                         self.config.sourceConfig.config.databaseFilterPattern,
-                        database_fqn
-                        if self.config.sourceConfig.config.useFqnForFiltering
-                        else catalog_name,
+                        (
+                            database_fqn
+                            if self.config.sourceConfig.config.useFqnForFiltering
+                            else catalog_name
+                        ),
                     ):
                         self.status.filter(
                             database_fqn,
@@ -196,9 +205,11 @@ class UnitycatalogSource(
                 )
                 if filter_by_schema(
                     self.config.sourceConfig.config.schemaFilterPattern,
-                    schema_fqn
-                    if self.config.sourceConfig.config.useFqnForFiltering
-                    else schema.name,
+                    (
+                        schema_fqn
+                        if self.config.sourceConfig.config.useFqnForFiltering
+                        else schema.name
+                    ),
                 ):
                     self.status.filter(schema_fqn, "Schema Filtered Out")
                     continue
@@ -260,9 +271,11 @@ class UnitycatalogSource(
                 )
                 if filter_by_table(
                     self.config.sourceConfig.config.tableFilterPattern,
-                    table_fqn
-                    if self.config.sourceConfig.config.useFqnForFiltering
-                    else table_name,
+                    (
+                        table_fqn
+                        if self.config.sourceConfig.config.useFqnForFiltering
+                        else table_name
+                    ),
                 ):
                     self.status.filter(
                         table_fqn,
@@ -330,6 +343,7 @@ class UnitycatalogSource(
                         schema_name=schema_name,
                     )
                 ),
+                owners=self.get_owner_ref(table_name),
             )
             yield Either(right=table_request)
 
@@ -447,12 +461,12 @@ class UnitycatalogSource(
         """
         try:
             if column.children is None:
-                if column_json.metadata:
-                    column.description = column_json.metadata.comment
+                if column_json.metadata and column_json.metadata.comment:
+                    column.description = Markdown(column_json.metadata.comment)
             else:
                 for i, child in enumerate(column.children):
-                    if column_json.metadata:
-                        column.description = column_json.metadata.comment
+                    if column_json.metadata and column_json.metadata.comment:
+                        column.description = Markdown(column_json.metadata.comment)
                     if (
                         column_json.type
                         and isinstance(column_json.type, Type)
@@ -483,17 +497,23 @@ class UnitycatalogSource(
         """
 
         for column in column_data:
-            if column.type_text.lower().startswith("union"):
-                column.type_text = column.Type.replace(" ", "")
-            if column.type_text.lower() == "struct":
-                column.type_text = "struct<>"
+            parsed_string = {}
+            if column.type_text:
+                if column.type_text.lower().startswith("union"):
+                    column.type_text = column.type_text.replace(" ", "")
+                if (
+                    column.type_text.lower() == "struct"
+                    or column.type_text.lower() == "array"
+                ):
+                    column.type_text = column.type_text.lower() + "<>"
 
-            parsed_string = ColumnTypeParser._parse_datatype_string(  # pylint: disable=protected-access
-                column.type_text.lower()
-            )
+                parsed_string = ColumnTypeParser._parse_datatype_string(  # pylint: disable=protected-access
+                    column.type_text.lower()
+                )
             parsed_string["name"] = column.name[:256]
             parsed_string["dataLength"] = parsed_string.get("dataLength", 1)
-            parsed_string["description"] = column.comment
+            if column.comment:
+                parsed_string["description"] = Markdown(column.comment)
             parsed_column = Column(**parsed_string)
             self.add_complex_datatype_descriptions(
                 column=parsed_column,
@@ -537,3 +557,19 @@ class UnitycatalogSource(
 
     def close(self):
         """Nothing to close"""
+
+    def get_owner_ref(self, table_name: str) -> Optional[EntityReferenceList]:
+        """
+        Method to process the table owners
+        """
+        try:
+            full_table_name = f"{self.context.get().database}.{self.context.get().database_schema}.{table_name}"
+            owner = self.api_client.get_owner_info(full_table_name)
+            if not owner:
+                return
+            owner_ref = self.metadata.get_reference_by_email(email=owner)
+            return owner_ref
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error processing owner for table {table_name}: {exc}")
+        return

metadata/ingestion/source/pipeline/dagster/metadata.py

@@ -133,9 +133,11 @@ class DagsterSource(PipelineServiceSource):
             pipeline_request = CreatePipelineRequest(
                 name=EntityName(pipeline_details.id.replace(":", "")),
                 displayName=pipeline_details.name,
-                description=Markdown(pipeline_details.description)
-                if pipeline_details.description
-                else None,
+                description=(
+                    Markdown(pipeline_details.description)
+                    if pipeline_details.description
+                    else None
+                ),
                 tasks=self._get_task_list(pipeline_name=pipeline_details.name),
                 service=FullyQualifiedEntityName(self.context.get().pipeline_service),
                 tags=get_tag_labels(
@@ -178,24 +180,25 @@ class DagsterSource(PipelineServiceSource):
                 executionStatus=STATUS_MAP.get(
                     run.status.lower(), StatusType.Pending.value
                 ),
-                startTime=round(convert_timestamp_to_milliseconds(run.startTime))
-                if run.startTime
-                else None,
-                endTime=round(convert_timestamp_to_milliseconds(run.endTime))
-                if run.endTime
-                else None,
+                startTime=(
+                    round(convert_timestamp_to_milliseconds(run.startTime))
+                    if run.startTime
+                    else None
+                ),
+                endTime=(
+                    round(convert_timestamp_to_milliseconds(run.endTime))
+                    if run.endTime
+                    else None
+                ),
             )
-
             pipeline_status = PipelineStatus(
                 taskStatus=[task_status],
                 executionStatus=STATUS_MAP.get(
                     run.status.lower(), StatusType.Pending.value
                 ),
                 timestamp=Timestamp(
-                    round(convert_timestamp_to_milliseconds(run.endTime))
-                )
-                if run.endTime
-                else None,
+                    round(convert_timestamp_to_milliseconds(timestamp=run.startTime))
+                ),
             )
             pipeline_fqn = fqn.build(
                 metadata=self.metadata,

metadata/ingestion/source/pipeline/dbtcloud/client.py

@@ -26,7 +26,10 @@ from metadata.ingestion.source.pipeline.dbtcloud.models import (
     DBTRun,
     DBTRunList,
 )
-from metadata.ingestion.source.pipeline.dbtcloud.queries import DBT_QUERY
+from metadata.ingestion.source.pipeline.dbtcloud.queries import (
+    DBT_GET_MODEL_DEPENDS_ON,
+    DBT_GET_MODELS_SEEDS,
+)
 from metadata.utils.constants import AUTHORIZATION_HEADER
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ometa_logger
@@ -45,7 +48,7 @@ class DBTCloudClient:
             base_url=clean_uri(self.config.host),
             api_version="api/v2",
             auth_header=AUTHORIZATION_HEADER,
-            auth_token=lambda: (self.config.token, 0),
+            auth_token=lambda: (self.config.token.get_secret_value(), 0),
             allow_redirects=True,
         )
 
@@ -53,7 +56,7 @@ class DBTCloudClient:
             base_url=clean_uri(self.config.discoveryAPI),
             api_version="",
             auth_header=AUTHORIZATION_HEADER,
-            auth_token=lambda: (self.config.token, 0),
+            auth_token=lambda: (self.config.token.get_secret_value(), 0),
             allow_redirects=True,
         )
 
@@ -102,7 +105,7 @@ class DBTCloudClient:
         """
         try:
             query_params = {
-                "query": DBT_QUERY,
+                "query": DBT_GET_MODEL_DEPENDS_ON,
                 "variables": {"jobId": job_id, "runId": run_id},
             }
 
@@ -116,3 +119,25 @@ class DBTCloudClient:
             logger.debug(traceback.format_exc())
             logger.warning(f"Unable to get model info :{exc}")
         return None
+
+    def get_models_and_seeds_details(self, job_id: int, run_id: int):
+        """
+        get model details for a job in dbt cloud for lineage
+        """
+        try:
+            query_params = {
+                "query": DBT_GET_MODELS_SEEDS,
+                "variables": {"jobId": job_id, "runId": run_id},
+            }
+
+            result = self.graphql_client.post("", json=query_params)
+
+            if result.get("data") and result["data"].get("job"):
+                result = DBTModelList(**result["data"]["job"])
+                parents_list = result.models + result.seeds
+                return parents_list
+
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Unable to get model info :{exc}")
+        return None