openmetadata-ingestion 1.5.11.0__py3-none-any.whl → 1.6.0.0rc1__py3-none-any.whl

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -11,6 +11,7 @@
 """
 Mixin class with common Stored Procedures logic aimed at lineage.
 """
+import json
 import re
 import traceback
 from abc import ABC, abstractmethod
@@ -27,8 +28,8 @@ from metadata.generated.schema.entity.data.storedProcedure import StoredProcedur
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
-from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import (
-    DatabaseServiceMetadataPipeline,
+from metadata.generated.schema.metadataIngestion.databaseServiceQueryLineagePipeline import (
+    DatabaseServiceQueryLineagePipeline,
 )
 from metadata.generated.schema.type.basic import SqlQuery, Timestamp
 from metadata.generated.schema.type.entityLineage import Source as LineageSource
@@ -37,7 +38,6 @@ from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.status import Status
 from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
 from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query
-from metadata.ingestion.models.topology import TopologyContextManager
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.stored_procedures import get_procedure_name_from_call
@@ -66,7 +66,18 @@ class QueryByProcedure(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
 
-class StoredProcedureMixin(ABC):
+class ProcedureAndQuery(BaseModel):
+    """
+    Model to hold the procedure and its queries
+    """
+
+    procedure: StoredProcedure
+    query_by_procedure: QueryByProcedure
+
+    model_config = ConfigDict(populate_by_name=True)
+
+
+class StoredProcedureLineageMixin(ABC):
     """
     The full flow is:
     1. List Stored Procedures
@@ -80,10 +91,10 @@ class StoredProcedureMixin(ABC):
     It should be inherited in those Sources that implement Stored Procedure ingestion.
     """
 
-    context: TopologyContextManager
     status: Status
-    source_config: DatabaseServiceMetadataPipeline
+    source_config: DatabaseServiceQueryLineagePipeline
     engine: Engine
+    stored_procedure_query_lineage: bool
     metadata: OpenMetadata
 
     @abstractmethod
@@ -143,29 +154,34 @@ class StoredProcedureMixin(ABC):
 
         return False
 
-    def yield_procedure_lineage(
+    def _yield_procedure_lineage(
         self, query_by_procedure: QueryByProcedure, procedure: StoredProcedure
     ) -> Iterable[Either[AddLineageRequest]]:
         """Add procedure lineage from its query"""
-        self.context.get().stored_procedure_query_lineage = False
+        self.stored_procedure_query_lineage = False
         if self.is_lineage_query(
             query_type=query_by_procedure.query_type,
             query_text=query_by_procedure.query_text,
         ):
-            self.context.get().stored_procedure_query_lineage = True
+            self.stored_procedure_query_lineage = True
             for either_lineage in get_lineage_by_query(
                 self.metadata,
                 query=query_by_procedure.query_text,
-                service_name=self.context.get().database_service,
+                service_name=self.service_name,
                 database_name=query_by_procedure.query_database_name,
                 schema_name=query_by_procedure.query_schema_name,
                 dialect=ConnectionTypeDialectMapper.dialect_of(
                     self.service_connection.type.value
                 ),
-                timeout_seconds=self.source_config.queryParsingTimeoutLimit,
+                timeout_seconds=self.source_config.parsingTimeoutLimit,
                 lineage_source=LineageSource.QueryLineage,
             ):
-                if either_lineage.right.edge.lineageDetails:
+                print("&& " * 100)
+                print(either_lineage)
+                if (
+                    either_lineage.left is None
+                    and either_lineage.right.edge.lineageDetails
+                ):
                     either_lineage.right.edge.lineageDetails.pipeline = EntityReference(
                         id=procedure.id,
                         type="storedProcedure",
@@ -192,34 +208,59 @@ class StoredProcedureMixin(ABC):
                     id=procedure.id,
                     type="storedProcedure",
                 ),
-                processedLineage=bool(
-                    self.context.get().stored_procedure_query_lineage
-                ),
-                service=self.context.get().database_service,
+                processedLineage=bool(self.stored_procedure_query_lineage),
+                service=self.service_name,
             )
         )
 
-    def yield_procedure_lineage_and_queries(
+    def procedure_lineage_processor(
+        self, procedure_and_query: ProcedureAndQuery
+    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
+
+        yield from self._yield_procedure_lineage(
+            query_by_procedure=procedure_and_query.query_by_procedure,
+            procedure=procedure_and_query.procedure,
+        )
+        yield from self.yield_procedure_query(
+            query_by_procedure=procedure_and_query.query_by_procedure,
+            procedure=procedure_and_query.procedure,
+        )
+
+    def procedure_lineage_generator(self) -> Iterable[ProcedureAndQuery]:
+        query = {
+            "query": {
+                "bool": {
+                    "must": [
+                        {"term": {"service.name.keyword": self.service_name}},
+                        {"term": {"deleted": False}},
+                    ]
+                }
+            }
+        }
+        query_filter = json.dumps(query)
+
+        logger.info("Processing Lineage for Stored Procedures")
+        # First, get all the query history
+        queries_dict = self.get_stored_procedure_queries_dict()
+        # Then for each procedure, iterate over all its queries
+        for procedure in (
+            self.metadata.paginate_es(entity=StoredProcedure, query_filter=query_filter)
+            or []
+        ):
+            if procedure:
+                logger.debug(f"Processing Lineage for [{procedure.name}]")
+                for query_by_procedure in (
+                    queries_dict.get(procedure.name.root.lower()) or []
+                ):
+                    yield ProcedureAndQuery(
+                        procedure=procedure, query_by_procedure=query_by_procedure
+                    )
+
+    def yield_procedure_lineage(
         self,
     ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
         """Get all the queries and procedures list and yield them"""
-        if self.context.get().stored_procedures:
-            logger.info("Processing Lineage for Stored Procedures")
-            # First, get all the query history
-            queries_dict = self.get_stored_procedure_queries_dict()
-            # Then for each procedure, iterate over all its queries
-            for procedure_fqn in self.context.get().stored_procedures:
-                procedure: StoredProcedure = self.metadata.get_by_name(
-                    entity=StoredProcedure, fqn=procedure_fqn
-                )
-                if procedure:
-                    logger.debug(f"Processing Lineage for [{procedure.name}]")
-                    for query_by_procedure in (
-                        queries_dict.get(procedure.name.root.lower()) or []
-                    ):
-                        yield from self.yield_procedure_lineage(
-                            query_by_procedure=query_by_procedure, procedure=procedure
-                        )
-                        yield from self.yield_procedure_query(
-                            query_by_procedure=query_by_procedure, procedure=procedure
-                        )
+        logger.info("Processing Lineage for Stored Procedures")
+        producer_fn = self.procedure_lineage_generator
+        processor_fn = self.procedure_lineage_processor
+        yield from self.generate_lineage_in_thread(producer_fn, processor_fn)

metadata/ingestion/source/database/teradata/lineage.py

@@ -0,0 +1,46 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Teradata lineage module
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.teradataConnection import (
+    TeradataConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class TeradataLineageSource(LineageSource):
+    """
+    Teradata lineage source implements view lineage
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        """Create class instance"""
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: TeradataConnection = config.serviceConnection.root.config
+        if not isinstance(connection, TeradataConnection):
+            raise InvalidSourceException(
+                f"Expected TeradataConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/teradata/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.teradata.metadata import TeradataSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=TeradataSource)

metadata/ingestion/source/database/trino/connection.py

@@ -12,11 +12,13 @@
 """
 Source connection handler
 """
+from copy import deepcopy
 from typing import Optional
 from urllib.parse import quote_plus
 
 from requests import Session
 from sqlalchemy.engine import Engine
+from trino.auth import BasicAuthentication, JWTAuthentication, OAuth2Authentication
 
 from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
@@ -25,6 +27,7 @@ from metadata.generated.schema.entity.automations.workflow import (
 from metadata.generated.schema.entity.services.connections.database.common import (
     basicAuth,
     jwtAuth,
+    noConfigAuthenticationTypes,
 )
 from metadata.generated.schema.entity.services.connections.database.trinoConnection import (
     TrinoConnection,
@@ -36,7 +39,6 @@ from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
     init_empty_connection_arguments,
-    init_empty_connection_options,
 )
 from metadata.ingestion.connections.secrets import connection_with_options_secrets
 from metadata.ingestion.connections.test_connections import (
@@ -52,26 +54,17 @@ def get_connection_url(connection: TrinoConnection) -> str:
     Prepare the connection url for trino
     """
     url = f"{connection.scheme.value}://"
+
+    # leaving username here as, even though with basic auth is used directly
+    # in BasicAuthentication class, it's often also required as a part of url.
+    # For example - it will be used by OAuth2Authentication to persist token in
+    # cache more efficiently (per user instead of per host)
     if connection.username:
-        # we need to encode twice because trino dialect internally
-        # url decodes the username and if there is an special char in username
-        # it will fail to authenticate
-        url += f"{quote_plus(quote_plus(connection.username))}"
-        if (
-            isinstance(connection.authType, basicAuth.BasicAuth)
-            and connection.authType.password
-        ):
-            url += f":{quote_plus(connection.authType.password.get_secret_value())}"
-        url += "@"
+        url += f"{quote_plus(connection.username)}@"
+
     url += f"{connection.hostPort}"
     if connection.catalog:
         url += f"/{connection.catalog}"
-    if isinstance(connection.authType, jwtAuth.JwtAuth):
-        if not connection.connectionOptions:
-            connection.connectionOptions = init_empty_connection_options()
-        connection.connectionOptions.root[
-            "access_token"
-        ] = connection.authType.jwt.get_secret_value()
     if connection.connectionOptions is not None:
         params = "&".join(
             f"{key}={quote_plus(value)}"
@@ -84,14 +77,54 @@ def get_connection_url(connection: TrinoConnection) -> str:
 
 @connection_with_options_secrets
 def get_connection_args(connection: TrinoConnection):
+    if not connection.connectionArguments:
+        connection.connectionArguments = init_empty_connection_arguments()
+
     if connection.proxies:
         session = Session()
         session.proxies = connection.proxies
-        if not connection.connectionArguments:
-            connection.connectionArguments = init_empty_connection_arguments()
 
         connection.connectionArguments.root["http_session"] = session
 
+    if isinstance(connection.authType, basicAuth.BasicAuth):
+        connection.connectionArguments.root["auth"] = BasicAuthentication(
+            connection.username,
+            connection.authType.password.get_secret_value()
+            if connection.authType.password
+            else None,
+        )
+        connection.connectionArguments.root["http_scheme"] = "https"
+
+    elif isinstance(connection.authType, jwtAuth.JwtAuth):
+        connection.connectionArguments.root["auth"] = JWTAuthentication(
+            connection.authType.jwt.get_secret_value()
+        )
+        connection.connectionArguments.root["http_scheme"] = "https"
+
+    elif hasattr(connection.authType, "azureConfig"):
+        if not connection.authType.azureConfig.scopes:
+            raise ValueError(
+                "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default"
+            )
+
+        azure_client = AzureClient(connection.authType.azureConfig).create_client()
+
+        access_token_obj = azure_client.get_token(
+            *connection.authType.azureConfig.scopes.split(",")
+        )
+
+        connection.connectionArguments.root["auth"] = JWTAuthentication(
+            access_token_obj.token
+        )
+        connection.connectionArguments.root["http_scheme"] = "https"
+
+    elif (
+        connection.authType
+        == noConfigAuthenticationTypes.NoConfigAuthenticationTypes.OAuth2
+    ):
+        connection.connectionArguments.root["auth"] = OAuth2Authentication()
+        connection.connectionArguments.root["http_scheme"] = "https"
+
     return get_connection_args_common(connection)
 
 
@@ -99,9 +132,13 @@ def get_connection(connection: TrinoConnection) -> Engine:
     """
     Create connection
     """
-    if connection.verify:
-        connection.connectionArguments = (
-            connection.connectionArguments or init_empty_connection_arguments()
+    # here we are creating a copy of connection, because we need to dynamically
+    # add auth params to connectionArguments, which we do no intend to store
+    # in original connection object and in OpenMetadata database
+    connection_copy = deepcopy(connection)
+    if connection_copy.verify:
+        connection_copy.connectionArguments = (
+            connection_copy.connectionArguments or init_empty_connection_arguments()
         )
         connection.connectionArguments.root["verify"] = {"verify": connection.verify}
     if hasattr(connection.authType, "azureConfig"):
@@ -117,7 +154,7 @@ def get_connection(connection: TrinoConnection) -> Engine:
             connection.connectionOptions = init_empty_connection_options()
         connection.connectionOptions.root["access_token"] = access_token_obj.token
     return create_generic_db_connection(
-        connection=connection,
+        connection=connection_copy,
         get_connection_url_fn=get_connection_url,
         get_connection_args_fn=get_connection_args,
     )

metadata/ingestion/source/database/trino/profiler/system_tables_profiler.py

@@ -0,0 +1,167 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+System table profiler
+"""
+from datetime import datetime
+from decimal import Decimal
+from typing import Any, Dict, List, Optional, Set, Union
+
+from more_itertools import partition
+from pydantic import field_validator
+from sqlalchemy import Table, text
+from sqlalchemy.engine import Engine
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+from metadata.profiler.interface.sqlalchemy.stored_statistics_profiler import (
+    StoredStatisticsSource,
+)
+from metadata.profiler.metrics.core import Metric
+from metadata.profiler.metrics.registry import Metrics
+from metadata.utils.logger import profiler_logger
+from metadata.utils.lru_cache import LRU_CACHE_SIZE, LRUCache
+from metadata.utils.ssl_manager import get_ssl_connection
+
+logger = profiler_logger()
+
+
+class ColumnStats(BaseModel):
+    """Based on https://trino.io/docs/current/sql/show-stats.html"""
+
+    column_name: Optional[str] = None
+    data_size: Optional[int] = None
+    distinct_values_count: Optional[int] = None
+    nulls_fraction: Optional[float] = None
+    low_value: Optional[Union[int, float, datetime, Decimal]] = None
+    high_value: Optional[Union[int, float, datetime, Decimal]] = None
+
+    @field_validator("data_size", mode="before")
+    @classmethod
+    def data_size_validator(cls, value):
+        """Data size validator
+
+        Args:
+            value: value
+        """
+        if value is None:
+            return None
+        return int(value)
+
+
+class TableStats(BaseModel):
+    row_count: Optional[int] = None
+    columns: Dict[str, ColumnStats] = {}
+
+
+class TrinoStoredStatisticsSource(StoredStatisticsSource):
+    """Trino system profile source"""
+
+    metric_stats_map: Dict[Metrics, str] = {
+        Metrics.NULL_RATIO: "nulls_fractions",
+        Metrics.DISTINCT_COUNT: "distinct_values_count",
+        Metrics.ROW_COUNT: "row_count",
+        Metrics.MAX: "high_value",
+        Metrics.MIN: "low_value",
+    }
+
+    metric_stats_by_name: Dict[str, str] = {
+        k.name: v for k, v in metric_stats_map.items()
+    }
+
+    def get_statistics_metrics(self) -> Set[Metrics]:
+        return set(self.metric_stats_map.keys())
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        service_connection_config = kwargs["service_connection_config"]
+        self.session: Engine = get_ssl_connection(service_connection_config)
+        self.stats_cache = LRUCache(capacity=LRU_CACHE_SIZE)
+
+    def get_column_statistics(
+        self, metric: List[Metric], schema: str, table_name: Table, column: str
+    ) -> Dict[str, Any]:
+        table_stats = self._get_cached_stats(schema, table_name)
+        try:
+            column_stats = table_stats.columns[column]
+        except KeyError:
+            raise RuntimeError(
+                f"Column {column} not found in table {table_name}. Statistics might be stale or missing."
+            )
+        result = {
+            m.name(): getattr(column_stats, self.metric_stats_by_name[m.name()])
+            for m in metric
+        }
+        result.update(self.get_hybrid_statistics(table_stats, column_stats))
+        self.warn_for_missing_stats(schema, table_name, column_stats)
+        return result
+
+    def get_table_statistics(
+        self, metric: List[Metric], schema: str, table_name: Table
+    ) -> dict:
+        table_stats = self._get_cached_stats(schema, table_name)
+        return {
+            m.name(): getattr(table_stats, self.metric_stats_by_name[m.name()])
+            for m in metric
+        }
+
+    def warn_for_missing_stats(self, schema: str, table: str, stats: BaseModel):
+        if (
+            isinstance(stats, ColumnStats)
+            and all(map(lambda x: x is None, stats.model_dump().values()))
+        ) or (
+            isinstance(stats, TableStats)
+            and all(
+                map(
+                    lambda x: x is None,
+                    [v for k, v in stats.model_dump().items() if k != "columns"],
+                )
+            )
+        ):
+            logger.warning(
+                'Statistics are missing for table "{schema}.{table}". Profiling might be inaccurate.\n'
+                "Gather statistics for the table by running:"
+                f"  ANALYZE {schema}.{table}"
+            )
+
+    def _get_cached_stats(self, schema: str, table: str) -> TableStats:
+        path = f"{schema}.{table}"
+        if path in self.stats_cache:
+            return self.stats_cache.get(path)
+        stats = self._get_db_stats(schema, table)
+        self.stats_cache.put(path, stats)
+        return stats
+
+    def _get_db_stats(self, schema, table) -> TableStats:
+        rows = self.session.execute(text(f'SHOW STATS FOR "{schema}"."{table}"'))
+        table_rows, column_rows = map(
+            list, partition(lambda row: row.get("column_name"), map(dict, rows))
+        )
+        if len(table_rows) != 1:
+            raise RuntimeError(
+                f"Expected one row for table {table}, got {len(table_rows)}"
+            )
+        table = table_rows[0]
+        columns_dict = {
+            row.get("column_name"): ColumnStats(**row) for row in column_rows
+        }
+        return TableStats(row_count=table["row_count"], columns=columns_dict)
+
+    def get_hybrid_statistics(
+        self, table_stats: TableStats, column_stats: ColumnStats
+    ) -> Dict[str, Any]:
+        return {
+            # trino stats are in fractions, so we need to convert them to counts (unlike our default profiler)
+            Metrics.NULL_COUNT.name: (
+                int(table_stats.row_count * column_stats.nulls_fraction)
+                if None not in [table_stats.row_count, column_stats.nulls_fraction]
+                else None
+            ),
+        }

metadata/ingestion/source/database/trino/service_spec.py

@@ -0,0 +1,14 @@
+from metadata.ingestion.source.database.trino.lineage import TrinoLineageSource
+from metadata.ingestion.source.database.trino.metadata import TrinoSource
+from metadata.ingestion.source.database.trino.usage import TrinoUsageSource
+from metadata.profiler.interface.sqlalchemy.trino.profiler_interface import (
+    TrinoProfilerInterface,
+)
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(
+    metadata_source_class=TrinoSource,
+    lineage_source_class=TrinoLineageSource,
+    usage_source_class=TrinoUsageSource,
+    profiler_class=TrinoProfilerInterface,
+)

metadata/ingestion/source/database/unitycatalog/metadata.py

@@ -13,7 +13,7 @@ Databricks Unity Catalog Source source methods.
 """
 import json
 import traceback
-from typing import Any, Iterable, List, Optional, Tuple, Union
+from typing import Any, Iterable, List, Optional, Tuple
 
 from databricks.sdk.service.catalog import ColumnInfo
 from databricks.sdk.service.catalog import TableConstraint as DBTableConstraint
@@ -22,12 +22,10 @@ from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequ
 from metadata.generated.schema.api.data.createDatabaseSchema import (
     CreateDatabaseSchemaRequest,
 )
-from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
 from metadata.generated.schema.api.data.createStoredProcedure import (
     CreateStoredProcedureRequest,
 )
 from metadata.generated.schema.api.data.createTable import CreateTableRequest
-from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.table import (
@@ -76,7 +74,6 @@ from metadata.ingestion.source.database.unitycatalog.models import (
 )
 from metadata.ingestion.source.models import TableView
 from metadata.utils import fqn
-from metadata.utils.db_utils import get_view_lineage
 from metadata.utils.filters import filter_by_database, filter_by_schema, filter_by_table
 from metadata.utils.logger import ingestion_logger
 
@@ -325,7 +322,7 @@ class UnitycatalogSource(
             ) = self.get_table_constraints(table.table_constraints)
 
             table_constraints = self.update_table_constraints(
-                primary_constraints, foreign_constraints
+                primary_constraints, foreign_constraints, columns
             )
 
             table_request = CreateTableRequest(
@@ -436,7 +433,7 @@ class UnitycatalogSource(
         return table_constraints
 
     def update_table_constraints(
-        self, table_constraints, foreign_columns
+        self, table_constraints, foreign_columns, columns
     ) -> List[TableConstraint]:
         """
         From topology.
@@ -521,18 +518,6 @@ class UnitycatalogSource(
             )
             yield parsed_column
 
-    def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
-        logger.info("Processing Lineage for Views")
-        for view in [
-            v for v in self.context.get().table_views if v.view_definition is not None
-        ]:
-            yield from get_view_lineage(
-                view=view,
-                metadata=self.metadata,
-                service_name=self.context.get().database_service,
-                connection_type=self.service_connection.type.value,
-            )
-
     def yield_tag(
         self, schema_name: str
     ) -> Iterable[Either[OMetaTagAndClassification]]:
@@ -549,12 +534,6 @@ class UnitycatalogSource(
     def get_stored_procedure_queries(self) -> Iterable[QueryByProcedure]:
         """Not Implemented"""
 
-    def yield_procedure_lineage_and_queries(
-        self,
-    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
-        """Not Implemented"""
-        yield from []
-
     def close(self):
         """Nothing to close"""
 

metadata/ingestion/source/database/unitycatalog/service_spec.py

@@ -0,0 +1,18 @@
+from metadata.ingestion.source.database.unitycatalog.lineage import (
+    UnitycatalogLineageSource,
+)
+from metadata.ingestion.source.database.unitycatalog.metadata import UnitycatalogSource
+from metadata.ingestion.source.database.unitycatalog.usage import (
+    UnitycatalogUsageSource,
+)
+from metadata.profiler.interface.sqlalchemy.unity_catalog.profiler_interface import (
+    UnityCatalogProfilerInterface,
+)
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(
+    metadata_source_class=UnitycatalogSource,
+    lineage_source_class=UnitycatalogLineageSource,
+    usage_source_class=UnitycatalogUsageSource,
+    profiler_class=UnityCatalogProfilerInterface,
+)

metadata/ingestion/source/database/usage_source.py

@@ -152,7 +152,10 @@ class UsageSource(QueryParserSource, ABC):
             except Exception as exc:
                 if query:
                     logger.debug(
-                        f"###### USAGE QUERY #######\n{mask_query(query, self.dialect.value)}\n##########################"
+                        (
+                            f"###### USAGE QUERY #######\n{mask_query(query, self.dialect.value)}"
+                            "\n##########################"
+                        )
                     )
                 logger.debug(traceback.format_exc())
                 logger.error(f"Source usage processing error: {exc}")