openmetadata-ingestion 1.5.11.0__py3-none-any.whl → 1.6.0.0rc1__py3-none-any.whl

metadata/ingestion/source/database/glue/metadata.py

@@ -12,21 +12,24 @@
 Glue source methods.
 """
 import traceback
-from typing import Any, Iterable, Optional, Tuple, Union
+from typing import Any, Iterable, Optional, Tuple
 
 from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
 from metadata.generated.schema.api.data.createDatabaseSchema import (
     CreateDatabaseSchemaRequest,
 )
-from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
 from metadata.generated.schema.api.data.createStoredProcedure import (
     CreateStoredProcedureRequest,
 )
 from metadata.generated.schema.api.data.createTable import CreateTableRequest
-from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
-from metadata.generated.schema.entity.data.table import Column, Table, TableType
+from metadata.generated.schema.entity.data.table import (
+    Column,
+    FileFormat,
+    Table,
+    TableType,
+)
 from metadata.generated.schema.entity.services.connections.database.glueConnection import (
     GlueConnection,
 )
@@ -39,7 +42,11 @@ from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+)
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -48,6 +55,9 @@ from metadata.ingestion.source.connections import get_connection
 from metadata.ingestion.source.database.column_helpers import truncate_column_name
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.ingestion.source.database.database_service import DatabaseServiceSource
+from metadata.ingestion.source.database.external_table_lineage_mixin import (
+    ExternalTableLineageMixin,
+)
 from metadata.ingestion.source.database.glue.models import Column as GlueColumn
 from metadata.ingestion.source.database.glue.models import (
     DatabasePage,
@@ -62,7 +72,7 @@ from metadata.utils.logger import ingestion_logger
 logger = ingestion_logger()
 
 
-class GlueSource(DatabaseServiceSource):
+class GlueSource(ExternalTableLineageMixin, DatabaseServiceSource):
     """
     Implements the necessary methods to extract
     Database metadata from Glue Source
@@ -79,6 +89,8 @@ class GlueSource(DatabaseServiceSource):
         self.glue = get_connection(self.service_connection)
 
         self.connection_obj = self.glue
+        self.schema_description_map = {}
+        self.external_location_map = {}
         self.test_connection()
 
     @classmethod
@@ -191,6 +203,10 @@ class GlueSource(DatabaseServiceSource):
                     ):
                         self.status.filter(schema_fqn, "Schema Filtered Out")
                         continue
+                    if schema.Description:
+                        self.schema_description_map[schema.Name] = Markdown(
+                            schema.Description
+                        )
                     yield schema.Name
                 except Exception as exc:
                     self.status.failed(
@@ -211,6 +227,7 @@ class GlueSource(DatabaseServiceSource):
         yield Either(
             right=CreateDatabaseSchemaRequest(
                 name=EntityName(schema_name),
+                description=self.schema_description_map.get(schema_name),
                 database=FullyQualifiedEntityName(
                     fqn.build(
                         metadata=self.metadata,
@@ -295,9 +312,16 @@ class GlueSource(DatabaseServiceSource):
         table_name, table_type = table_name_and_type
         table = self.context.get().table_data
         table_constraints = None
+        storage_descriptor = table.StorageDescriptor
+        database_name = self.context.get().database
+        schema_name = self.context.get().database_schema
+        if storage_descriptor.Location:
+            # s3a doesn't occur as a path in containers, so it needs to be replaced for lineage to work
+            self.external_location_map[
+                (database_name, schema_name, table_name)
+            ] = storage_descriptor.Location.replace("s3a://", "s3://")
         try:
-            columns = self.get_columns(table.StorageDescriptor)
-
+            columns = self.get_columns(storage_descriptor)
             table_request = CreateTableRequest(
                 name=EntityName(table_name),
                 tableType=table_type,
@@ -309,8 +333,8 @@ class GlueSource(DatabaseServiceSource):
                         metadata=self.metadata,
                         entity_type=DatabaseSchema,
                         service_name=self.context.get().database_service,
-                        database_name=self.context.get().database,
-                        schema_name=self.context.get().database_schema,
+                        database_name=database_name,
+                        schema_name=schema_name,
                     )
                 ),
                 sourceUrl=self.get_source_url(
@@ -318,6 +342,8 @@ class GlueSource(DatabaseServiceSource):
                     schema_name=self.context.get().database_schema,
                     database_name=self.context.get().database,
                 ),
+                fileFormat=self.get_format(storage_descriptor),
+                locationPath=storage_descriptor.Location,
             )
             yield Either(right=table_request)
             self.register_record(table_request=table_request)
@@ -360,12 +386,22 @@ class GlueSource(DatabaseServiceSource):
         for column in self.context.get().table_data.PartitionKeys:
             yield self._get_column_object(column)
 
+    @classmethod
+    def get_format(cls, storage: StorageDetails) -> Optional[FileFormat]:
+        library = storage.SerdeInfo.SerializationLibrary
+        if library is None:
+            return None
+        if library.endswith(".LazySimpleSerDe"):
+            return (
+                FileFormat.tsv
+                if storage.SerdeInfo.Parameters.get("serialization.format") == "\t"
+                else FileFormat.csv
+            )
+        return next((fmt for fmt in FileFormat if fmt.value in library.lower()), None)
+
     def standardize_table_name(self, _: str, table: str) -> str:
         return table[:128]
 
-    def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
-        yield from []
-
     def yield_tag(
         self, schema_name: str
     ) -> Iterable[Either[OMetaTagAndClassification]]:
@@ -382,12 +418,6 @@ class GlueSource(DatabaseServiceSource):
     def get_stored_procedure_queries(self) -> Iterable[QueryByProcedure]:
         """Not Implemented"""
 
-    def yield_procedure_lineage_and_queries(
-        self,
-    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
-        """Not Implemented"""
-        yield from []
-
     def get_source_url(
         self,
         database_name: Optional[str],

metadata/ingestion/source/database/glue/models.py

@@ -20,6 +20,7 @@ from pydantic import BaseModel
 class GlueSchema(BaseModel):
     CatalogId: Optional[str] = None
     Name: str
+    Description: Optional[str] = None
 
 
 class DatabasePage(BaseModel):
@@ -36,8 +37,15 @@ class Column(BaseModel):
     Comment: Optional[str] = None
 
 
+class SerializationDetails(BaseModel):
+    SerializationLibrary: Optional[str] = None
+    Parameters: Optional[dict] = {}
+
+
 class StorageDetails(BaseModel):
     Columns: Optional[List[Column]] = []
+    Location: Optional[str] = None
+    SerdeInfo: Optional[SerializationDetails] = SerializationDetails()
 
 
 class GlueTable(BaseModel):

metadata/ingestion/source/database/glue/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.glue.metadata import GlueSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=GlueSource)

metadata/ingestion/source/database/greenplum/lineage.py

@@ -0,0 +1,46 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Greenplum lineage module
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.greenplumConnection import (
+    GreenplumConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class GreenplumLineageSource(LineageSource):
+    """
+    Greenplum lineage source implements view lineage
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        """Create class instance"""
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: GreenplumConnection = config.serviceConnection.root.config
+        if not isinstance(connection, GreenplumConnection):
+            raise InvalidSourceException(
+                f"Expected GreenplumConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/greenplum/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.greenplum.metadata import GreenplumSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=GreenplumSource)

metadata/ingestion/source/database/hive/lineage.py

@@ -0,0 +1,46 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Hive lineage module
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.hiveConnection import (
+    HiveConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class HiveLineageSource(LineageSource):
+    """
+    Hive lineage source implements view lineage
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        """Create class instance"""
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: HiveConnection = config.serviceConnection.root.config
+        if not isinstance(connection, HiveConnection):
+            raise InvalidSourceException(
+                f"Expected HiveConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/hive/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.hive.metadata import HiveSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=HiveSource)

metadata/ingestion/source/database/iceberg/metadata.py

@@ -12,7 +12,7 @@
 Iceberg source methods.
 """
 import traceback
-from typing import Any, Iterable, Optional, Tuple, Union
+from typing import Any, Iterable, Optional, Tuple
 
 import pyiceberg
 import pyiceberg.exceptions
@@ -21,12 +21,10 @@ from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequ
 from metadata.generated.schema.api.data.createDatabaseSchema import (
     CreateDatabaseSchemaRequest,
 )
-from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
 from metadata.generated.schema.api.data.createStoredProcedure import (
     CreateStoredProcedureRequest,
 )
 from metadata.generated.schema.api.data.createTable import CreateTableRequest
-from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.table import Table, TableType
@@ -293,13 +291,6 @@ class IcebergSource(DatabaseServiceSource):
         """
         yield from []
 
-    def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
-        """
-        From topology.
-        Parses view definition to get lineage information
-        """
-        yield from []
-
     def get_stored_procedures(self) -> Iterable[Any]:
         """Not Implemented"""
 
@@ -309,11 +300,5 @@ class IcebergSource(DatabaseServiceSource):
         """Process the stored procedure information"""
         yield from []
 
-    def yield_procedure_lineage_and_queries(
-        self,
-    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
-        """Extracts the lineage information from Stored Procedures"""
-        yield from []
-
     def close(self):
         """There is no connection to close."""

metadata/ingestion/source/database/iceberg/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.iceberg.metadata import IcebergSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=IcebergSource)

metadata/ingestion/source/database/impala/lineage.py

@@ -0,0 +1,46 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Impala lineage module
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.impalaConnection import (
+    ImpalaConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class ImpalaLineageSource(LineageSource):
+    """
+    Impala lineage source implements view lineage
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        """Create class instance"""
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: ImpalaConnection = config.serviceConnection.root.config
+        if not isinstance(connection, ImpalaConnection):
+            raise InvalidSourceException(
+                f"Expected ImpalaConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/impala/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.impala.metadata import ImpalaSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=ImpalaSource)

metadata/ingestion/source/database/life_cycle_query_mixin.py

@@ -36,7 +36,7 @@ from metadata.ingestion.models.topology import TopologyContextManager
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils import fqn
 from metadata.utils.logger import ingestion_logger
-from metadata.utils.time_utils import convert_timestamp_to_milliseconds
+from metadata.utils.time_utils import datetime_to_timestamp
 
 logger = ingestion_logger()
 
@@ -104,10 +104,8 @@ class LifeCycleQueryMixin:
                 life_cycle = LifeCycle(
                     created=AccessDetails(
                         timestamp=Timestamp(
-                            int(
-                                convert_timestamp_to_milliseconds(
-                                    life_cycle_data.created_at.timestamp()
-                                )
+                            datetime_to_timestamp(
+                                life_cycle_data.created_at, milliseconds=True
                             )
                         )
                     )

metadata/ingestion/source/database/lineage_source.py

@@ -15,17 +15,22 @@ import csv
 import os
 import traceback
 from abc import ABC
-from typing import Iterable, Iterator, Union
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import partial
+from typing import Callable, Iterable, Iterator, Union
 
 from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.type.basic import FullyQualifiedEntityName, SqlQuery
 from metadata.generated.schema.type.tableQuery import TableQuery
 from metadata.ingestion.api.models import Either
-from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
+from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
 from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query
+from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
 from metadata.ingestion.source.database.query_parser_source import QueryParserSource
+from metadata.ingestion.source.models import TableView
 from metadata.utils import fqn
+from metadata.utils.db_utils import get_view_lineage
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -43,12 +48,14 @@ class LineageSource(QueryParserSource, ABC):
     - schema
     """
 
+    dialect: Dialect
+
     def yield_table_queries_from_logs(self) -> Iterator[TableQuery]:
         """
         Method to handle the usage from query logs
         """
         try:
-            query_log_path = self.config.sourceConfig.config.queryLogFilePath
+            query_log_path = self.source_config.queryLogFilePath
             if os.path.isfile(query_log_path):
                 file_paths = [query_log_path]
             elif os.path.isdir(query_log_path):
@@ -89,6 +96,28 @@ class LineageSource(QueryParserSource, ABC):
             )
             yield from self.yield_table_query()
 
+    def generate_lineage_in_thread(self, producer_fn: Callable, processor_fn: Callable):
+        with ThreadPoolExecutor(max_workers=self.source_config.threads) as executor:
+            futures = []
+
+            for produced_input in producer_fn():
+                futures.append(executor.submit(processor_fn, produced_input))
+
+            # Handle remaining futures after the loop
+            for future in as_completed(
+                futures, timeout=self.source_config.parsingTimeoutLimit
+            ):
+                try:
+                    results = future.result(
+                        timeout=self.source_config.parsingTimeoutLimit
+                    )
+                    yield from results
+                except Exception as exc:
+                    logger.debug(traceback.format_exc())
+                    logger.warning(
+                        f"Error processing result for {produced_input}: {exc}"
+                    )
+
     def yield_table_query(self) -> Iterator[TableQuery]:
         """
         Given an engine, iterate over the query results to
@@ -128,40 +157,103 @@ class LineageSource(QueryParserSource, ABC):
         )
         return fqn.get_query_checksum(table_query.query) in checksums or {}
 
-    def _iter(
-        self, *_, **__
+    def query_lineage_generator(
+        self, table_query: TableQuery
+    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
+        if not self._query_already_processed(table_query):
+            lineages: Iterable[Either[AddLineageRequest]] = get_lineage_by_query(
+                self.metadata,
+                query=table_query.query,
+                service_name=table_query.serviceName,
+                database_name=table_query.databaseName,
+                schema_name=table_query.databaseSchema,
+                dialect=self.dialect,
+                timeout_seconds=self.source_config.parsingTimeoutLimit,
+            )
+
+            for lineage_request in lineages or []:
+                yield lineage_request
+
+                # If we identified lineage properly, ingest the original query
+                if lineage_request.right:
+                    yield Either(
+                        right=CreateQueryRequest(
+                            query=SqlQuery(table_query.query),
+                            query_type=table_query.query_type,
+                            duration=table_query.duration,
+                            processedLineage=True,
+                            service=FullyQualifiedEntityName(self.config.serviceName),
+                        )
+                    )
+
+    def yield_query_lineage(
+        self,
     ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
         """
         Based on the query logs, prepare the lineage
         and send it to the sink
         """
         connection_type = str(self.service_connection.type.value)
-        dialect = ConnectionTypeDialectMapper.dialect_of(connection_type)
-        for table_query in self.get_table_query():
-            if not self._query_already_processed(table_query):
-                lineages: Iterable[Either[AddLineageRequest]] = get_lineage_by_query(
-                    self.metadata,
-                    query=table_query.query,
-                    service_name=table_query.serviceName,
-                    database_name=table_query.databaseName,
-                    schema_name=table_query.databaseSchema,
-                    dialect=dialect,
-                    timeout_seconds=self.source_config.parsingTimeoutLimit,
-                )
+        self.dialect = ConnectionTypeDialectMapper.dialect_of(connection_type)
+        producer_fn = self.get_table_query
+        processor_fn = self.query_lineage_generator
+        yield from self.generate_lineage_in_thread(producer_fn, processor_fn)
 
-                for lineage_request in lineages or []:
-                    yield lineage_request
-
-                    # If we identified lineage properly, ingest the original query
-                    if lineage_request.right:
-                        yield Either(
-                            right=CreateQueryRequest(
-                                query=SqlQuery(table_query.query),
-                                query_type=table_query.query_type,
-                                duration=table_query.duration,
-                                processedLineage=True,
-                                service=FullyQualifiedEntityName(
-                                    self.config.serviceName
-                                ),
-                            )
+    def view_lineage_generator(
+        self, view: TableView
+    ) -> Iterable[Either[AddLineageRequest]]:
+        try:
+            for lineage in get_view_lineage(
+                view=view,
+                metadata=self.metadata,
+                service_name=self.config.serviceName,
+                connection_type=self.service_connection.type.value,
+                timeout_seconds=self.source_config.parsingTimeoutLimit,
+            ):
+                if lineage.right is not None:
+                    yield Either(
+                        right=OMetaLineageRequest(
+                            lineage_request=lineage.right,
+                            override_lineage=self.source_config.overrideViewLineage,
                         )
+                    )
+                else:
+                    yield lineage
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error processing view {view}: {exc}")
+
+    def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
+        logger.info("Processing View Lineage")
+        producer_fn = partial(self.metadata.yield_es_view_def, self.config.serviceName)
+        processor_fn = self.view_lineage_generator
+        yield from self.generate_lineage_in_thread(producer_fn, processor_fn)
+
+    def yield_procedure_lineage(
+        self,
+    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
+        """
+        By default stored procedure lineage is not supported.
+        """
+        logger.info(
+            f"Processing Procedure Lineage not supported for {str(self.service_connection.type.value)}"
+        )
+
+    def _iter(
+        self, *_, **__
+    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
+        """
+        Based on the query logs, prepare the lineage
+        and send it to the sink
+        """
+        if self.source_config.processViewLineage:
+            yield from self.yield_view_lineage() or []
+        if self.source_config.processStoredProcedureLineage:
+            yield from self.yield_procedure_lineage() or []
+        if self.source_config.processQueryLineage:
+            if hasattr(self.service_connection, "supportsLineageExtraction"):
+                yield from self.yield_query_lineage() or []
+            else:
+                logger.warning(
+                    f"Lineage extraction is not supported for {str(self.service_connection.type.value)} connection"
+                )