openmetadata-ingestion 1.5.13.0__py3-none-any.whl → 1.6.0.0rc2__py3-none-any.whl

metadata/ingestion/source/database/dbt/models.py

@@ -20,12 +20,14 @@ from pydantic import BaseModel
 class DbtFiles(BaseModel):
     dbt_catalog: Optional[dict] = None
     dbt_manifest: dict
+    dbt_sources: Optional[dict] = None
     dbt_run_results: Optional[List[dict]] = None
 
 
 class DbtObjects(BaseModel):
     dbt_catalog: Optional[Any] = None
     dbt_manifest: Any
+    dbt_sources: Optional[Any] = None
     dbt_run_results: Optional[List[Any]] = None
 
 

metadata/ingestion/source/database/dbt/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.dbt.metadata import DbtSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=DbtSource)

metadata/ingestion/source/database/deltalake/metadata.py

@@ -12,18 +12,16 @@
 Deltalake source methods.
 """
 import traceback
-from typing import Any, Iterable, Optional, Tuple, Union
+from typing import Any, Iterable, Optional, Tuple
 
 from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
 from metadata.generated.schema.api.data.createDatabaseSchema import (
     CreateDatabaseSchemaRequest,
 )
-from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
 from metadata.generated.schema.api.data.createStoredProcedure import (
     CreateStoredProcedureRequest,
 )
 from metadata.generated.schema.api.data.createTable import CreateTableRequest
-from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.table import Table, TablePartition, TableType
@@ -283,9 +281,6 @@ class DeltalakeSource(DatabaseServiceSource):
     def prepare(self):
         """Nothing to prepare"""
 
-    def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
-        yield from []
-
     def yield_tag(
         self, schema_name: str
     ) -> Iterable[Either[OMetaTagAndClassification]]:
@@ -302,11 +297,5 @@ class DeltalakeSource(DatabaseServiceSource):
     def get_stored_procedure_queries(self) -> Iterable[QueryByProcedure]:
         """Not Implemented"""
 
-    def yield_procedure_lineage_and_queries(
-        self,
-    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
-        """Not Implemented"""
-        yield from []
-
     def close(self):
         """No client to close"""

metadata/ingestion/source/database/deltalake/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.deltalake.metadata import DeltalakeSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=DeltalakeSource)

metadata/ingestion/source/database/domodatabase/metadata.py

@@ -14,18 +14,16 @@ Domo Database source to extract metadata
 """
 
 import traceback
-from typing import Any, Iterable, Optional, Tuple, Union
+from typing import Any, Iterable, Optional, Tuple
 
 from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
 from metadata.generated.schema.api.data.createDatabaseSchema import (
     CreateDatabaseSchemaRequest,
 )
-from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
 from metadata.generated.schema.api.data.createStoredProcedure import (
     CreateStoredProcedureRequest,
 )
 from metadata.generated.schema.api.data.createTable import CreateTableRequest
-from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.table import (
@@ -294,15 +292,6 @@ class DomodatabaseSource(DatabaseServiceSource):
     def get_stored_procedure_queries(self) -> Iterable[QueryByProcedure]:
         """Not Implemented"""
 
-    def yield_procedure_lineage_and_queries(
-        self,
-    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
-        """Not Implemented"""
-        yield from []
-
-    def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
-        yield from []
-
     def get_source_url(
         self,
         table_name: Optional[str] = None,

metadata/ingestion/source/database/domodatabase/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.domodatabase.metadata import DomodatabaseSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=DomodatabaseSource)

metadata/ingestion/source/database/doris/metadata.py

@@ -8,7 +8,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-"""Mysql source module"""
+"""Doris source module"""
 import re
 import traceback
 from typing import Dict, Iterable, List, Optional, Tuple, cast

metadata/ingestion/source/database/doris/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.doris.metadata import DorisSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=DorisSource)

metadata/ingestion/source/database/druid/lineage.py

@@ -0,0 +1,46 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Druid lineage module
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.druidConnection import (
+    DruidConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class DruidLineageSource(LineageSource):
+    """
+    Druid lineage source implements view lineage
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        """Create class instance"""
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: DruidConnection = config.serviceConnection.root.config
+        if not isinstance(connection, DruidConnection):
+            raise InvalidSourceException(
+                f"Expected DruidConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/druid/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.druid.metadata import DruidSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=DruidSource)

metadata/ingestion/source/database/dynamodb/service_spec.py

@@ -0,0 +1,10 @@
+from metadata.ingestion.source.database.dynamodb.metadata import DynamodbSource
+from metadata.profiler.interface.nosql.profiler_interface import NoSQLProfilerInterface
+from metadata.sampler.nosql.sampler import NoSQLSampler
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(
+    metadata_source_class=DynamodbSource,
+    profiler_class=NoSQLProfilerInterface,
+    sampler_class=NoSQLSampler,
+)

metadata/ingestion/source/database/exasol/connection.py

@@ -0,0 +1,87 @@
+from typing import Optional
+from urllib.parse import quote_plus
+
+from pydantic import SecretStr
+from sqlalchemy.engine import Engine
+
+from metadata.generated.schema.entity.automations.workflow import (
+    Workflow as AutomationWorkflow,
+)
+from metadata.generated.schema.entity.services.connections.database.exasolConnection import (
+    ExasolConnection,
+)
+from metadata.ingestion.connections.builders import (
+    create_generic_db_connection,
+    get_connection_args_common,
+)
+from metadata.ingestion.connections.test_connections import test_query
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+def get_connection_url(connection: ExasolConnection) -> str:
+    """
+    Common method for building the source connection urls
+    """
+
+    url = f"{connection.scheme.value}://"
+
+    if connection.username:
+        url += f"{quote_plus(connection.username)}"
+        connection.password = (
+            SecretStr("") if not connection.password else connection.password
+        )
+        url += (
+            f":{quote_plus(connection.password.get_secret_value())}"
+            if connection
+            else ""
+        )
+        url += "@"
+
+    url += connection.hostPort
+
+    if hasattr(connection, "databaseSchema"):
+        url += f"/{connection.databaseSchema}" if connection.databaseSchema else ""
+
+    tls_settings = {
+        "validate-certificate": {},
+        "ignore-certificate": {"SSLCertificate": "SSL_VERIFY_NONE"},
+        "disable-tls": {"SSLCertificate": "SSL_VERIFY_NONE", "ENCRYPTION": "no"},
+    }
+    options = tls_settings[connection.tls.value]
+    if options:
+        if (hasattr(connection, "database") and not connection.database) or (
+            hasattr(connection, "databaseSchema") and not connection.databaseSchema
+        ):
+            url += "/"
+        params = "&".join(
+            f"{key}={quote_plus(value)}" for (key, value) in options.items() if value
+        )
+        url = f"{url}?{params}"
+    return url
+
+
+def get_connection(connection: ExasolConnection) -> Engine:
+    """
+    Create connection
+    """
+    return create_generic_db_connection(
+        connection=connection,
+        get_connection_url_fn=get_connection_url,
+        get_connection_args_fn=get_connection_args_common,
+    )
+
+
+def test_connection(
+    metadata: OpenMetadata,
+    engine: Engine,
+    service_connection: ExasolConnection,
+    automation_workflow: Optional[AutomationWorkflow] = None,
+) -> None:
+    """
+    Test connection. This can be executed either as part
+    of a metadata workflow or during an Automation Workflow
+    """
+    test_query(engine, "SELECT 1;")

metadata/ingestion/source/database/exasol/metadata.py

@@ -0,0 +1,27 @@
+from typing import Optional, cast
+
+from metadata.generated.schema.entity.services.connections.database.exasolConnection import (
+    ExasolConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
+
+
+class ExasolSource(CommonDbSourceService):
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        if config.serviceConnection is None:
+            raise InvalidSourceException("Missing service connection")
+        connection = cast(ExasolConnection, config.serviceConnection.root.config)
+        if not isinstance(connection, ExasolConnection):
+            raise InvalidSourceException(
+                f"Expected ExasolConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/exasol/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.exasol.metadata import ExasolSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=ExasolSource)

metadata/ingestion/source/database/glue/metadata.py

@@ -12,21 +12,24 @@
 Glue source methods.
 """
 import traceback
-from typing import Any, Iterable, Optional, Tuple, Union
+from typing import Any, Iterable, Optional, Tuple
 
 from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
 from metadata.generated.schema.api.data.createDatabaseSchema import (
     CreateDatabaseSchemaRequest,
 )
-from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
 from metadata.generated.schema.api.data.createStoredProcedure import (
     CreateStoredProcedureRequest,
 )
 from metadata.generated.schema.api.data.createTable import CreateTableRequest
-from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
-from metadata.generated.schema.entity.data.table import Column, Table, TableType
+from metadata.generated.schema.entity.data.table import (
+    Column,
+    FileFormat,
+    Table,
+    TableType,
+)
 from metadata.generated.schema.entity.services.connections.database.glueConnection import (
     GlueConnection,
 )
@@ -39,7 +42,11 @@ from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+)
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -48,6 +55,9 @@ from metadata.ingestion.source.connections import get_connection
 from metadata.ingestion.source.database.column_helpers import truncate_column_name
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.ingestion.source.database.database_service import DatabaseServiceSource
+from metadata.ingestion.source.database.external_table_lineage_mixin import (
+    ExternalTableLineageMixin,
+)
 from metadata.ingestion.source.database.glue.models import Column as GlueColumn
 from metadata.ingestion.source.database.glue.models import (
     DatabasePage,
@@ -62,7 +72,7 @@ from metadata.utils.logger import ingestion_logger
 logger = ingestion_logger()
 
 
-class GlueSource(DatabaseServiceSource):
+class GlueSource(ExternalTableLineageMixin, DatabaseServiceSource):
     """
     Implements the necessary methods to extract
     Database metadata from Glue Source
@@ -79,6 +89,8 @@ class GlueSource(DatabaseServiceSource):
         self.glue = get_connection(self.service_connection)
 
         self.connection_obj = self.glue
+        self.schema_description_map = {}
+        self.external_location_map = {}
         self.test_connection()
 
     @classmethod
@@ -191,6 +203,10 @@ class GlueSource(DatabaseServiceSource):
                     ):
                         self.status.filter(schema_fqn, "Schema Filtered Out")
                         continue
+                    if schema.Description:
+                        self.schema_description_map[schema.Name] = Markdown(
+                            schema.Description
+                        )
                     yield schema.Name
                 except Exception as exc:
                     self.status.failed(
@@ -211,6 +227,7 @@ class GlueSource(DatabaseServiceSource):
         yield Either(
             right=CreateDatabaseSchemaRequest(
                 name=EntityName(schema_name),
+                description=self.schema_description_map.get(schema_name),
                 database=FullyQualifiedEntityName(
                     fqn.build(
                         metadata=self.metadata,
@@ -295,9 +312,16 @@ class GlueSource(DatabaseServiceSource):
         table_name, table_type = table_name_and_type
         table = self.context.get().table_data
         table_constraints = None
+        storage_descriptor = table.StorageDescriptor
+        database_name = self.context.get().database
+        schema_name = self.context.get().database_schema
+        if storage_descriptor.Location:
+            # s3a doesn't occur as a path in containers, so it needs to be replaced for lineage to work
+            self.external_location_map[
+                (database_name, schema_name, table_name)
+            ] = storage_descriptor.Location.replace("s3a://", "s3://")
         try:
-            columns = self.get_columns(table.StorageDescriptor)
-
+            columns = self.get_columns(storage_descriptor)
             table_request = CreateTableRequest(
                 name=EntityName(table_name),
                 tableType=table_type,
@@ -309,8 +333,8 @@ class GlueSource(DatabaseServiceSource):
                         metadata=self.metadata,
                         entity_type=DatabaseSchema,
                         service_name=self.context.get().database_service,
-                        database_name=self.context.get().database,
-                        schema_name=self.context.get().database_schema,
+                        database_name=database_name,
+                        schema_name=schema_name,
                     )
                 ),
                 sourceUrl=self.get_source_url(
@@ -318,6 +342,8 @@ class GlueSource(DatabaseServiceSource):
                     schema_name=self.context.get().database_schema,
                     database_name=self.context.get().database,
                 ),
+                fileFormat=self.get_format(storage_descriptor),
+                locationPath=storage_descriptor.Location,
             )
             yield Either(right=table_request)
             self.register_record(table_request=table_request)
@@ -360,12 +386,22 @@ class GlueSource(DatabaseServiceSource):
         for column in self.context.get().table_data.PartitionKeys:
             yield self._get_column_object(column)
 
+    @classmethod
+    def get_format(cls, storage: StorageDetails) -> Optional[FileFormat]:
+        library = storage.SerdeInfo.SerializationLibrary
+        if library is None:
+            return None
+        if library.endswith(".LazySimpleSerDe"):
+            return (
+                FileFormat.tsv
+                if storage.SerdeInfo.Parameters.get("serialization.format") == "\t"
+                else FileFormat.csv
+            )
+        return next((fmt for fmt in FileFormat if fmt.value in library.lower()), None)
+
     def standardize_table_name(self, _: str, table: str) -> str:
         return table[:128]
 
-    def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
-        yield from []
-
     def yield_tag(
         self, schema_name: str
     ) -> Iterable[Either[OMetaTagAndClassification]]:
@@ -382,12 +418,6 @@ class GlueSource(DatabaseServiceSource):
     def get_stored_procedure_queries(self) -> Iterable[QueryByProcedure]:
         """Not Implemented"""
 
-    def yield_procedure_lineage_and_queries(
-        self,
-    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
-        """Not Implemented"""
-        yield from []
-
     def get_source_url(
         self,
         database_name: Optional[str],

metadata/ingestion/source/database/glue/models.py

@@ -20,6 +20,7 @@ from pydantic import BaseModel
 class GlueSchema(BaseModel):
     CatalogId: Optional[str] = None
     Name: str
+    Description: Optional[str] = None
 
 
 class DatabasePage(BaseModel):
@@ -36,8 +37,15 @@ class Column(BaseModel):
     Comment: Optional[str] = None
 
 
+class SerializationDetails(BaseModel):
+    SerializationLibrary: Optional[str] = None
+    Parameters: Optional[dict] = {}
+
+
 class StorageDetails(BaseModel):
     Columns: Optional[List[Column]] = []
+    Location: Optional[str] = None
+    SerdeInfo: Optional[SerializationDetails] = SerializationDetails()
 
 
 class GlueTable(BaseModel):

metadata/ingestion/source/database/glue/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.glue.metadata import GlueSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=GlueSource)

metadata/ingestion/source/database/greenplum/lineage.py

@@ -0,0 +1,46 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Greenplum lineage module
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.greenplumConnection import (
+    GreenplumConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class GreenplumLineageSource(LineageSource):
+    """
+    Greenplum lineage source implements view lineage
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        """Create class instance"""
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: GreenplumConnection = config.serviceConnection.root.config
+        if not isinstance(connection, GreenplumConnection):
+            raise InvalidSourceException(
+                f"Expected GreenplumConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/greenplum/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.greenplum.metadata import GreenplumSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=GreenplumSource)

metadata/ingestion/source/database/hive/lineage.py

@@ -0,0 +1,46 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Hive lineage module
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.hiveConnection import (
+    HiveConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class HiveLineageSource(LineageSource):
+    """
+    Hive lineage source implements view lineage
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        """Create class instance"""
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: HiveConnection = config.serviceConnection.root.config
+        if not isinstance(connection, HiveConnection):
+            raise InvalidSourceException(
+                f"Expected HiveConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/hive/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.hive.metadata import HiveSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=HiveSource)

metadata/ingestion/source/database/iceberg/metadata.py

@@ -12,7 +12,7 @@
 Iceberg source methods.
 """
 import traceback
-from typing import Any, Iterable, Optional, Tuple, Union
+from typing import Any, Iterable, Optional, Tuple
 
 import pyiceberg
 import pyiceberg.exceptions
@@ -21,12 +21,10 @@ from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequ
 from metadata.generated.schema.api.data.createDatabaseSchema import (
     CreateDatabaseSchemaRequest,
 )
-from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
 from metadata.generated.schema.api.data.createStoredProcedure import (
     CreateStoredProcedureRequest,
 )
 from metadata.generated.schema.api.data.createTable import CreateTableRequest
-from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.table import Table, TableType
@@ -293,13 +291,6 @@ class IcebergSource(DatabaseServiceSource):
         """
         yield from []
 
-    def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
-        """
-        From topology.
-        Parses view definition to get lineage information
-        """
-        yield from []
-
     def get_stored_procedures(self) -> Iterable[Any]:
         """Not Implemented"""
 
@@ -309,11 +300,5 @@ class IcebergSource(DatabaseServiceSource):
         """Process the stored procedure information"""
         yield from []
 
-    def yield_procedure_lineage_and_queries(
-        self,
-    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
-        """Extracts the lineage information from Stored Procedures"""
-        yield from []
-
     def close(self):
         """There is no connection to close."""

metadata/ingestion/source/database/iceberg/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.iceberg.metadata import IcebergSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=IcebergSource)