openmetadata-ingestion 1.3.3.0rc2__py3-none-any.whl → 1.4.0.0__py3-none-any.whl

metadata/ingestion/source/database/dbt/dbt_service.py

@@ -31,7 +31,7 @@ from metadata.ingestion.models.ometa_classification import OMetaTagAndClassifica
 from metadata.ingestion.models.topology import (
     NodeStage,
     ServiceTopology,
-    TopologyContext,
+    TopologyContextManager,
     TopologyNode,
 )
 from metadata.ingestion.source.database.database_service import DataModelLink
@@ -133,7 +133,7 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
     """
 
     topology = DbtServiceTopology()
-    context = TopologyContext.create(topology)
+    context = TopologyContextManager(topology)
     source_config: DbtPipeline
 
     @property
@@ -162,20 +162,22 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
     def get_dbt_files(self) -> Iterable[DbtFiles]:
         dbt_files = get_dbt_details(self.source_config.dbtConfigSource)
         for dbt_file in dbt_files:
-            self.context.dbt_file = dbt_file
+            self.context.get().dbt_file = dbt_file
             yield dbt_file
 
     def get_dbt_objects(self) -> Iterable[DbtObjects]:
         self.remove_manifest_non_required_keys(
-            manifest_dict=self.context.dbt_file.dbt_manifest
+            manifest_dict=self.context.get().dbt_file.dbt_manifest
         )
         dbt_objects = DbtObjects(
-            dbt_catalog=parse_catalog(self.context.dbt_file.dbt_catalog)
-            if self.context.dbt_file.dbt_catalog
+            dbt_catalog=parse_catalog(self.context.get().dbt_file.dbt_catalog)
+            if self.context.get().dbt_file.dbt_catalog
             else None,
-            dbt_manifest=parse_manifest(self.context.dbt_file.dbt_manifest),
-            dbt_run_results=parse_run_results(self.context.dbt_file.dbt_run_results)
-            if self.context.dbt_file.dbt_run_results
+            dbt_manifest=parse_manifest(self.context.get().dbt_file.dbt_manifest),
+            dbt_run_results=parse_run_results(
+                self.context.get().dbt_file.dbt_run_results
+            )
+            if self.context.get().dbt_file.dbt_run_results
             else None,
         )
         yield dbt_objects
@@ -204,7 +206,7 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
         """
         Prepare the data models
         """
-        for data_model_link in self.context.data_model_links:
+        for data_model_link in self.context.get().data_model_links:
             yield data_model_link
 
     @abstractmethod
@@ -231,7 +233,7 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
         """
         Prepare the DBT tests
         """
-        for _, dbt_test in self.context.dbt_tests.items():
+        for _, dbt_test in self.context.get().dbt_tests.items():
             yield dbt_test
 
     @abstractmethod

metadata/ingestion/source/database/dbt/metadata.py

@@ -21,13 +21,13 @@ from metadata.generated.schema.api.tests.createTestDefinition import (
     CreateTestDefinitionRequest,
 )
 from metadata.generated.schema.entity.classification.tag import Tag
+from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
 from metadata.generated.schema.entity.data.table import (
     Column,
     DataModel,
     ModelType,
     Table,
 )
-from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
@@ -83,6 +83,7 @@ from metadata.ingestion.source.database.dbt.dbt_utils import (
     get_dbt_model_name,
     get_dbt_raw_query,
 )
+from metadata.ingestion.source.database.dbt.models import DbtMeta
 from metadata.utils import fqn
 from metadata.utils.elasticsearch import get_entity_from_es_result
 from metadata.utils.logger import ingestion_logger
@@ -115,7 +116,9 @@ class DbtSource(DbtServiceSource):
         )
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         return cls(config, metadata)
 
@@ -128,13 +131,7 @@ class DbtSource(DbtServiceSource):
         """
         By default for DBT nothing is required to be prepared
         """
-        database_service = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=self.config.serviceName
-        )
-        if not database_service:
-            raise InvalidServiceException(
-                f"Service with name {self.config.serviceName} not found"
-            )
+        pass
 
     def get_dbt_owner(
         self, manifest_node: dict, catalog_node: Optional[dict]
@@ -297,11 +294,13 @@ class DbtSource(DbtServiceSource):
         """
         Method to append dbt test cases for later processing
         """
-        self.context.dbt_tests[key] = {DbtCommonEnum.MANIFEST_NODE.value: manifest_node}
-        self.context.dbt_tests[key][
+        self.context.get().dbt_tests[key] = {
+            DbtCommonEnum.MANIFEST_NODE.value: manifest_node
+        }
+        self.context.get().dbt_tests[key][
             DbtCommonEnum.UPSTREAM.value
         ] = self.parse_upstream_nodes(manifest_entities, manifest_node)
-        self.context.dbt_tests[key][DbtCommonEnum.RESULTS.value] = next(
+        self.context.get().dbt_tests[key][DbtCommonEnum.RESULTS.value] = next(
             (
                 item
                 for item in dbt_objects.dbt_run_results.results
@@ -328,14 +327,14 @@ class DbtSource(DbtServiceSource):
                     **dbt_objects.dbt_catalog.sources,
                     **dbt_objects.dbt_catalog.nodes,
                 }
-            self.context.data_model_links = []
-            self.context.dbt_tests = {}
-            self.context.run_results_generate_time = None
+            self.context.get().data_model_links = []
+            self.context.get().dbt_tests = {}
+            self.context.get().run_results_generate_time = None
             if (
                 dbt_objects.dbt_run_results
                 and dbt_objects.dbt_run_results.metadata.generated_at
             ):
-                self.context.run_results_generate_time = (
+                self.context.get().run_results_generate_time = (
                     dbt_objects.dbt_run_results.metadata.generated_at
                 )
             for key, manifest_node in manifest_entities.items():
@@ -385,7 +384,7 @@ class DbtSource(DbtServiceSource):
                     if dbt_objects.dbt_catalog:
                         catalog_node = catalog_entities.get(key)
 
-                    dbt_table_tags_list = None
+                    dbt_table_tags_list = []
                     if manifest_node.tags:
                         dbt_table_tags_list = get_tag_labels(
                             metadata=self.metadata,
@@ -394,6 +393,11 @@ class DbtSource(DbtServiceSource):
                             include_tags=self.source_config.includeTags,
                         )
 
+                    if manifest_node.meta:
+                        dbt_table_tags_list.extend(
+                            self.process_dbt_meta(manifest_node.meta) or []
+                        )
+
                     dbt_compiled_query = get_dbt_compiled_query(manifest_node)
                     dbt_raw_query = get_dbt_raw_query(manifest_node)
 
@@ -402,7 +406,7 @@ class DbtSource(DbtServiceSource):
                     table_fqn = fqn.build(
                         self.metadata,
                         entity_type=Table,
-                        service_name=self.config.serviceName,
+                        service_name="*",
                         database_name=get_corrected_name(manifest_node.database),
                         schema_name=get_corrected_name(manifest_node.schema_),
                         table_name=model_name,
@@ -441,11 +445,11 @@ class DbtSource(DbtServiceSource):
                                     manifest_node=manifest_node,
                                     catalog_node=catalog_node,
                                 ),
-                                tags=dbt_table_tags_list,
+                                tags=dbt_table_tags_list or None,
                             ),
                         )
                         yield Either(right=data_model_link)
-                        self.context.data_model_links.append(data_model_link)
+                        self.context.get().data_model_links.append(data_model_link)
                     else:
                         logger.warning(
                             f"Unable to find the table '{table_fqn}' in OpenMetadata"
@@ -496,7 +500,7 @@ class DbtSource(DbtServiceSource):
                         parent_fqn = fqn.build(
                             self.metadata,
                             entity_type=Table,
-                            service_name=self.config.serviceName,
+                            service_name="*",
                             database_name=get_corrected_name(parent_node.database),
                             schema_name=get_corrected_name(parent_node.schema_),
                             table_name=table_name,
@@ -544,12 +548,41 @@ class DbtSource(DbtServiceSource):
                 if catalog_column and catalog_column.comment:
                     column_description = catalog_column.comment
 
+                dbt_column_tag_list = []
+                dbt_column_tag_list.extend(
+                    get_tag_labels(
+                        metadata=self.metadata,
+                        tags=manifest_column.tags,
+                        classification_name=self.tag_classification_name,
+                        include_tags=self.source_config.includeTags,
+                    )
+                    or []
+                )
+
+                if manifest_column.meta:
+                    dbt_column_meta = DbtMeta(**manifest_column.meta)
+                    logger.debug(f"Processing DBT column glossary: {key}")
+                    if (
+                        dbt_column_meta.openmetadata
+                        and dbt_column_meta.openmetadata.glossary
+                    ):
+                        dbt_column_tag_list.extend(
+                            get_tag_labels(
+                                metadata=self.metadata,
+                                tags=dbt_column_meta.openmetadata.glossary,
+                                include_tags=self.source_config.includeTags,
+                                tag_type=GlossaryTerm,
+                            )
+                            or []
+                        )
+
                 columns.append(
                     Column(
                         name=column_name,
-                        description=manifest_column.description
-                        if manifest_column.description
-                        else column_description,
+                        # If the catalog description is present, use it, else use the manifest description
+                        description=column_description
+                        if column_description
+                        else manifest_column.description,
                         dataType=ColumnTypeParser.get_column_type(
                             catalog_column.type
                             if catalog_column
@@ -559,12 +592,7 @@ class DbtSource(DbtServiceSource):
                         ordinalPosition=catalog_column.index
                         if catalog_column
                         else None,
-                        tags=get_tag_labels(
-                            metadata=self.metadata,
-                            tags=manifest_column.tags,
-                            classification_name=self.tag_classification_name,
-                            include_tags=self.source_config.includeTags,
-                        ),
+                        tags=dbt_column_tag_list or None,
                     )
                 )
                 logger.debug(f"Successfully processed DBT column: {key}")
@@ -670,6 +698,42 @@ class DbtSource(DbtServiceSource):
                 )
             )
 
+    def process_dbt_meta(self, manifest_meta):
+        """
+        Method to process DBT meta for Tags and GlossaryTerms
+        """
+        dbt_table_tags_list = []
+        try:
+            dbt_meta_info = DbtMeta(**manifest_meta)
+            if dbt_meta_info.openmetadata and dbt_meta_info.openmetadata.glossary:
+                dbt_table_tags_list.extend(
+                    get_tag_labels(
+                        metadata=self.metadata,
+                        tags=dbt_meta_info.openmetadata.glossary,
+                        include_tags=self.source_config.includeTags,
+                        tag_type=GlossaryTerm,
+                    )
+                    or []
+                )
+
+            if dbt_meta_info.openmetadata and dbt_meta_info.openmetadata.tier:
+                tier_fqn = dbt_meta_info.openmetadata.tier
+                dbt_table_tags_list.extend(
+                    get_tag_labels(
+                        metadata=self.metadata,
+                        tags=[tier_fqn.split(fqn.FQN_SEPARATOR)[-1]],
+                        classification_name=tier_fqn.split(fqn.FQN_SEPARATOR)[0],
+                        include_tags=self.source_config.includeTags,
+                    )
+                    or []
+                )
+
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to process meta dbt Tags and GlossaryTerms: {exc}")
+
+        return dbt_table_tags_list or []
+
     def process_dbt_descriptions(self, data_model_link: DataModelLink):
         """
         Method to process DBT descriptions using patch APIs
@@ -847,8 +911,8 @@ class DbtSource(DbtServiceSource):
                 dbt_timestamp = None
                 if dbt_test_completed_at:
                     dbt_timestamp = dbt_test_completed_at
-                elif self.context.run_results_generate_time:
-                    dbt_timestamp = self.context.run_results_generate_time
+                elif self.context.get().run_results_generate_time:
+                    dbt_timestamp = self.context.get().run_results_generate_time
 
                 # check if the timestamp is a str type and convert accordingly
                 if isinstance(dbt_timestamp, str):
@@ -878,7 +942,7 @@ class DbtSource(DbtServiceSource):
                     test_case_fqn = fqn.build(
                         self.metadata,
                         entity_type=TestCase,
-                        service_name=self.config.serviceName,
+                        service_name=source_elements[0],
                         database_name=source_elements[1],
                         schema_name=source_elements[2],
                         table_name=source_elements[3],

metadata/ingestion/source/database/dbt/models.py

@@ -12,7 +12,7 @@
 Models required for dbt 
 """
 
-from typing import Any, Optional
+from typing import Any, List, Optional
 
 from pydantic import BaseModel
 
@@ -33,3 +33,12 @@ class DbtFilteredModel(BaseModel):
     is_filtered: Optional[bool] = False
     message: Optional[str]
     model_fqn: Optional[str]
+
+
+class DbtMetaGlossaryTier(BaseModel):
+    tier: Optional[str]
+    glossary: Optional[List[str]]
+
+
+class DbtMeta(BaseModel):
+    openmetadata: Optional[DbtMetaGlossaryTier]

metadata/ingestion/source/database/deltalake/connection.py

@@ -65,13 +65,13 @@ def get_connection(connection: DeltaLakeConnection) -> SparkSession:
             )
         if connection.metastoreConnection.jdbcDriverClassPath:
             builder.config(
-                "sparks.driver.extraClassPath",
+                "spark.driver.extraClassPath",
                 connection.metastoreConnection.jdbcDriverClassPath,
             )
         if connection.metastoreConnection.username:
             builder.config(
                 "spark.hadoop.javax.jdo.option.ConnectionUserName",
-                connection.metastoreConnection.metastoreDb,
+                connection.metastoreConnection.username,
             )
         if connection.metastoreConnection.password:
             builder.config(

metadata/ingestion/source/database/deltalake/metadata.py

@@ -117,7 +117,9 @@ class DeltalakeSource(DatabaseServiceSource):
         self.test_connection()
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: DeltaLakeConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, DeltaLakeConnection):
@@ -148,7 +150,7 @@ class DeltalakeSource(DatabaseServiceSource):
         yield Either(
             right=CreateDatabaseRequest(
                 name=database_name,
-                service=self.context.database_service,
+                service=self.context.get().database_service,
             )
         )
 
@@ -161,8 +163,8 @@ class DeltalakeSource(DatabaseServiceSource):
             schema_fqn = fqn.build(
                 self.metadata,
                 entity_type=DatabaseSchema,
-                service_name=self.context.database_service,
-                database_name=self.context.database,
+                service_name=self.context.get().database_service,
+                database_name=self.context.get().database,
                 schema_name=schema.name,
             )
             if filter_by_schema(
@@ -188,8 +190,8 @@ class DeltalakeSource(DatabaseServiceSource):
                 database=fqn.build(
                     metadata=self.metadata,
                     entity_type=Database,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
                 ),
             )
         )
@@ -203,16 +205,16 @@ class DeltalakeSource(DatabaseServiceSource):
 
         :return: tables or views, depending on config
         """
-        schema_name = self.context.database_schema
+        schema_name = self.context.get().database_schema
         for table in self.spark.catalog.listTables(dbName=schema_name):
             try:
                 table_name = table.name
                 table_fqn = fqn.build(
                     self.metadata,
                     entity_type=Table,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
-                    schema_name=self.context.database_schema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
                     table_name=table.name,
                 )
                 if filter_by_table(
@@ -234,7 +236,7 @@ class DeltalakeSource(DatabaseServiceSource):
                         logger.debug(f"Skipping temporary table {table.name}")
                         continue
 
-                    self.context.table_description = table.description
+                    self.context.get().table_description = table.description
                     yield table_name, TABLE_TYPE_MAP.get(
                         table.tableType, TableType.Regular
                     )
@@ -244,7 +246,7 @@ class DeltalakeSource(DatabaseServiceSource):
                     and table.tableType
                     and table.tableType == SparkTableType.VIEW.value
                 ):
-                    self.context.table_description = table.description
+                    self.context.get().table_description = table.description
                     yield table_name, TableType.View
 
             except Exception as exc:
@@ -260,7 +262,7 @@ class DeltalakeSource(DatabaseServiceSource):
         Prepare a table request and pass it to the sink
         """
         table_name, table_type = table_name_and_type
-        schema_name = self.context.database_schema
+        schema_name = self.context.get().database_schema
         try:
             columns = self.get_columns(schema_name, table_name)
             view_definition = (
@@ -272,17 +274,17 @@ class DeltalakeSource(DatabaseServiceSource):
             table_request = CreateTableRequest(
                 name=table_name,
                 tableType=table_type,
-                description=self.context.table_description,
+                description=self.context.get().table_description,
                 columns=columns,
                 tableConstraints=None,
                 databaseSchema=fqn.build(
                     metadata=self.metadata,
                     entity_type=DatabaseSchema,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
                     schema_name=schema_name,
                 ),
-                viewDefinition=view_definition,
+                schemaDefinition=view_definition,
             )
 
             yield Either(right=table_request)

metadata/ingestion/source/database/domodatabase/metadata.py

@@ -83,7 +83,12 @@ class DomodatabaseSource(DatabaseServiceSource):
         self.test_connection()
 
     @classmethod
-    def create(cls, config_dict: dict, metadata: OpenMetadata):
+    def create(
+        cls,
+        config_dict: dict,
+        metadata: OpenMetadata,
+        pipeline_name: Optional[str] = None,
+    ):
         config = WorkflowSource.parse_obj(config_dict)
         connection: DomoDatabaseConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, DomoDatabaseConnection):
@@ -102,7 +107,7 @@ class DomodatabaseSource(DatabaseServiceSource):
         yield Either(
             right=CreateDatabaseRequest(
                 name=database_name,
-                service=self.context.database_service,
+                service=self.context.get().database_service,
             )
         )
 
@@ -119,14 +124,14 @@ class DomodatabaseSource(DatabaseServiceSource):
                 database=fqn.build(
                     metadata=self.metadata,
                     entity_type=Database,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
                 ),
             )
         )
 
     def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]:
-        schema_name = self.context.database_schema
+        schema_name = self.context.get().database_schema
         try:
             tables = list(self.domo_client.datasets.list())
             for table in tables:
@@ -135,9 +140,9 @@ class DomodatabaseSource(DatabaseServiceSource):
                 table_fqn = fqn.build(
                     self.metadata,
                     entity_type=Table,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
-                    schema_name=self.context.database_schema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
                     table_name=table["name"],
                 )
 
@@ -194,9 +199,9 @@ class DomodatabaseSource(DatabaseServiceSource):
                 databaseSchema=fqn.build(
                     metadata=self.metadata,
                     entity_type=DatabaseSchema,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
-                    schema_name=self.context.database_schema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
                 ),
                 sourceUrl=self.get_source_url(
                     table_name=table_id,

metadata/ingestion/source/database/doris/connection.py

@@ -26,7 +26,6 @@ from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
     get_connection_url_common,
-    init_empty_connection_options,
 )
 from metadata.ingestion.connections.test_connections import (
     test_connection_db_schema_sources,
@@ -38,15 +37,6 @@ def get_connection(connection: DorisConnection) -> Engine:
     """
     Create connection
     """
-    if connection.sslCA or connection.sslCert or connection.sslKey:
-        if not connection.connectionOptions:
-            connection.connectionOptions = init_empty_connection_options()
-        if connection.sslCA:
-            connection.connectionOptions.__root__["ssl_ca"] = connection.sslCA
-        if connection.sslCert:
-            connection.connectionOptions.__root__["ssl_cert"] = connection.sslCert
-        if connection.sslKey:
-            connection.connectionOptions.__root__["ssl_key"] = connection.sslKey
     return create_generic_db_connection(
         connection=connection,
         get_connection_url_fn=get_connection_url_common,

metadata/ingestion/source/database/doris/metadata.py

@@ -21,7 +21,8 @@ from sqlalchemy.engine.reflection import Inspector
 
 from metadata.generated.schema.entity.data.table import (
     Column,
-    IntervalType,
+    PartitionColumnDetails,
+    PartitionIntervalTypes,
     TableConstraint,
     TablePartition,
     TableType,
@@ -49,6 +50,7 @@ from metadata.ingestion.source.database.doris.utils import (
 )
 from metadata.ingestion.source.database.mysql.utils import parse_column
 from metadata.utils.logger import ingestion_logger
+from metadata.utils.ssl_manager import SSLManager, check_ssl_and_init
 
 MySQLTableDefinitionParser._parse_column = (  # pylint: disable=protected-access
     parse_column
@@ -143,8 +145,18 @@ class DorisSource(CommonDbSourceService):
     Database metadata from Mysql Source
     """
 
+    def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
+        self.ssl_manager = None
+        service_connection = config.serviceConnection.__root__.config
+        self.ssl_manager: SSLManager = check_ssl_and_init(service_connection)
+        if self.ssl_manager:
+            service_connection = self.ssl_manager.setup_ssl(service_connection)
+        super().__init__(config, metadata)
+
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         if config.serviceConnection is None:
             raise InvalidSourceException("Missing service connection")
@@ -306,9 +318,16 @@ class DorisSource(CommonDbSourceService):
 
             if result and result[0].PartitionKey != "":
                 partition_details = TablePartition(
-                    intervalType=IntervalType.TIME_UNIT.value,
-                    columns=result[0].PartitionKey.split(", "),
+                    columns=[
+                        PartitionColumnDetails(
+                            columnName=partition_key,
+                            intervalType=PartitionIntervalTypes.TIME_UNIT,
+                            interval=None,
+                        )
+                        for partition_key in result[0].PartitionKey.split(", ")
+                    ]
                 )
+
                 return True, partition_details
             return False, None
         except Exception:

metadata/ingestion/source/database/druid/metadata.py

@@ -12,6 +12,8 @@
 Druid source methods.
 """
 
+from typing import Optional
+
 from metadata.generated.schema.entity.services.connections.database.druidConnection import (
     DruidConnection,
 )
@@ -25,7 +27,9 @@ from metadata.ingestion.source.database.common_db_source import CommonDbSourceSe
 
 class DruidSource(CommonDbSourceService):
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         """Create class instance"""
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: DruidConnection = config.serviceConnection.__root__.config

metadata/ingestion/source/database/dynamodb/metadata.py

@@ -46,7 +46,9 @@ class DynamodbSource(CommonNoSQLSource):
         self.dynamodb = self.connection_obj
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: DynamoDBConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, DynamoDBConnection):

metadata/ingestion/source/database/extended_sample_data.py

@@ -13,7 +13,7 @@ Sample Data source ingestion
 """
 import json
 from collections import namedtuple
-from typing import Iterable
+from typing import Iterable, Optional
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
@@ -194,7 +194,9 @@ class ExtendedSampleDataSource(Source):  # pylint: disable=too-many-instance-att
         self.db_name = None
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         """Create class instance"""
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: CustomDatabaseConnection = config.serviceConnection.__root__.config