openmetadata-ingestion 1.3.3.0rc1__py3-none-any.whl → 1.4.0.0rc1__py3-none-any.whl

metadata/ingestion/source/database/dbt/metadata.py

@@ -21,6 +21,7 @@ from metadata.generated.schema.api.tests.createTestDefinition import (
     CreateTestDefinitionRequest,
 )
 from metadata.generated.schema.entity.classification.tag import Tag
+from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
 from metadata.generated.schema.entity.data.table import (
     Column,
     DataModel,
@@ -83,6 +84,7 @@ from metadata.ingestion.source.database.dbt.dbt_utils import (
     get_dbt_model_name,
     get_dbt_raw_query,
 )
+from metadata.ingestion.source.database.dbt.models import DbtMeta
 from metadata.utils import fqn
 from metadata.utils.elasticsearch import get_entity_from_es_result
 from metadata.utils.logger import ingestion_logger
@@ -115,7 +117,9 @@ class DbtSource(DbtServiceSource):
         )
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         return cls(config, metadata)
 
@@ -297,11 +301,13 @@ class DbtSource(DbtServiceSource):
         """
         Method to append dbt test cases for later processing
         """
-        self.context.dbt_tests[key] = {DbtCommonEnum.MANIFEST_NODE.value: manifest_node}
-        self.context.dbt_tests[key][
+        self.context.get().dbt_tests[key] = {
+            DbtCommonEnum.MANIFEST_NODE.value: manifest_node
+        }
+        self.context.get().dbt_tests[key][
             DbtCommonEnum.UPSTREAM.value
         ] = self.parse_upstream_nodes(manifest_entities, manifest_node)
-        self.context.dbt_tests[key][DbtCommonEnum.RESULTS.value] = next(
+        self.context.get().dbt_tests[key][DbtCommonEnum.RESULTS.value] = next(
             (
                 item
                 for item in dbt_objects.dbt_run_results.results
@@ -328,14 +334,14 @@ class DbtSource(DbtServiceSource):
                     **dbt_objects.dbt_catalog.sources,
                     **dbt_objects.dbt_catalog.nodes,
                 }
-            self.context.data_model_links = []
-            self.context.dbt_tests = {}
-            self.context.run_results_generate_time = None
+            self.context.get().data_model_links = []
+            self.context.get().dbt_tests = {}
+            self.context.get().run_results_generate_time = None
             if (
                 dbt_objects.dbt_run_results
                 and dbt_objects.dbt_run_results.metadata.generated_at
             ):
-                self.context.run_results_generate_time = (
+                self.context.get().run_results_generate_time = (
                     dbt_objects.dbt_run_results.metadata.generated_at
                 )
             for key, manifest_node in manifest_entities.items():
@@ -385,7 +391,7 @@ class DbtSource(DbtServiceSource):
                     if dbt_objects.dbt_catalog:
                         catalog_node = catalog_entities.get(key)
 
-                    dbt_table_tags_list = None
+                    dbt_table_tags_list = []
                     if manifest_node.tags:
                         dbt_table_tags_list = get_tag_labels(
                             metadata=self.metadata,
@@ -394,6 +400,11 @@ class DbtSource(DbtServiceSource):
                             include_tags=self.source_config.includeTags,
                         )
 
+                    if manifest_node.meta:
+                        dbt_table_tags_list.extend(
+                            self.process_dbt_meta(manifest_node.meta) or []
+                        )
+
                     dbt_compiled_query = get_dbt_compiled_query(manifest_node)
                     dbt_raw_query = get_dbt_raw_query(manifest_node)
 
@@ -441,11 +452,11 @@ class DbtSource(DbtServiceSource):
                                     manifest_node=manifest_node,
                                     catalog_node=catalog_node,
                                 ),
-                                tags=dbt_table_tags_list,
+                                tags=dbt_table_tags_list or None,
                             ),
                         )
                         yield Either(right=data_model_link)
-                        self.context.data_model_links.append(data_model_link)
+                        self.context.get().data_model_links.append(data_model_link)
                     else:
                         logger.warning(
                             f"Unable to find the table '{table_fqn}' in OpenMetadata"
@@ -544,6 +555,34 @@ class DbtSource(DbtServiceSource):
                 if catalog_column and catalog_column.comment:
                     column_description = catalog_column.comment
 
+                dbt_column_tag_list = []
+                dbt_column_tag_list.extend(
+                    get_tag_labels(
+                        metadata=self.metadata,
+                        tags=manifest_column.tags,
+                        classification_name=self.tag_classification_name,
+                        include_tags=self.source_config.includeTags,
+                    )
+                    or []
+                )
+
+                if manifest_column.meta:
+                    dbt_column_meta = DbtMeta(**manifest_column.meta)
+                    logger.debug(f"Processing DBT column glossary: {key}")
+                    if (
+                        dbt_column_meta.openmetadata
+                        and dbt_column_meta.openmetadata.glossary
+                    ):
+                        dbt_column_tag_list.extend(
+                            get_tag_labels(
+                                metadata=self.metadata,
+                                tags=dbt_column_meta.openmetadata.glossary,
+                                include_tags=self.source_config.includeTags,
+                                tag_type=GlossaryTerm,
+                            )
+                            or []
+                        )
+
                 columns.append(
                     Column(
                         name=column_name,
@@ -559,12 +598,7 @@ class DbtSource(DbtServiceSource):
                         ordinalPosition=catalog_column.index
                         if catalog_column
                         else None,
-                        tags=get_tag_labels(
-                            metadata=self.metadata,
-                            tags=manifest_column.tags,
-                            classification_name=self.tag_classification_name,
-                            include_tags=self.source_config.includeTags,
-                        ),
+                        tags=dbt_column_tag_list or None,
                     )
                 )
                 logger.debug(f"Successfully processed DBT column: {key}")
@@ -670,6 +704,42 @@ class DbtSource(DbtServiceSource):
                 )
             )
 
+    def process_dbt_meta(self, manifest_meta):
+        """
+        Method to process DBT meta for Tags and GlossaryTerms
+        """
+        dbt_table_tags_list = []
+        try:
+            dbt_meta_info = DbtMeta(**manifest_meta)
+            if dbt_meta_info.openmetadata and dbt_meta_info.openmetadata.glossary:
+                dbt_table_tags_list.extend(
+                    get_tag_labels(
+                        metadata=self.metadata,
+                        tags=dbt_meta_info.openmetadata.glossary,
+                        include_tags=self.source_config.includeTags,
+                        tag_type=GlossaryTerm,
+                    )
+                    or []
+                )
+
+            if dbt_meta_info.openmetadata and dbt_meta_info.openmetadata.tier:
+                tier_fqn = dbt_meta_info.openmetadata.tier
+                dbt_table_tags_list.extend(
+                    get_tag_labels(
+                        metadata=self.metadata,
+                        tags=[tier_fqn.split(fqn.FQN_SEPARATOR)[-1]],
+                        classification_name=tier_fqn.split(fqn.FQN_SEPARATOR)[0],
+                        include_tags=self.source_config.includeTags,
+                    )
+                    or []
+                )
+
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to process meta dbt Tags and GlossaryTerms: {exc}")
+
+        return dbt_table_tags_list or []
+
     def process_dbt_descriptions(self, data_model_link: DataModelLink):
         """
         Method to process DBT descriptions using patch APIs
@@ -847,8 +917,8 @@ class DbtSource(DbtServiceSource):
                 dbt_timestamp = None
                 if dbt_test_completed_at:
                     dbt_timestamp = dbt_test_completed_at
-                elif self.context.run_results_generate_time:
-                    dbt_timestamp = self.context.run_results_generate_time
+                elif self.context.get().run_results_generate_time:
+                    dbt_timestamp = self.context.get().run_results_generate_time
 
                 # check if the timestamp is a str type and convert accordingly
                 if isinstance(dbt_timestamp, str):

metadata/ingestion/source/database/dbt/models.py

@@ -12,7 +12,7 @@
 Models required for dbt 
 """
 
-from typing import Any, Optional
+from typing import Any, List, Optional
 
 from pydantic import BaseModel
 
@@ -33,3 +33,12 @@ class DbtFilteredModel(BaseModel):
     is_filtered: Optional[bool] = False
     message: Optional[str]
     model_fqn: Optional[str]
+
+
+class DbtMetaGlossaryTier(BaseModel):
+    tier: Optional[str]
+    glossary: Optional[List[str]]
+
+
+class DbtMeta(BaseModel):
+    openmetadata: Optional[DbtMetaGlossaryTier]

metadata/ingestion/source/database/deltalake/metadata.py

@@ -117,7 +117,9 @@ class DeltalakeSource(DatabaseServiceSource):
         self.test_connection()
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: DeltaLakeConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, DeltaLakeConnection):
@@ -148,7 +150,7 @@ class DeltalakeSource(DatabaseServiceSource):
         yield Either(
             right=CreateDatabaseRequest(
                 name=database_name,
-                service=self.context.database_service,
+                service=self.context.get().database_service,
             )
         )
 
@@ -161,8 +163,8 @@ class DeltalakeSource(DatabaseServiceSource):
             schema_fqn = fqn.build(
                 self.metadata,
                 entity_type=DatabaseSchema,
-                service_name=self.context.database_service,
-                database_name=self.context.database,
+                service_name=self.context.get().database_service,
+                database_name=self.context.get().database,
                 schema_name=schema.name,
             )
             if filter_by_schema(
@@ -188,8 +190,8 @@ class DeltalakeSource(DatabaseServiceSource):
                 database=fqn.build(
                     metadata=self.metadata,
                     entity_type=Database,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
                 ),
             )
         )
@@ -203,16 +205,16 @@ class DeltalakeSource(DatabaseServiceSource):
 
         :return: tables or views, depending on config
         """
-        schema_name = self.context.database_schema
+        schema_name = self.context.get().database_schema
         for table in self.spark.catalog.listTables(dbName=schema_name):
             try:
                 table_name = table.name
                 table_fqn = fqn.build(
                     self.metadata,
                     entity_type=Table,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
-                    schema_name=self.context.database_schema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
                     table_name=table.name,
                 )
                 if filter_by_table(
@@ -234,7 +236,7 @@ class DeltalakeSource(DatabaseServiceSource):
                         logger.debug(f"Skipping temporary table {table.name}")
                         continue
 
-                    self.context.table_description = table.description
+                    self.context.get().table_description = table.description
                     yield table_name, TABLE_TYPE_MAP.get(
                         table.tableType, TableType.Regular
                     )
@@ -244,7 +246,7 @@ class DeltalakeSource(DatabaseServiceSource):
                     and table.tableType
                     and table.tableType == SparkTableType.VIEW.value
                 ):
-                    self.context.table_description = table.description
+                    self.context.get().table_description = table.description
                     yield table_name, TableType.View
 
             except Exception as exc:
@@ -260,7 +262,7 @@ class DeltalakeSource(DatabaseServiceSource):
         Prepare a table request and pass it to the sink
         """
         table_name, table_type = table_name_and_type
-        schema_name = self.context.database_schema
+        schema_name = self.context.get().database_schema
         try:
             columns = self.get_columns(schema_name, table_name)
             view_definition = (
@@ -272,14 +274,14 @@ class DeltalakeSource(DatabaseServiceSource):
             table_request = CreateTableRequest(
                 name=table_name,
                 tableType=table_type,
-                description=self.context.table_description,
+                description=self.context.get().table_description,
                 columns=columns,
                 tableConstraints=None,
                 databaseSchema=fqn.build(
                     metadata=self.metadata,
                     entity_type=DatabaseSchema,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
                     schema_name=schema_name,
                 ),
                 viewDefinition=view_definition,

metadata/ingestion/source/database/domodatabase/metadata.py

@@ -83,7 +83,12 @@ class DomodatabaseSource(DatabaseServiceSource):
         self.test_connection()
 
     @classmethod
-    def create(cls, config_dict: dict, metadata: OpenMetadata):
+    def create(
+        cls,
+        config_dict: dict,
+        metadata: OpenMetadata,
+        pipeline_name: Optional[str] = None,
+    ):
         config = WorkflowSource.parse_obj(config_dict)
         connection: DomoDatabaseConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, DomoDatabaseConnection):
@@ -102,7 +107,7 @@ class DomodatabaseSource(DatabaseServiceSource):
         yield Either(
             right=CreateDatabaseRequest(
                 name=database_name,
-                service=self.context.database_service,
+                service=self.context.get().database_service,
             )
         )
 
@@ -119,14 +124,14 @@ class DomodatabaseSource(DatabaseServiceSource):
                 database=fqn.build(
                     metadata=self.metadata,
                     entity_type=Database,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
                 ),
             )
         )
 
     def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]:
-        schema_name = self.context.database_schema
+        schema_name = self.context.get().database_schema
         try:
             tables = list(self.domo_client.datasets.list())
             for table in tables:
@@ -135,9 +140,9 @@ class DomodatabaseSource(DatabaseServiceSource):
                 table_fqn = fqn.build(
                     self.metadata,
                     entity_type=Table,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
-                    schema_name=self.context.database_schema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
                     table_name=table["name"],
                 )
 
@@ -194,9 +199,9 @@ class DomodatabaseSource(DatabaseServiceSource):
                 databaseSchema=fqn.build(
                     metadata=self.metadata,
                     entity_type=DatabaseSchema,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
-                    schema_name=self.context.database_schema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
                 ),
                 sourceUrl=self.get_source_url(
                     table_name=table_id,

metadata/ingestion/source/database/doris/connection.py

@@ -26,7 +26,6 @@ from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
     get_connection_url_common,
-    init_empty_connection_options,
 )
 from metadata.ingestion.connections.test_connections import (
     test_connection_db_schema_sources,
@@ -38,15 +37,6 @@ def get_connection(connection: DorisConnection) -> Engine:
     """
     Create connection
     """
-    if connection.sslCA or connection.sslCert or connection.sslKey:
-        if not connection.connectionOptions:
-            connection.connectionOptions = init_empty_connection_options()
-        if connection.sslCA:
-            connection.connectionOptions.__root__["ssl_ca"] = connection.sslCA
-        if connection.sslCert:
-            connection.connectionOptions.__root__["ssl_cert"] = connection.sslCert
-        if connection.sslKey:
-            connection.connectionOptions.__root__["ssl_key"] = connection.sslKey
     return create_generic_db_connection(
         connection=connection,
         get_connection_url_fn=get_connection_url_common,

metadata/ingestion/source/database/doris/metadata.py

@@ -21,7 +21,8 @@ from sqlalchemy.engine.reflection import Inspector
 
 from metadata.generated.schema.entity.data.table import (
     Column,
-    IntervalType,
+    PartitionColumnDetails,
+    PartitionIntervalTypes,
     TableConstraint,
     TablePartition,
     TableType,
@@ -49,6 +50,7 @@ from metadata.ingestion.source.database.doris.utils import (
 )
 from metadata.ingestion.source.database.mysql.utils import parse_column
 from metadata.utils.logger import ingestion_logger
+from metadata.utils.ssl_manager import SSLManager, check_ssl_and_init
 
 MySQLTableDefinitionParser._parse_column = (  # pylint: disable=protected-access
     parse_column
@@ -143,8 +145,18 @@ class DorisSource(CommonDbSourceService):
     Database metadata from Mysql Source
     """
 
+    def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
+        self.ssl_manager = None
+        service_connection = config.serviceConnection.__root__.config
+        self.ssl_manager: SSLManager = check_ssl_and_init(service_connection)
+        if self.ssl_manager:
+            service_connection = self.ssl_manager.setup_ssl(service_connection)
+        super().__init__(config, metadata)
+
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         if config.serviceConnection is None:
             raise InvalidSourceException("Missing service connection")
@@ -306,9 +318,16 @@ class DorisSource(CommonDbSourceService):
 
             if result and result[0].PartitionKey != "":
                 partition_details = TablePartition(
-                    intervalType=IntervalType.TIME_UNIT.value,
-                    columns=result[0].PartitionKey.split(", "),
+                    columns=[
+                        PartitionColumnDetails(
+                            columnName=partition_key,
+                            intervalType=PartitionIntervalTypes.TIME_UNIT,
+                            interval=None,
+                        )
+                        for partition_key in result[0].PartitionKey.split(", ")
+                    ]
                 )
+
                 return True, partition_details
             return False, None
         except Exception:

metadata/ingestion/source/database/druid/metadata.py

@@ -12,6 +12,8 @@
 Druid source methods.
 """
 
+from typing import Optional
+
 from metadata.generated.schema.entity.services.connections.database.druidConnection import (
     DruidConnection,
 )
@@ -25,7 +27,9 @@ from metadata.ingestion.source.database.common_db_source import CommonDbSourceSe
 
 class DruidSource(CommonDbSourceService):
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         """Create class instance"""
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: DruidConnection = config.serviceConnection.__root__.config

metadata/ingestion/source/database/dynamodb/metadata.py

@@ -46,7 +46,9 @@ class DynamodbSource(CommonNoSQLSource):
         self.dynamodb = self.connection_obj
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: DynamoDBConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, DynamoDBConnection):

metadata/ingestion/source/database/extended_sample_data.py

@@ -13,7 +13,7 @@ Sample Data source ingestion
 """
 import json
 from collections import namedtuple
-from typing import Iterable
+from typing import Iterable, Optional
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
@@ -194,7 +194,9 @@ class ExtendedSampleDataSource(Source):  # pylint: disable=too-many-instance-att
         self.db_name = None
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         """Create class instance"""
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: CustomDatabaseConnection = config.serviceConnection.__root__.config

metadata/ingestion/source/database/external_table_lineage_mixin.py

@@ -0,0 +1,82 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+External Table Lineage Mixin
+"""
+
+import traceback
+from abc import ABC
+from typing import Iterable
+
+from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.type.entityLineage import EntitiesEdge
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.api.models import Either
+from metadata.utils import fqn
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class ExternalTableLineageMixin(ABC):
+    """
+    This mixin class is for deriving lineage between external table and container source/
+    """
+
+    def yield_external_table_lineage(self) -> Iterable[AddLineageRequest]:
+        """
+        Yield external table lineage
+        """
+        for table_qualified_tuple, location in self.external_location_map.items() or []:
+            try:
+                location_entity = self.metadata.es_search_container_by_path(
+                    full_path=location
+                )
+                database_name, schema_name, table_name = table_qualified_tuple
+
+                table_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=Table,
+                    service_name=self.context.get().database_service,
+                    database_name=database_name,
+                    schema_name=schema_name,
+                    table_name=table_name,
+                    skip_es_search=True,
+                )
+                table_entity = self.metadata.es_search_from_fqn(
+                    entity_type=Table,
+                    fqn_search_string=table_fqn,
+                )
+
+                if (
+                    location_entity
+                    and location_entity[0]
+                    and table_entity
+                    and table_entity[0]
+                ):
+                    yield Either(
+                        right=AddLineageRequest(
+                            edge=EntitiesEdge(
+                                fromEntity=EntityReference(
+                                    id=location_entity[0].id,
+                                    type="container",
+                                ),
+                                toEntity=EntityReference(
+                                    id=table_entity[0].id,
+                                    type="table",
+                                ),
+                            )
+                        )
+                    )
+            except Exception as exc:
+                logger.warning(f"Failed to yield external table lineage due to - {exc}")
+                logger.debug(traceback.format_exc())