openmetadata-ingestion 1.4.8.0__py3-none-any.whl → 1.5.0.0__py3-none-any.whl

metadata/ingestion/source/database/column_type_parser.py

@@ -88,7 +88,7 @@ class ColumnTypeParser:
         "BIGNUMERIC": "NUMERIC",
         "BIGSERIAL": "BIGINT",
         "BINARY": "BINARY",
-        "BIT": "INT",
+        "BIT": "BIT",
         "BLOB": "BLOB",
         "BOOL": "BOOLEAN",
         "BOOLEAN": "BOOLEAN",
@@ -148,7 +148,7 @@ class ColumnTypeParser:
         "MEDIUMBLOB": "MEDIUMBLOB",
         "MEDIUMINT": "INT",
         "MEDIUMTEXT": "MEDIUMTEXT",
-        "MONEY": "NUMBER",
+        "MONEY": "MONEY",
         "NCHAR": "CHAR",
         "NTEXT": "NTEXT",
         "NULL": "NULL",
@@ -276,21 +276,30 @@ class ColumnTypeParser:
         "WDC_BOOL": "BOOLEAN",
         "WDC_DATE": "DATE",
         "WDC_GEOMETRY": "GEOMETRY",
+        # SAP ERP
+        "CLNT": "INT",
+        "INT1": "INT",
+        "LRAW": "BLOB",
+        "UNIT": "CHAR",
+        "NUMC": "CHAR",
+        "LANG": "CHAR",
+        "CUKY": "CHAR",
+        "DATS": "DATE",
+        "TIMS": "TIME",
+        "FLTP": "FLOAT",
+        "QUAN": "DECIMAL",
+        "DEC": "DECIMAL",
+        "CURR": "DECIMAL",
+        "STRG": "STRING",
+        "RSTR": "STRING",
+        # azuresql
+        "HIERARCHYID": "UNKNOWN",
     }
 
     _COMPLEX_TYPE = re.compile("^(struct|map|array|uniontype)")
 
     _FIXED_DECIMAL = re.compile(r"(decimal|numeric)(\(\s*(\d+)\s*,\s*(\d+)\s*\))?")
 
-    try:
-        # pylint: disable=import-outside-toplevel
-        from sqlalchemy.dialects.mssql import BIT
-
-        _COLUMN_TYPE_MAPPING[BIT] = "BINARY"
-        _SOURCE_TYPE_TO_OM_TYPE["BIT"] = "BINARY"
-    except ImportError:
-        pass
-
     try:
         # pylint: disable=import-outside-toplevel
         from teradatasqlalchemy import BYTE, VARBYTE
@@ -340,7 +349,6 @@ class ColumnTypeParser:
             arr_data_type = ColumnTypeParser._parse_primitive_datatype_string(
                 data_type[6:-1]
             )["dataType"]
-
             data_type_string = {
                 "dataType": "ARRAY",
                 "arrayDataType": arr_data_type,

metadata/ingestion/source/database/common_db_source.py

@@ -50,9 +50,16 @@ from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+    SourceUrl,
+)
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.connections.session import create_and_bind_thread_safe_session
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
+from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import (
     get_connection,
@@ -107,7 +114,7 @@ class CommonDbSourceService(
         self.metadata = metadata
 
         # It will be one of the Unions. We don't know the specific type here.
-        self.service_connection = self.config.serviceConnection.__root__.config
+        self.service_connection = self.config.serviceConnection.root.config
 
         self.ssl_manager = None
         self.ssl_manager: SSLManager = check_ssl_and_init(self.service_connection)
@@ -187,12 +194,23 @@ class CommonDbSourceService(
         Prepare a database request and pass it to the sink
         """
 
+        description = (
+            Markdown(db_description)
+            if (db_description := self.get_database_description(database_name))
+            else None
+        )
+        source_url = (
+            SourceUrl(source_url)
+            if (source_url := self.get_source_url(database_name=database_name))
+            else None
+        )
+
         yield Either(
             right=CreateDatabaseRequest(
-                name=database_name,
-                service=self.context.get().database_service,
-                description=self.get_database_description(database_name),
-                sourceUrl=self.get_source_url(database_name=database_name),
+                name=EntityName(database_name),
+                service=FullyQualifiedEntityName(self.context.get().database_service),
+                description=description,
+                sourceUrl=source_url,
                 tags=self.get_database_tag_labels(database_name=database_name),
             )
         )
@@ -219,20 +237,30 @@ class CommonDbSourceService(
         Prepare a database schema request and pass it to the sink
         """
 
+        description = (
+            Markdown(db_description)
+            if (db_description := self.get_schema_description(schema_name))
+            else None
+        )
+        source_url = (
+            SourceUrl(source_url)
+            if (source_url := self.get_source_url(database_name=schema_name))
+            else None
+        )
+
         yield Either(
             right=CreateDatabaseSchemaRequest(
-                name=schema_name,
-                database=fqn.build(
-                    metadata=self.metadata,
-                    entity_type=Database,
-                    service_name=self.context.get().database_service,
-                    database_name=self.context.get().database,
-                ),
-                description=self.get_schema_description(schema_name),
-                sourceUrl=self.get_source_url(
-                    database_name=self.context.get().database,
-                    schema_name=schema_name,
+                name=EntityName(schema_name),
+                database=FullyQualifiedEntityName(
+                    fqn.build(
+                        metadata=self.metadata,
+                        entity_type=Database,
+                        service_name=self.context.get().database_service,
+                        database_name=self.context.get().database,
+                    )
                 ),
+                description=description,
+                sourceUrl=source_url,
                 tags=self.get_schema_tag_labels(schema_name=schema_name),
             )
         )
@@ -365,7 +393,11 @@ class CommonDbSourceService(
 
     @calculate_execution_time()
     def get_schema_definition(
-        self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
+        self,
+        table_type: TableType,
+        table_name: str,
+        schema_name: str,
+        inspector: Inspector,
     ) -> Optional[str]:
         """
         Get the DDL statement or View Definition for a table
@@ -443,7 +475,7 @@ class CommonDbSourceService(
 
     @calculate_execution_time_generator()
     def yield_table(
-        self, table_name_and_type: Tuple[str, str]
+        self, table_name_and_type: Tuple[str, TableType]
     ) -> Iterable[Either[CreateTableRequest]]:
         """
         From topology.
@@ -463,32 +495,48 @@ class CommonDbSourceService(
                 inspector=self.inspector,
             )
 
-            schema_definition = self.get_schema_definition(
-                table_type=table_type,
-                table_name=table_name,
-                schema_name=schema_name,
-                inspector=self.inspector,
+            schema_definition = (
+                self.get_schema_definition(
+                    table_type=table_type,
+                    table_name=table_name,
+                    schema_name=schema_name,
+                    inspector=self.inspector,
+                )
+                if self.source_config.includeDDL
+                else None
             )
+
             table_constraints = self.update_table_constraints(
                 table_constraints, foreign_columns
             )
+
+            description = (
+                Markdown(db_description)
+                if (
+                    db_description := self.get_table_description(
+                        schema_name=schema_name,
+                        table_name=table_name,
+                        inspector=self.inspector,
+                    )
+                )
+                else None
+            )
+
             table_request = CreateTableRequest(
-                name=table_name,
+                name=EntityName(table_name),
                 tableType=table_type,
-                description=self.get_table_description(
-                    schema_name=schema_name,
-                    table_name=table_name,
-                    inspector=self.inspector,
-                ),
+                description=description,
                 columns=columns,
                 tableConstraints=table_constraints,
                 schemaDefinition=schema_definition,
-                databaseSchema=fqn.build(
-                    metadata=self.metadata,
-                    entity_type=DatabaseSchema,
-                    service_name=self.context.get().database_service,
-                    database_name=self.context.get().database,
-                    schema_name=schema_name,
+                databaseSchema=FullyQualifiedEntityName(
+                    fqn.build(
+                        metadata=self.metadata,
+                        entity_type=DatabaseSchema,
+                        service_name=self.context.get().database_service,
+                        database_name=self.context.get().database,
+                        schema_name=schema_name,
+                    )
                 ),
                 tags=self.get_tag_labels(
                     table_name=table_name
@@ -499,7 +547,7 @@ class CommonDbSourceService(
                     database_name=self.context.get().database,
                     table_type=table_type,
                 ),
-                owner=self.get_owner_ref(table_name=table_name),
+                owners=self.get_owner_ref(table_name=table_name),
             )
 
             is_partitioned, partition_details = self.get_table_partition_details(
@@ -516,18 +564,19 @@ class CommonDbSourceService(
 
             # Flag view as visited
             if table_type == TableType.View and schema_definition:
-                table_view = TableView.parse_obj(
-                    {
-                        "table_name": table_name,
-                        "schema_name": schema_name,
-                        "db_name": self.context.get().database,
-                        "view_definition": schema_definition,
-                    }
+                table_view = TableView(
+                    table_name=table_name,
+                    schema_name=schema_name,
+                    db_name=self.context.get().database,
+                    view_definition=schema_definition,
                 )
                 self.context.get_global().table_views.append(table_view)
 
         except Exception as exc:
-            error = f"Unexpected exception to yield table [{table_name}]: {exc}"
+            error = (
+                f"Unexpected exception to yield table "
+                f"(database=[{self.context.get().database}], schema=[{schema_name}], table=[{table_name}]): {exc}"
+            )
             yield Either(
                 left=StackTraceError(
                     name=table_name, error=error, stackTrace=traceback.format_exc()
@@ -535,18 +584,27 @@ class CommonDbSourceService(
             )
 
     @calculate_execution_time_generator()
-    def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
+    def yield_view_lineage(self) -> Iterable[Either[OMetaLineageRequest]]:
         logger.info("Processing Lineage for Views")
         for view in [
             v for v in self.context.get().table_views if v.view_definition is not None
         ]:
-            yield from get_view_lineage(
+            for lineage in get_view_lineage(
                 view=view,
                 metadata=self.metadata,
                 service_name=self.context.get().database_service,
                 connection_type=self.service_connection.type.value,
                 timeout_seconds=self.source_config.queryParsingTimeoutLimit,
-            )
+            ):
+                if lineage.right is not None:
+                    yield Either(
+                        right=OMetaLineageRequest(
+                            lineage_request=lineage.right,
+                            override_lineage=self.source_config.overrideViewLineage,
+                        )
+                    )
+                else:
+                    yield lineage
 
     def _get_foreign_constraints(self, foreign_columns) -> List[TableConstraint]:
         """
@@ -576,7 +634,7 @@ class CommonDbSourceService(
                         referred_table_fqn, referred_column, quote=False
                     )
                     if col_fqn:
-                        referred_column_fqns.append(col_fqn)
+                        referred_column_fqns.append(FullyQualifiedEntityName(col_fqn))
             else:
                 # do not build partial foreign constraint. It will updated in next run.
                 continue

metadata/ingestion/source/database/common_nosql_source.py

@@ -42,6 +42,7 @@ from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -73,7 +74,7 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
             self.config.sourceConfig.config
         )
         self.metadata = metadata
-        self.service_connection = self.config.serviceConnection.__root__.config
+        self.service_connection = self.config.serviceConnection.root.config
         self.connection_obj = get_connection(self.service_connection)
         self.test_connection()
 
@@ -103,7 +104,7 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
 
         yield Either(
             right=CreateDatabaseRequest(
-                name=database_name,
+                name=EntityName(database_name),
                 service=self.context.get().database_service,
                 sourceUrl=self.get_source_url(database_name=database_name),
             )
@@ -145,12 +146,14 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
 
         yield Either(
             right=CreateDatabaseSchemaRequest(
-                name=schema_name,
-                database=fqn.build(
-                    metadata=self.metadata,
-                    entity_type=Database,
-                    service_name=self.context.get().database_service,
-                    database_name=self.context.get().database,
+                name=EntityName(schema_name),
+                database=FullyQualifiedEntityName(
+                    fqn.build(
+                        metadata=self.metadata,
+                        entity_type=Database,
+                        service_name=self.context.get().database_service,
+                        database_name=self.context.get().database,
+                    )
                 ),
                 sourceUrl=self.get_source_url(
                     database_name=self.context.get().database,
@@ -166,7 +169,7 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
         need to be overridden by sources
         """
 
-    def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]:
+    def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, TableType]]]:
         """
         Handle table and views.
 
@@ -217,7 +220,7 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
         return None
 
     def yield_table(
-        self, table_name_and_type: Tuple[str, str]
+        self, table_name_and_type: Tuple[str, TableType]
     ) -> Iterable[Either[CreateTableRequest]]:
         """
         From topology.
@@ -233,7 +236,7 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
             column_parser = DataFrameColumnParser.create(df)
             columns = column_parser.get_columns()
             table_request = CreateTableRequest(
-                name=table_name,
+                name=EntityName(table_name),
                 tableType=table_type,
                 columns=columns,
                 tableConstraints=self.get_table_constraints(
@@ -241,12 +244,14 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
                     table_name=table_name,
                     db_name=self.context.get().database,
                 ),
-                databaseSchema=fqn.build(
-                    metadata=self.metadata,
-                    entity_type=DatabaseSchema,
-                    service_name=self.context.get().database_service,
-                    database_name=self.context.get().database,
-                    schema_name=schema_name,
+                databaseSchema=FullyQualifiedEntityName(
+                    fqn.build(
+                        metadata=self.metadata,
+                        entity_type=DatabaseSchema,
+                        service_name=self.context.get().database_service,
+                        database_name=self.context.get().database,
+                        schema_name=schema_name,
+                    )
                 ),
                 sourceUrl=self.get_source_url(
                     database_name=self.context.get().database,

metadata/ingestion/source/database/couchbase/connection.py

@@ -59,7 +59,7 @@ def test_connection(
     from couchbase.cluster import Cluster
 
     class SchemaHolder(BaseModel):
-        database: Optional[str]
+        database: Optional[str] = None
 
     holder = SchemaHolder()
 

metadata/ingestion/source/database/couchbase/metadata.py

@@ -50,8 +50,8 @@ class CouchbaseSource(CommonNoSQLSource):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: CouchbaseConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: CouchbaseConnection = config.serviceConnection.root.config
         if not isinstance(connection, CouchbaseConnection):
             raise InvalidSourceException(
                 f"Expected CouchbaseConnection, but got {connection}"
@@ -75,8 +75,8 @@ class CouchbaseSource(CommonNoSQLSource):
         Method to get list of schema names available within NoSQL db
         need to be overridden by sources
         """
+        database_name = self.context.get().database
         try:
-            database_name = self.context.get().database
             bucket = self.couchbase.bucket(database_name)
             collection_manager = bucket.collections()
             self.context.get().scope_dict = {

metadata/ingestion/source/database/database_service.py

@@ -15,8 +15,9 @@ import traceback
 from abc import ABC, abstractmethod
 from typing import Any, Iterable, List, Optional, Set, Tuple, Union
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 from sqlalchemy.engine import Inspector
+from typing_extensions import Annotated
 
 from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
 from metadata.generated.schema.api.data.createDatabaseSchema import (
@@ -51,7 +52,7 @@ from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.generated.schema.type.tagLabel import TagLabel
 from metadata.ingestion.api.delete import delete_entity_from_source
 from metadata.ingestion.api.models import Either
@@ -93,7 +94,9 @@ class DatabaseServiceTopology(ServiceTopology):
     data that has been produced by any parent node.
     """
 
-    root = TopologyNode(
+    root: Annotated[
+        TopologyNode, Field(description="Root node for the topology")
+    ] = TopologyNode(
         producer="get_services",
         stages=[
             NodeStage(
@@ -115,7 +118,9 @@ class DatabaseServiceTopology(ServiceTopology):
             "yield_external_table_lineage",
         ],
     )
-    database = TopologyNode(
+    database: Annotated[
+        TopologyNode, Field(description="Database Node")
+    ] = TopologyNode(
         producer="get_database_names",
         stages=[
             NodeStage(
@@ -136,7 +141,9 @@ class DatabaseServiceTopology(ServiceTopology):
         ],
         children=["databaseSchema"],
     )
-    databaseSchema = TopologyNode(
+    databaseSchema: Annotated[
+        TopologyNode, Field(description="Database Schema Node")
+    ] = TopologyNode(
         producer="get_database_schema_names",
         stages=[
             NodeStage(
@@ -159,7 +166,9 @@ class DatabaseServiceTopology(ServiceTopology):
         post_process=["mark_tables_as_deleted", "mark_stored_procedures_as_deleted"],
         threads=True,
     )
-    table = TopologyNode(
+    table: Annotated[
+        TopologyNode, Field(description="Main table processing logic")
+    ] = TopologyNode(
         producer="get_tables_name_and_type",
         stages=[
             NodeStage(
@@ -183,7 +192,9 @@ class DatabaseServiceTopology(ServiceTopology):
             ),
         ],
     )
-    stored_procedure = TopologyNode(
+    stored_procedure: Annotated[
+        TopologyNode, Field(description="Stored Procedure Node")
+    ] = TopologyNode(
         producer="get_stored_procedures",
         stages=[
             NodeStage(
@@ -212,7 +223,7 @@ class DatabaseServiceSource(
     database_source_state: Set = set()
     stored_procedure_source_state: Set = set()
     # Big union of types we want to fetch dynamically
-    service_connection: DatabaseConnection.__fields__["config"].type_
+    service_connection: DatabaseConnection.model_fields["config"].annotation
 
     # When processing the database, the source will update the inspector if needed
     inspector: Inspector
@@ -254,7 +265,7 @@ class DatabaseServiceSource(
         """
 
     @abstractmethod
-    def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]:
+    def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, TableType]]]:
         """
         Prepares the table name to be sent to stage.
         Filtering happens here.
@@ -305,7 +316,7 @@ class DatabaseServiceSource(
         """
 
     def yield_table_tag_details(
-        self, table_name_and_type: str
+        self, table_name_and_type: Tuple[str, TableType]
     ) -> Iterable[Either[OMetaTagAndClassification]]:
         """
         From topology. To be run for each table
@@ -387,11 +398,11 @@ class DatabaseServiceSource(
 
         tag_labels = []
         for tag_and_category in self.context.get().tags or []:
-            if tag_and_category.fqn and tag_and_category.fqn.__root__ == entity_fqn:
+            if tag_and_category.fqn and tag_and_category.fqn.root == entity_fqn:
                 tag_label = get_tag_label(
                     metadata=self.metadata,
-                    tag_name=tag_and_category.tag_request.name.__root__,
-                    classification_name=tag_and_category.classification_request.name.__root__,
+                    tag_name=tag_and_category.tag_request.name.root,
+                    classification_name=tag_and_category.classification_request.name.root,
                 )
                 if tag_label:
                     tag_labels.append(tag_label)
@@ -472,7 +483,7 @@ class DatabaseServiceSource(
             service_name=self.context.get().database_service,
             database_name=self.context.get().database,
             schema_name=self.context.get().database_schema,
-            table_name=table_request.name.__root__,
+            table_name=table_request.name.root,
             skip_es_search=True,
         )
 
@@ -490,7 +501,7 @@ class DatabaseServiceSource(
             service_name=self.context.get().database_service,
             database_name=self.context.get().database,
             schema_name=self.context.get().database_schema,
-            procedure_name=stored_proc_request.name.__root__,
+            procedure_name=stored_proc_request.name.root,
         )
 
         self.stored_procedure_source_state.add(table_fqn)
@@ -516,7 +527,7 @@ class DatabaseServiceSource(
             yield schema_fqn if return_fqn else schema_name
 
     @calculate_execution_time()
-    def get_owner_ref(self, table_name: str) -> Optional[EntityReference]:
+    def get_owner_ref(self, table_name: str) -> Optional[EntityReferenceList]:
         """
         Method to process the table owners
         """
@@ -529,7 +540,9 @@ class DatabaseServiceSource(
                     table_name=table_name,
                     schema=self.context.get().database_schema,
                 )
-                owner_ref = self.metadata.get_reference_by_name(name=owner_name)
+                owner_ref = self.metadata.get_reference_by_name(
+                    name=owner_name, is_owner=True
+                )
                 return owner_ref
         except Exception as exc:
             logger.debug(traceback.format_exc())

metadata/ingestion/source/database/databricks/connection.py

@@ -57,7 +57,7 @@ def get_connection(connection: DatabricksConnection) -> Engine:
     if connection.httpPath:
         if not connection.connectionArguments:
             connection.connectionArguments = init_empty_connection_arguments()
-        connection.connectionArguments.__root__["http_path"] = connection.httpPath
+        connection.connectionArguments.root["http_path"] = connection.httpPath
 
     return create_generic_db_connection(
         connection=connection,

metadata/ingestion/source/database/databricks/lineage.py

@@ -15,6 +15,7 @@ import traceback
 from datetime import datetime
 from typing import Iterator
 
+from metadata.generated.schema.type.basic import DateTime
 from metadata.generated.schema.type.tableQuery import TableQuery
 from metadata.ingestion.source.database.databricks.query_parser import (
     DatabricksQueryParserSource,
@@ -41,9 +42,9 @@ class DatabricksLineageSource(DatabricksQueryParserSource, LineageSource):
                     yield TableQuery(
                         query=row.get("query_text"),
                         userName=row.get("user_name"),
-                        startTime=row.get("query_start_time_ms"),
-                        endTime=row.get("execution_end_time_ms"),
-                        analysisDate=datetime.now(),
+                        startTime=str(row.get("query_start_time_ms")),
+                        endTime=str(row.get("execution_end_time_ms")),
+                        analysisDate=DateTime(datetime.now()),
                         serviceName=self.config.serviceName,
                     )
             except Exception as exc: