openmetadata-ingestion 1.3.1.3__py3-none-any.whl → 1.3.2.0rc2__py3-none-any.whl

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-26T16:14:58+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-26T16:14:58+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-26T16:14:58+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-26T16:14:58+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-26T16:14:58+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-26T16:14:58+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-26T16:14:58+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-26T16:14:58+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-26T16:14:58+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-26T16:14:58+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-26T16:14:58+00:00
 
 from __future__ import annotations
 

metadata/great_expectations/action.py

@@ -15,6 +15,7 @@ Open Metadata table quality.
 This subpackage needs to be used in Great Expectations
 checkpoints actions.
 """
+import logging
 import traceback
 from datetime import datetime, timezone
 from typing import Dict, List, Optional, Union, cast
@@ -72,9 +73,10 @@ from metadata.great_expectations.utils.ometa_config_handler import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils import fqn
 from metadata.utils.entity_link import get_entity_link
-from metadata.utils.logger import great_expectations_logger
 
-logger = great_expectations_logger()
+logger = logging.getLogger(
+    "great_expectations.validation_operators.validation_operators.openmetadata"
+)
 
 
 class OpenMetadataValidationAction(ValidationAction):
@@ -429,7 +431,7 @@ class OpenMetadataValidationAction(ValidationAction):
                 test_case_fqn=test_case.fullyQualifiedName.__root__,
             )
 
-            logger.info(
+            logger.debug(
                 f"Test case result for {test_case.fullyQualifiedName.__root__} successfully ingested"
             )
 

metadata/ingestion/api/topology_runner.py

@@ -265,11 +265,12 @@ class TopologyRunnerMixin(Generic[C]):
             if entity:
                 same_fingerprint = True
 
-        create_entity_request_hash = generate_source_hash(
-            create_request=entity_request.right,
-        )
+        create_entity_request_hash = None
 
         if hasattr(entity_request.right, "sourceHash"):
+            create_entity_request_hash = generate_source_hash(
+                create_request=entity_request.right,
+            )
             entity_request.right.sourceHash = create_entity_request_hash
 
         if entity is None and stage.use_cache:

metadata/ingestion/lineage/parser.py

@@ -319,7 +319,10 @@ class LineageParser:
             )
 
             if not table_left or not table_right:
-                logger.warning(f"Cannot find ingredients from {comparison}")
+                logger.warning(
+                    f"Can't extract table names when parsing JOIN information from {comparison}"
+                )
+                logger.debug(f"Query: {sql_statement}")
                 continue
 
             left_table_column = TableColumn(table=table_left, column=column_left)

metadata/ingestion/ometa/mixins/patch_mixin.py

@@ -46,7 +46,7 @@ from metadata.ingestion.ometa.mixins.patch_mixin_utils import (
 )
 from metadata.ingestion.ometa.utils import model_str
 from metadata.utils.deprecation import deprecated
-from metadata.utils.logger import ometa_logger
+from metadata.utils.logger import get_log_name, ometa_logger
 
 logger = ometa_logger()
 
@@ -155,9 +155,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
 
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.error(
-                f"Error trying to PATCH {entity.__name__} [{source.id.__root__}]: {exc}"
-            )
+            logger.error(f"Error trying to PATCH {get_log_name(source)}: {exc}")
 
         return None
 

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -19,6 +19,9 @@ from typing import List, Optional, Tuple
 
 import msal
 
+from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import (
+    PowerBIConnection,
+)
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.client import REST, ClientConfig
 from metadata.ingestion.source.dashboard.powerbi.models import (
@@ -52,7 +55,7 @@ class PowerBiApiClient:
 
     client: REST
 
-    def __init__(self, config):
+    def __init__(self, config: PowerBIConnection):
         self.config = config
         self.msal_client = msal.ConfidentialClientApplication(
             client_id=self.config.clientId,

metadata/ingestion/source/database/azuresql/connection.py

@@ -15,12 +15,13 @@ Source connection handler
 from typing import Optional, Union
 from urllib.parse import quote_plus
 
-from sqlalchemy.engine import Engine
+from sqlalchemy.engine import URL, Engine
 
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
 from metadata.generated.schema.entity.services.connections.database.azureSQLConnection import (
+    Authentication,
     AzureSQLConnection,
 )
 from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
@@ -40,13 +41,29 @@ def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) -
     Build the connection URL
     """
 
+    if connection.authenticationMode:
+        connection_string = f"Driver={connection.driver};Server={connection.hostPort};Database={connection.database};"
+        connection_string += f"Uid={connection.username};"
+        if (
+            connection.authenticationMode.authentication
+            == Authentication.ActiveDirectoryPassword
+        ):
+            connection_string += f"Pwd={connection.password.get_secret_value()};"
+
+        connection_string += f"Encrypt={'yes' if connection.authenticationMode.encrypt else 'no'};TrustServerCertificate={'yes' if connection.authenticationMode.trustServerCertificate else 'no'};"
+        connection_string += f"Connection Timeout={connection.authenticationMode.connectionTimeout or 30};Authentication={connection.authenticationMode.authentication.value};"
+
+        connection_url = URL.create(
+            "mssql+pyodbc", query={"odbc_connect": connection_string}
+        )
+        return connection_url
     url = f"{connection.scheme.value}://"
 
     if connection.username:
         url += f"{quote_plus(connection.username)}"
         url += (
             f":{quote_plus(connection.password.get_secret_value())}"
-            if connection
+            if connection.password
             else ""
         )
         url += "@"
@@ -54,12 +71,13 @@ def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) -
     url += f"{connection.hostPort}"
     url += f"/{quote_plus(connection.database)}" if connection.database else ""
     url += f"?driver={quote_plus(connection.driver)}"
+
     options = get_connection_options_dict(connection)
     if options:
         if not connection.database:
             url += "/"
         params = "&".join(
-            f"{key}={quote_plus(value)}" for (key, value) in options.items() if value
+            f"{key}={quote_plus(value)}" for key, value in options.items() if value
         )
         url = f"{url}?{params}"
 

metadata/ingestion/source/database/datalake/connection.py

@@ -20,6 +20,7 @@ from typing import Optional
 
 from google.cloud import storage
 
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
@@ -88,22 +89,9 @@ def _(config: GCSConfig):
 
 @get_datalake_client.register
 def _(config: AzureConfig):
-    from azure.identity import ClientSecretCredential
-    from azure.storage.blob import BlobServiceClient
 
     try:
-        credentials = ClientSecretCredential(
-            config.securityConfig.tenantId,
-            config.securityConfig.clientId,
-            config.securityConfig.clientSecret.get_secret_value(),
-        )
-
-        azure_client = BlobServiceClient(
-            f"https://{config.securityConfig.accountName}.blob.core.windows.net/",
-            credential=credentials,
-        )
-        return azure_client
-
+        return AzureClient(config.securityConfig).create_blob_client()
     except Exception as exc:
         raise RuntimeError(
             f"Unknown error connecting with {config.securityConfig}: {exc}."

metadata/ingestion/source/database/datalake/metadata.py

@@ -407,7 +407,7 @@ class DatalakeSource(DatabaseServiceSource):
         schema_name = self.context.database_schema
         try:
             table_constraints = None
-            data_frame = fetch_dataframe(
+            data_frame, raw_data = fetch_dataframe(
                 config_source=self.config_source,
                 client=self.client,
                 file_fqn=DatalakeTableSchemaWrapper(
@@ -415,10 +415,11 @@ class DatalakeSource(DatabaseServiceSource):
                     bucket_name=schema_name,
                     file_extension=table_extension,
                 ),
+                fetch_raw_data=True,
             )
             if data_frame:
                 column_parser = DataFrameColumnParser.create(
-                    data_frame[0], table_extension
+                    data_frame[0], table_extension, raw_data=raw_data
                 )
                 columns = column_parser.get_columns()
             else:

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -20,6 +20,7 @@ from typing import Dict, Iterable, List, Optional, Tuple
 import requests
 
 from metadata.clients.aws_client import AWSClient
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.metadataIngestion.dbtconfig.dbtAzureConfig import (
     DbtAzureConfig,
 )
@@ -357,21 +358,8 @@ def _(config: DbtGcsConfig):
 def _(config: DbtAzureConfig):
     try:
         bucket_name, prefix = get_dbt_prefix_config(config)
-        from azure.identity import (  # pylint: disable=import-outside-toplevel
-            ClientSecretCredential,
-        )
-        from azure.storage.blob import (  # pylint: disable=import-outside-toplevel
-            BlobServiceClient,
-        )
 
-        client = BlobServiceClient(
-            f"https://{config.dbtSecurityConfig.accountName}.blob.core.windows.net/",
-            credential=ClientSecretCredential(
-                config.dbtSecurityConfig.tenantId,
-                config.dbtSecurityConfig.clientId,
-                config.dbtSecurityConfig.clientSecret.get_secret_value(),
-            ),
-        )
+        client = AzureClient(config.dbtSecurityConfig).create_blob_client()
 
         if not bucket_name:
             container_dicts = client.list_containers()

metadata/ingestion/source/database/mssql/lineage.py

@@ -27,5 +27,6 @@ class MssqlLineageSource(MssqlQueryParserSource, LineageSource):
             OR lower(t.text) LIKE '%%merge%%'
         )
         AND lower(t.text) NOT LIKE '%%create%%procedure%%'
+        AND lower(t.text) NOT LIKE '%%create%%function%%'
         AND lower(t.text) NOT LIKE '%%declare%%'
     """

metadata/ingestion/source/database/mssql/usage.py

@@ -19,4 +19,8 @@ from metadata.ingestion.source.database.usage_source import UsageSource
 class MssqlUsageSource(MssqlQueryParserSource, UsageSource):
     sql_stmt = MSSQL_SQL_STATEMENT
 
-    filters = ""  # No filtering in the queries
+    filters = """
+        AND lower(t.text) NOT LIKE '%%create%%procedure%%'
+        AND lower(t.text) NOT LIKE '%%create%%function%%'
+        AND lower(t.text) NOT LIKE '%%declare%%'
+        """

metadata/ingestion/source/database/mysql/connection.py

@@ -16,9 +16,13 @@ from typing import Optional
 
 from sqlalchemy.engine import Engine
 
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
+from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
+    BasicAuth,
+)
 from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
     MysqlConnection,
 )
@@ -38,6 +42,16 @@ def get_connection(connection: MysqlConnection) -> Engine:
     """
     Create connection
     """
+    if hasattr(connection.authType, "azureConfig"):
+        azure_client = AzureClient(connection.authType.azureConfig).create_client()
+        if not connection.authType.azureConfig.scopes:
+            raise ValueError(
+                "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default"
+            )
+        access_token_obj = azure_client.get_token(
+            *connection.authType.azureConfig.scopes.split(",")
+        )
+        connection.authType = BasicAuth(password=access_token_obj.token)
     if connection.sslCA or connection.sslCert or connection.sslKey:
         if not connection.connectionOptions:
             connection.connectionOptions = init_empty_connection_options()

metadata/ingestion/source/database/postgres/connection.py

@@ -17,9 +17,13 @@ from typing import Optional
 
 from sqlalchemy.engine import Engine
 
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
+from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
+    BasicAuth,
+)
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
     PostgresConnection,
     SslMode,
@@ -46,6 +50,17 @@ def get_connection(connection: PostgresConnection) -> Engine:
     """
     Create connection
     """
+
+    if hasattr(connection.authType, "azureConfig"):
+        azure_client = AzureClient(connection.authType.azureConfig).create_client()
+        if not connection.authType.azureConfig.scopes:
+            raise ValueError(
+                "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/postgresql/flexible-server/how-to-configure-sign-in-azure-ad-authentication#retrieve-the-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default"
+            )
+        access_token_obj = azure_client.get_token(
+            *connection.authType.azureConfig.scopes.split(",")
+        )
+        connection.authType = BasicAuth(password=access_token_obj.token)
     if connection.sslMode:
         if not connection.connectionArguments:
             connection.connectionArguments = init_empty_connection_arguments()

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -138,7 +138,7 @@ class StoredProcedureMixin(ABC):
             return True
 
         if query_type == "INSERT" and re.search(
-            "^.*insert.*into.*select.*$", query_text, re.IGNORECASE
+            "^.*insert.*into.*select.*$", query_text.replace("\n", " "), re.IGNORECASE
         ):
             return True
 

metadata/ingestion/source/storage/storage_service.py

@@ -260,7 +260,7 @@ class StorageServiceSource(TopologyRunnerMixin, Source, ABC):
         metadata_entry: MetadataEntry,
     ) -> List[Column]:
         """Extract Column related metadata from s3"""
-        data_structure_details = fetch_dataframe(
+        data_structure_details, raw_data = fetch_dataframe(
             config_source=config_source,
             client=client,
             file_fqn=DatalakeTableSchemaWrapper(
@@ -269,10 +269,13 @@ class StorageServiceSource(TopologyRunnerMixin, Source, ABC):
                 file_extension=SupportedTypes(metadata_entry.structureFormat),
                 separator=metadata_entry.separator,
             ),
+            fetch_raw_data=True,
         )
         columns = []
         column_parser = DataFrameColumnParser.create(
-            data_structure_details, SupportedTypes(metadata_entry.structureFormat)
+            data_structure_details,
+            SupportedTypes(metadata_entry.structureFormat),
+            raw_data=raw_data,
         )
         columns = column_parser.get_columns()
         return columns

metadata/parsers/json_schema_parser.py

@@ -18,6 +18,8 @@ import traceback
 from enum import Enum
 from typing import List, Optional
 
+from pydantic.main import ModelMetaclass
+
 from metadata.generated.schema.type.schema import FieldModel
 from metadata.utils.logger import ingestion_logger
 
@@ -36,20 +38,25 @@ class JsonSchemaDataTypes(Enum):
     NULL = "null"
     RECORD = "object"
     ARRAY = "array"
+    UNKNOWN = "unknown"
 
 
-def parse_json_schema(schema_text: str) -> Optional[List[FieldModel]]:
+def parse_json_schema(
+    schema_text: str, cls: ModelMetaclass = FieldModel
+) -> Optional[List[FieldModel]]:
     """
     Method to parse the jsonschema
     """
     try:
         json_schema_data = json.loads(schema_text)
         field_models = [
-            FieldModel(
+            cls(
                 name=json_schema_data.get("title", "default"),
                 dataType=JsonSchemaDataTypes(json_schema_data.get("type")).name,
                 description=json_schema_data.get("description"),
-                children=get_json_schema_fields(json_schema_data.get("properties")),
+                children=get_json_schema_fields(
+                    json_schema_data.get("properties", {}), cls=cls
+                ),
             )
         ]
         return field_models
@@ -59,7 +66,9 @@ def parse_json_schema(schema_text: str) -> Optional[List[FieldModel]]:
     return None
 
 
-def get_json_schema_fields(properties) -> Optional[List[FieldModel]]:
+def get_json_schema_fields(
+    properties, cls: ModelMetaclass = FieldModel
+) -> Optional[List[FieldModel]]:
     """
     Recursively convert the parsed schema into required models
     """
@@ -67,9 +76,10 @@ def get_json_schema_fields(properties) -> Optional[List[FieldModel]]:
     for key, value in properties.items():
         try:
             field_models.append(
-                FieldModel(
-                    name=value.get("title", key),
-                    dataType=JsonSchemaDataTypes(value.get("type")).name,
+                cls(
+                    name=key,
+                    displayName=value.get("title"),
+                    dataType=JsonSchemaDataTypes(value.get("type", "unknown")).name,
                     description=value.get("description"),
                     children=get_json_schema_fields(value.get("properties"))
                     if value.get("type") == "object"

metadata/pii/scanners/ner_scanner.py

@@ -21,7 +21,7 @@ from pydantic import BaseModel
 
 from metadata.generated.schema.entity.classification.tag import Tag
 from metadata.pii.constants import PII, SPACY_EN_MODEL
-from metadata.pii.models import TagAndConfidence, TagType
+from metadata.pii.models import TagAndConfidence
 from metadata.pii.ner import NEREntity
 from metadata.utils import fqn
 from metadata.utils.logger import pii_logger
@@ -119,13 +119,15 @@ class NERScanner:
 
         if entities_score:
             label, score = self.get_highest_score_label(entities_score)
-            tag_type = NEREntity.__members__.get(label, TagType.NONSENSITIVE).value
+            tag_type = NEREntity.__members__.get(label)
+            if not tag_type:
+                return None
             return TagAndConfidence(
                 tag_fqn=fqn.build(
                     metadata=None,
                     entity_type=Tag,
                     classification_name=PII,
-                    tag_name=tag_type,
+                    tag_name=tag_type.value,
                 ),
                 confidence=score,
             )

metadata/profiler/interface/profiler_interface.py

@@ -33,7 +33,7 @@ from metadata.generated.schema.entity.data.table import (
     TableData,
 )
 from metadata.generated.schema.entity.services.connections.connectionBasicType import (
-    SampleDataStorageConfig,
+    DataStorageConfig,
 )
 from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
     DatalakeConnection,
@@ -93,7 +93,7 @@ class ProfilerInterface(ABC):
         service_connection_config: Union[DatabaseConnection, DatalakeConnection],
         ometa_client: OpenMetadata,
         entity: Table,
-        storage_config: SampleDataStorageConfig,
+        storage_config: DataStorageConfig,
         profile_sample_config: Optional[ProfileSampleConfig],
         source_config: DatabaseServiceProfilerPipeline,
         sample_query: Optional[str],
@@ -248,7 +248,7 @@ class ProfilerInterface(ABC):
             DatabaseProfilerConfig,
             DatabaseAndSchemaConfig,
         ]
-    ):
+    ) -> Optional[DataStorageConfig]:
         if (
             config
             and config.sampleDataStorageConfig
@@ -264,7 +264,7 @@ class ProfilerInterface(ABC):
         database_profiler_config: Optional[DatabaseProfilerConfig],
         db_service: Optional[DatabaseService],
         profiler_config: ProfilerProcessorConfig,
-    ) -> Optional[SampleDataStorageConfig]:
+    ) -> Optional[DataStorageConfig]:
         """Get config for a specific entity
 
         Args: