openmetadata-ingestion 1.3.1.3__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/great_expectations/action.py

@@ -15,6 +15,7 @@ Open Metadata table quality.
 This subpackage needs to be used in Great Expectations
 checkpoints actions.
 """
+import logging
 import traceback
 from datetime import datetime, timezone
 from typing import Dict, List, Optional, Union, cast
@@ -72,9 +73,10 @@ from metadata.great_expectations.utils.ometa_config_handler import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils import fqn
 from metadata.utils.entity_link import get_entity_link
-from metadata.utils.logger import great_expectations_logger
 
-logger = great_expectations_logger()
+logger = logging.getLogger(
+    "great_expectations.validation_operators.validation_operators.openmetadata"
+)
 
 
 class OpenMetadataValidationAction(ValidationAction):
@@ -429,7 +431,7 @@ class OpenMetadataValidationAction(ValidationAction):
                 test_case_fqn=test_case.fullyQualifiedName.__root__,
             )
 
-            logger.info(
+            logger.debug(
                 f"Test case result for {test_case.fullyQualifiedName.__root__} successfully ingested"
             )
 

metadata/ingestion/lineage/parser.py

@@ -319,7 +319,10 @@ class LineageParser:
             )
 
             if not table_left or not table_right:
-                logger.warning(f"Cannot find ingredients from {comparison}")
+                logger.warning(
+                    f"Can't extract table names when parsing JOIN information from {comparison}"
+                )
+                logger.debug(f"Query: {sql_statement}")
                 continue
 
             left_table_column = TableColumn(table=table_left, column=column_left)

metadata/ingestion/ometa/mixins/patch_mixin.py

@@ -46,7 +46,7 @@ from metadata.ingestion.ometa.mixins.patch_mixin_utils import (
 )
 from metadata.ingestion.ometa.utils import model_str
 from metadata.utils.deprecation import deprecated
-from metadata.utils.logger import ometa_logger
+from metadata.utils.logger import get_log_name, ometa_logger
 
 logger = ometa_logger()
 
@@ -155,9 +155,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
 
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.error(
-                f"Error trying to PATCH {entity.__name__} [{source.id.__root__}]: {exc}"
-            )
+            logger.error(f"Error trying to PATCH {get_log_name(source)}: {exc}")
 
         return None
 

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -19,6 +19,9 @@ from typing import List, Optional, Tuple
 
 import msal
 
+from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import (
+    PowerBIConnection,
+)
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.client import REST, ClientConfig
 from metadata.ingestion.source.dashboard.powerbi.models import (
@@ -52,7 +55,7 @@ class PowerBiApiClient:
 
     client: REST
 
-    def __init__(self, config):
+    def __init__(self, config: PowerBIConnection):
         self.config = config
         self.msal_client = msal.ConfidentialClientApplication(
             client_id=self.config.clientId,

metadata/ingestion/source/database/azuresql/connection.py

@@ -15,12 +15,13 @@ Source connection handler
 from typing import Optional, Union
 from urllib.parse import quote_plus
 
-from sqlalchemy.engine import Engine
+from sqlalchemy.engine import URL, Engine
 
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
 from metadata.generated.schema.entity.services.connections.database.azureSQLConnection import (
+    Authentication,
     AzureSQLConnection,
 )
 from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
@@ -40,13 +41,29 @@ def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) -
     Build the connection URL
     """
 
+    if connection.authenticationMode:
+        connection_string = f"Driver={connection.driver};Server={connection.hostPort};Database={connection.database};"
+        connection_string += f"Uid={connection.username};"
+        if (
+            connection.authenticationMode.authentication
+            == Authentication.ActiveDirectoryPassword
+        ):
+            connection_string += f"Pwd={connection.password.get_secret_value()};"
+
+        connection_string += f"Encrypt={'yes' if connection.authenticationMode.encrypt else 'no'};TrustServerCertificate={'yes' if connection.authenticationMode.trustServerCertificate else 'no'};"
+        connection_string += f"Connection Timeout={connection.authenticationMode.connectionTimeout or 30};Authentication={connection.authenticationMode.authentication.value};"
+
+        connection_url = URL.create(
+            "mssql+pyodbc", query={"odbc_connect": connection_string}
+        )
+        return connection_url
     url = f"{connection.scheme.value}://"
 
     if connection.username:
         url += f"{quote_plus(connection.username)}"
         url += (
             f":{quote_plus(connection.password.get_secret_value())}"
-            if connection
+            if connection.password
             else ""
         )
         url += "@"
@@ -54,12 +71,13 @@ def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) -
     url += f"{connection.hostPort}"
     url += f"/{quote_plus(connection.database)}" if connection.database else ""
     url += f"?driver={quote_plus(connection.driver)}"
+
     options = get_connection_options_dict(connection)
     if options:
         if not connection.database:
             url += "/"
         params = "&".join(
-            f"{key}={quote_plus(value)}" for (key, value) in options.items() if value
+            f"{key}={quote_plus(value)}" for key, value in options.items() if value
         )
         url = f"{url}?{params}"
 

metadata/ingestion/source/database/datalake/connection.py

@@ -20,6 +20,7 @@ from typing import Optional
 
 from google.cloud import storage
 
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
@@ -88,22 +89,9 @@ def _(config: GCSConfig):
 
 @get_datalake_client.register
 def _(config: AzureConfig):
-    from azure.identity import ClientSecretCredential
-    from azure.storage.blob import BlobServiceClient
 
     try:
-        credentials = ClientSecretCredential(
-            config.securityConfig.tenantId,
-            config.securityConfig.clientId,
-            config.securityConfig.clientSecret.get_secret_value(),
-        )
-
-        azure_client = BlobServiceClient(
-            f"https://{config.securityConfig.accountName}.blob.core.windows.net/",
-            credential=credentials,
-        )
-        return azure_client
-
+        return AzureClient(config.securityConfig).create_blob_client()
     except Exception as exc:
         raise RuntimeError(
             f"Unknown error connecting with {config.securityConfig}: {exc}."

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -20,6 +20,7 @@ from typing import Dict, Iterable, List, Optional, Tuple
 import requests
 
 from metadata.clients.aws_client import AWSClient
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.metadataIngestion.dbtconfig.dbtAzureConfig import (
     DbtAzureConfig,
 )
@@ -357,21 +358,8 @@ def _(config: DbtGcsConfig):
 def _(config: DbtAzureConfig):
     try:
         bucket_name, prefix = get_dbt_prefix_config(config)
-        from azure.identity import (  # pylint: disable=import-outside-toplevel
-            ClientSecretCredential,
-        )
-        from azure.storage.blob import (  # pylint: disable=import-outside-toplevel
-            BlobServiceClient,
-        )
 
-        client = BlobServiceClient(
-            f"https://{config.dbtSecurityConfig.accountName}.blob.core.windows.net/",
-            credential=ClientSecretCredential(
-                config.dbtSecurityConfig.tenantId,
-                config.dbtSecurityConfig.clientId,
-                config.dbtSecurityConfig.clientSecret.get_secret_value(),
-            ),
-        )
+        client = AzureClient(config.dbtSecurityConfig).create_blob_client()
 
         if not bucket_name:
             container_dicts = client.list_containers()

metadata/ingestion/source/database/mssql/lineage.py

@@ -27,5 +27,6 @@ class MssqlLineageSource(MssqlQueryParserSource, LineageSource):
             OR lower(t.text) LIKE '%%merge%%'
         )
         AND lower(t.text) NOT LIKE '%%create%%procedure%%'
+        AND lower(t.text) NOT LIKE '%%create%%function%%'
         AND lower(t.text) NOT LIKE '%%declare%%'
     """

metadata/ingestion/source/database/mssql/usage.py

@@ -19,4 +19,8 @@ from metadata.ingestion.source.database.usage_source import UsageSource
 class MssqlUsageSource(MssqlQueryParserSource, UsageSource):
     sql_stmt = MSSQL_SQL_STATEMENT
 
-    filters = ""  # No filtering in the queries
+    filters = """
+        AND lower(t.text) NOT LIKE '%%create%%procedure%%'
+        AND lower(t.text) NOT LIKE '%%create%%function%%'
+        AND lower(t.text) NOT LIKE '%%declare%%'
+        """

metadata/ingestion/source/database/mysql/connection.py

@@ -16,9 +16,13 @@ from typing import Optional
 
 from sqlalchemy.engine import Engine
 
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
+from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
+    BasicAuth,
+)
 from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
     MysqlConnection,
 )
@@ -38,6 +42,16 @@ def get_connection(connection: MysqlConnection) -> Engine:
     """
     Create connection
     """
+    if hasattr(connection.authType, "azureConfig"):
+        azure_client = AzureClient(connection.authType.azureConfig).create_client()
+        if not connection.authType.azureConfig.scopes:
+            raise ValueError(
+                "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default"
+            )
+        access_token_obj = azure_client.get_token(
+            *connection.authType.azureConfig.scopes.split(",")
+        )
+        connection.authType = BasicAuth(password=access_token_obj.token)
     if connection.sslCA or connection.sslCert or connection.sslKey:
         if not connection.connectionOptions:
             connection.connectionOptions = init_empty_connection_options()

metadata/ingestion/source/database/postgres/connection.py

@@ -17,9 +17,13 @@ from typing import Optional
 
 from sqlalchemy.engine import Engine
 
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
+from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
+    BasicAuth,
+)
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
     PostgresConnection,
     SslMode,
@@ -46,6 +50,17 @@ def get_connection(connection: PostgresConnection) -> Engine:
     """
     Create connection
     """
+
+    if hasattr(connection.authType, "azureConfig"):
+        azure_client = AzureClient(connection.authType.azureConfig).create_client()
+        if not connection.authType.azureConfig.scopes:
+            raise ValueError(
+                "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/postgresql/flexible-server/how-to-configure-sign-in-azure-ad-authentication#retrieve-the-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default"
+            )
+        access_token_obj = azure_client.get_token(
+            *connection.authType.azureConfig.scopes.split(",")
+        )
+        connection.authType = BasicAuth(password=access_token_obj.token)
     if connection.sslMode:
         if not connection.connectionArguments:
             connection.connectionArguments = init_empty_connection_arguments()

metadata/pii/scanners/ner_scanner.py

@@ -21,7 +21,7 @@ from pydantic import BaseModel
 
 from metadata.generated.schema.entity.classification.tag import Tag
 from metadata.pii.constants import PII, SPACY_EN_MODEL
-from metadata.pii.models import TagAndConfidence, TagType
+from metadata.pii.models import TagAndConfidence
 from metadata.pii.ner import NEREntity
 from metadata.utils import fqn
 from metadata.utils.logger import pii_logger
@@ -119,13 +119,15 @@ class NERScanner:
 
         if entities_score:
             label, score = self.get_highest_score_label(entities_score)
-            tag_type = NEREntity.__members__.get(label, TagType.NONSENSITIVE).value
+            tag_type = NEREntity.__members__.get(label)
+            if not tag_type:
+                return None
             return TagAndConfidence(
                 tag_fqn=fqn.build(
                     metadata=None,
                     entity_type=Tag,
                     classification_name=PII,
-                    tag_name=tag_type,
+                    tag_name=tag_type.value,
                 ),
                 confidence=score,
             )

metadata/profiler/source/databricks/profiler_source.py

@@ -0,0 +1,36 @@
+"""Extend the ProfilerSource class to add support for Databricks is_disconnect SQA method"""
+
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.metadataIngestion.workflow import (
+    OpenMetadataWorkflowConfig,
+)
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.profiler.source.base.profiler_source import ProfilerSource
+
+
+def is_disconnect(self, e, connection, cursor):
+    """is_disconnect method for the Databricks dialect"""
+    if "Invalid SessionHandle: SessionHandle" in str(e):
+        return True
+    return False
+
+
+class DataBricksProfilerSource(ProfilerSource):
+    """Databricks Profiler source"""
+
+    def __init__(
+        self,
+        config: OpenMetadataWorkflowConfig,
+        database: DatabaseService,
+        ometa_client: OpenMetadata,
+    ):
+        super().__init__(config, database, ometa_client)
+        self.set_is_disconnect()
+
+    def set_is_disconnect(self):
+        """Set the is_disconnect method for the Databricks dialect"""
+        from databricks.sqlalchemy import (
+            DatabricksDialect,  # pylint: disable=import-outside-toplevel
+        )
+
+        DatabricksDialect.is_disconnect = is_disconnect

metadata/profiler/source/metadata.py

@@ -43,6 +43,10 @@ from metadata.utils.logger import profiler_logger
 logger = profiler_logger()
 
 
+TABLE_FIELDS = ["tableProfilerConfig", "columns", "customMetrics"]
+TAGS_FIELD = ["tags"]
+
+
 class ProfilerSourceAndEntity(BaseModel):
     """Return class for the OpenMetadata Profiler Source"""
 
@@ -273,7 +277,9 @@ class OpenMetadataSource(Source):
         """
         tables = self.metadata.list_all_entities(
             entity=Table,
-            fields=["tableProfilerConfig", "columns", "customMetrics"],
+            fields=TABLE_FIELDS
+            if not self.source_config.processPiiSensitive
+            else TABLE_FIELDS + TAGS_FIELD,
             params={
                 "service": self.config.source.serviceName,
                 "database": fqn.build(

metadata/profiler/source/profiler_source_factory.py

@@ -16,8 +16,12 @@ Factory class for creating profiler source objects
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigqueryType,
 )
+from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
+    DatabricksType,
+)
 from metadata.profiler.source.base.profiler_source import ProfilerSource
 from metadata.profiler.source.bigquery.profiler_source import BigQueryProfilerSource
+from metadata.profiler.source.databricks.profiler_source import DataBricksProfilerSource
 
 
 class ProfilerSourceFactory:
@@ -44,3 +48,7 @@ profiler_source_factory.register_source(
     BigqueryType.BigQuery.value.lower(),
     BigQueryProfilerSource,
 )
+profiler_source_factory.register_source(
+    DatabricksType.Databricks.value.lower(),
+    DataBricksProfilerSource,
+)

metadata/utils/credentials.py

@@ -15,7 +15,7 @@ import base64
 import json
 import os
 import tempfile
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union
 
 from cryptography.hazmat.primitives import serialization
 from google import auth
@@ -25,6 +25,9 @@ from metadata.generated.schema.security.credentials.gcpCredentials import (
     GCPCredentials,
     GcpCredentialsPath,
 )
+from metadata.generated.schema.security.credentials.gcpExternalAccount import (
+    GcpExternalAccount,
+)
 from metadata.generated.schema.security.credentials.gcpValues import (
     GcpCredentialsValues,
 )
@@ -85,30 +88,44 @@ def create_credential_tmp_file(credentials: dict) -> str:
         return temp_file_path
 
 
-def build_google_credentials_dict(gcp_values: GcpCredentialsValues) -> Dict[str, str]:
+def build_google_credentials_dict(
+    gcp_values: Union[GcpCredentialsValues, GcpExternalAccount]
+) -> Dict[str, str]:
     """
     Given GcPCredentialsValues, build a dictionary as the JSON file
     downloaded from GCP with the service_account
     :param gcp_values: GCP credentials
     :return: Dictionary with credentials
     """
-    private_key_str = gcp_values.privateKey.get_secret_value()
-    # adding the replace string here to escape line break if passed from env
-    private_key_str = private_key_str.replace("\\n", "\n")
-    validate_private_key(private_key_str)
-
-    return {
-        "type": gcp_values.type,
-        "project_id": gcp_values.projectId.__root__,
-        "private_key_id": gcp_values.privateKeyId,
-        "private_key": private_key_str,
-        "client_email": gcp_values.clientEmail,
-        "client_id": gcp_values.clientId,
-        "auth_uri": str(gcp_values.authUri),
-        "token_uri": str(gcp_values.tokenUri),
-        "auth_provider_x509_cert_url": str(gcp_values.authProviderX509CertUrl),
-        "client_x509_cert_url": str(gcp_values.clientX509CertUrl),
-    }
+    if isinstance(gcp_values, GcpCredentialsValues):
+        private_key_str = gcp_values.privateKey.get_secret_value()
+        # adding the replace string here to escape line break if passed from env
+        private_key_str = private_key_str.replace("\\n", "\n")
+        validate_private_key(private_key_str)
+
+        return {
+            "type": gcp_values.type,
+            "project_id": gcp_values.projectId.__root__,
+            "private_key_id": gcp_values.privateKeyId,
+            "private_key": private_key_str,
+            "client_email": gcp_values.clientEmail,
+            "client_id": gcp_values.clientId,
+            "auth_uri": str(gcp_values.authUri),
+            "token_uri": str(gcp_values.tokenUri),
+            "auth_provider_x509_cert_url": str(gcp_values.authProviderX509CertUrl),
+            "client_x509_cert_url": str(gcp_values.clientX509CertUrl),
+        }
+    if isinstance(gcp_values, GcpExternalAccount):
+        return {
+            "type": gcp_values.externalType,
+            "audience": gcp_values.audience,
+            "subject_token_type": gcp_values.subjectTokenType,
+            "token_url": gcp_values.tokenURL,
+            "credential_source": gcp_values.credentialSource,
+        }
+    raise InvalidGcpConfigException(
+        f"Error trying to build GCP credentials dict due to Invalid GCP config {type(gcp_values)}"
+    )
 
 
 def set_google_credentials(gcp_credentials: GCPCredentials) -> None:

metadata/utils/secrets/azure_kv_secrets_manager.py

@@ -17,9 +17,9 @@ import traceback
 from abc import ABC
 from typing import Optional
 
-from azure.identity import ClientSecretCredential, DefaultAzureCredential
-from azure.keyvault.secrets import KeyVaultSecret, SecretClient
+from azure.keyvault.secrets import KeyVaultSecret
 
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.security.secrets.secretsManagerClientLoader import (
     SecretsManagerClientLoader,
 )
@@ -105,23 +105,7 @@ class AzureKVSecretsManager(ExternalSecretsManager, ABC):
     ):
         super().__init__(provider=SecretsManagerProvider.azure_kv, loader=loader)
 
-        if (
-            self.credentials.tenantId
-            and self.credentials.clientId
-            and self.credentials.clientSecret
-        ):
-            azure_identity = ClientSecretCredential(
-                tenant_id=self.credentials.tenantId,
-                client_id=self.credentials.clientId,
-                client_secret=self.credentials.clientSecret.get_secret_value(),
-            )
-        else:
-            azure_identity = DefaultAzureCredential()
-
-        self.client = SecretClient(
-            vault_url=f"https://{self.credentials.vaultName}.vault.azure.net/",
-            credential=azure_identity,
-        )
+        self.client = AzureClient(self.credentials).create_secret_client()
 
     def get_string_value(self, secret_id: str) -> str:
         """

metadata/utils/storage_metadata_config.py

@@ -17,6 +17,7 @@ from functools import singledispatch
 
 import requests
 
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.services.connections.database.datalake.azureConfig import (
     AzureConfig,
 )
@@ -153,21 +154,7 @@ def _(config: StorageMetadataAdlsConfig) -> ManifestMetadataConfig:
             else STORAGE_METADATA_MANIFEST_FILE_NAME
         )
 
-        from azure.identity import (  # pylint: disable=import-outside-toplevel
-            ClientSecretCredential,
-        )
-        from azure.storage.blob import (  # pylint: disable=import-outside-toplevel
-            BlobServiceClient,
-        )
-
-        blob_client = BlobServiceClient(
-            account_url=f"https://{config.securityConfig.accountName}.blob.core.windows.net/",
-            credential=ClientSecretCredential(
-                config.securityConfig.tenantId,
-                config.securityConfig.clientId,
-                config.securityConfig.clientSecret.get_secret_value(),
-            ),
-        )
+        blob_client = AzureClient(config.securityConfig).create_blob_client()
 
         reader = get_reader(
             config_source=AzureConfig(securityConfig=config.securityConfig),

metadata/workflow/base.py

@@ -108,7 +108,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
     @property
     def ingestion_pipeline(self):
         """Get or create the Ingestion Pipeline from the configuration"""
-        if not self._ingestion_pipeline:
+        if not self._ingestion_pipeline and self.config.ingestionPipelineFQN:
             self._ingestion_pipeline = self.get_or_create_ingestion_pipeline()
 
         return self._ingestion_pipeline