openmetadata-ingestion 1.3.1.2__py3-none-any.whl → 1.3.2.0__py3-none-any.whl

metadata/ingestion/api/parser.py

@@ -69,6 +69,28 @@ from metadata.generated.schema.metadataIngestion.databaseServiceQueryUsagePipeli
     DatabaseServiceQueryUsagePipeline,
     DatabaseUsageConfigType,
 )
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtAzureConfig import (
+    DbtAzureConfig,
+)
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtCloudConfig import (
+    DbtCloudConfig,
+)
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtGCSConfig import (
+    DbtGcsConfig,
+)
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtHttpConfig import (
+    DbtHttpConfig,
+)
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtLocalConfig import (
+    DbtLocalConfig,
+)
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtS3Config import (
+    DbtS3Config,
+)
+from metadata.generated.schema.metadataIngestion.dbtPipeline import (
+    DbtConfigType,
+    DbtPipeline,
+)
 from metadata.generated.schema.metadataIngestion.messagingServiceMetadataPipeline import (
     MessagingMetadataConfigType,
     MessagingServiceMetadataPipeline,
@@ -125,6 +147,16 @@ SOURCE_CONFIG_CLASS_MAP = {
     DatabaseMetadataConfigType.DatabaseMetadata.value: DatabaseServiceMetadataPipeline,
     StorageMetadataConfigType.StorageMetadata.value: StorageServiceMetadataPipeline,
     SearchMetadataConfigType.SearchMetadata.value: SearchServiceMetadataPipeline,
+    DbtConfigType.DBT.value: DbtPipeline,
+}
+
+DBT_CONFIG_TYPE_MAP = {
+    "cloud": DbtCloudConfig,
+    "local": DbtLocalConfig,
+    "http": DbtHttpConfig,
+    "s3": DbtS3Config,
+    "gcs": DbtGcsConfig,
+    "azure": DbtAzureConfig,
 }
 
 
@@ -171,6 +203,7 @@ def get_source_config_class(
     Type[PipelineServiceMetadataPipeline],
     Type[MlModelServiceMetadataPipeline],
     Type[DatabaseServiceMetadataPipeline],
+    Type[DbtPipeline],
 ]:
     """
     Return the source config type for a source string
@@ -179,7 +212,7 @@ def get_source_config_class(
     """
     source_config_class = SOURCE_CONFIG_CLASS_MAP.get(source_config_type)
 
-    if source_config_type:
+    if source_config_class:
         return source_config_class
 
     raise ValueError(f"Cannot find the service type of {source_config_type}")
@@ -266,6 +299,27 @@ def _unsafe_parse_config(config: dict, cls: Type[T], message: str) -> None:
         raise err
 
 
+def _unsafe_parse_dbt_config(config: dict, cls: Type[T], message: str) -> None:
+    """
+    Given a config dictionary and the class it should match,
+    try to parse it or log the given message
+    """
+    logger.debug(f"Parsing message: [{message}]")
+    try:
+        # Parse the oneOf config types of dbt to check
+        dbt_config_type = config["dbtConfigSource"]["dbtConfigType"]
+        dbt_config_class = DBT_CONFIG_TYPE_MAP.get(dbt_config_type)
+        dbt_config_class.parse_obj(config["dbtConfigSource"])
+
+        # Parse the entire dbtPipeline object
+        cls.parse_obj(config)
+    except ValidationError as err:
+        logger.debug(
+            f"The supported properties for {cls.__name__} are {list(cls.__fields__.keys())}"
+        )
+        raise err
+
+
 def _parse_inner_connection(config_dict: dict, source_type: str) -> None:
     """
     Parse the inner connection of the flagged connectors
@@ -291,32 +345,35 @@ def parse_service_connection(config_dict: dict) -> None:
     :param config_dict: JSON configuration
     """
     # Unsafe access to the keys. Allow a KeyError if the config is not well formatted
-    source_type = config_dict["source"]["serviceConnection"]["config"].get("type")
-    if source_type is None:
-        raise InvalidWorkflowException("Missing type in the serviceConnection config")
+    if config_dict["source"].get("serviceConnection"):
+        source_type = config_dict["source"]["serviceConnection"]["config"].get("type")
+        if source_type is None:
+            raise InvalidWorkflowException(
+                "Missing type in the serviceConnection config"
+            )
 
-    logger.debug(
-        f"Error parsing the Workflow Configuration for {source_type} ingestion"
-    )
+        logger.debug(
+            f"Error parsing the Workflow Configuration for {source_type} ingestion"
+        )
 
-    service_type = get_service_type(source_type)
-    connection_class = get_connection_class(source_type, service_type)
+        service_type = get_service_type(source_type)
+        connection_class = get_connection_class(source_type, service_type)
 
-    if source_type in HAS_INNER_CONNECTION:
-        # We will first parse the inner `connection` configuration
-        _parse_inner_connection(
-            config_dict["source"]["serviceConnection"]["config"]["connection"][
-                "config"
-            ]["connection"],
-            source_type,
-        )
+        if source_type in HAS_INNER_CONNECTION:
+            # We will first parse the inner `connection` configuration
+            _parse_inner_connection(
+                config_dict["source"]["serviceConnection"]["config"]["connection"][
+                    "config"
+                ]["connection"],
+                source_type,
+            )
 
-    # Parse the service connection dictionary with the scoped class
-    _unsafe_parse_config(
-        config=config_dict["source"]["serviceConnection"]["config"],
-        cls=connection_class,
-        message="Error parsing the service connection",
-    )
+        # Parse the service connection dictionary with the scoped class
+        _unsafe_parse_config(
+            config=config_dict["source"]["serviceConnection"]["config"],
+            cls=connection_class,
+            message="Error parsing the service connection",
+        )
 
 
 def parse_source_config(config_dict: dict) -> None:
@@ -334,6 +391,13 @@ def parse_source_config(config_dict: dict) -> None:
 
     source_config_class = get_source_config_class(source_config_type)
 
+    if source_config_class == DbtPipeline:
+        _unsafe_parse_dbt_config(
+            config=config_dict["source"]["sourceConfig"]["config"],
+            cls=source_config_class,
+            message="Error parsing the dbt source config",
+        )
+
     _unsafe_parse_config(
         config=config_dict["source"]["sourceConfig"]["config"],
         cls=source_config_class,

metadata/ingestion/api/topology_runner.py

@@ -265,11 +265,12 @@ class TopologyRunnerMixin(Generic[C]):
             if entity:
                 same_fingerprint = True
 
-        create_entity_request_hash = generate_source_hash(
-            create_request=entity_request.right,
-        )
+        create_entity_request_hash = None
 
         if hasattr(entity_request.right, "sourceHash"):
+            create_entity_request_hash = generate_source_hash(
+                create_request=entity_request.right,
+            )
             entity_request.right.sourceHash = create_entity_request_hash
 
         if entity is None and stage.use_cache:

metadata/ingestion/lineage/parser.py

@@ -217,6 +217,11 @@ class LineageParser:
         """
         aliases = self.table_aliases
         values = identifier.value.split(".")
+
+        if len(values) > 4:
+            logger.debug(f"Invalid comparison element from identifier: {identifier}")
+            return None, None
+
         database_name, schema_name, table_or_alias, column_name = (
             [None] * (4 - len(values))
         ) + values
@@ -307,29 +312,39 @@ class LineageParser:
                 comparisons.append(sub)
 
         for comparison in comparisons:
-            if "." not in comparison.left.value or "." not in comparison.right.value:
-                logger.debug(f"Ignoring comparison {comparison}")
-                continue
-
-            table_left, column_left = self.get_comparison_elements(
-                identifier=comparison.left
-            )
-            table_right, column_right = self.get_comparison_elements(
-                identifier=comparison.right
-            )
+            try:
+                if (
+                    "." not in comparison.left.value
+                    or "." not in comparison.right.value
+                ):
+                    logger.debug(f"Ignoring comparison {comparison}")
+                    continue
+
+                table_left, column_left = self.get_comparison_elements(
+                    identifier=comparison.left
+                )
+                table_right, column_right = self.get_comparison_elements(
+                    identifier=comparison.right
+                )
 
-            if not table_left or not table_right:
-                logger.warning(f"Cannot find ingredients from {comparison}")
-                continue
+                if not table_left or not table_right:
+                    logger.warning(
+                        f"Can't extract table names when parsing JOIN information from {comparison}"
+                    )
+                    logger.debug(f"Query: {sql_statement}")
+                    continue
 
-            left_table_column = TableColumn(table=table_left, column=column_left)
-            right_table_column = TableColumn(table=table_right, column=column_right)
+                left_table_column = TableColumn(table=table_left, column=column_left)
+                right_table_column = TableColumn(table=table_right, column=column_right)
 
-            # We just send the info once, from Left -> Right.
-            # The backend will prepare the symmetric information.
-            self.stateful_add_table_joins(
-                join_data, left_table_column, right_table_column
-            )
+                # We just send the info once, from Left -> Right.
+                # The backend will prepare the symmetric information.
+                self.stateful_add_table_joins(
+                    join_data, left_table_column, right_table_column
+                )
+            except Exception as exc:
+                logger.debug(f"Cannot process comparison {comparison}: {exc}")
+                logger.debug(traceback.format_exc())
 
     @cached_property
     def table_joins(self) -> Dict[str, List[TableColumnJoin]]:

metadata/ingestion/ometa/mixins/patch_mixin.py

@@ -46,7 +46,7 @@ from metadata.ingestion.ometa.mixins.patch_mixin_utils import (
 )
 from metadata.ingestion.ometa.utils import model_str
 from metadata.utils.deprecation import deprecated
-from metadata.utils.logger import ometa_logger
+from metadata.utils.logger import get_log_name, ometa_logger
 
 logger = ometa_logger()
 
@@ -155,9 +155,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
 
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.error(
-                f"Error trying to PATCH {entity.__name__} [{source.id.__root__}]: {exc}"
-            )
+            logger.error(f"Error trying to PATCH {get_log_name(source)}: {exc}")
 
         return None
 

metadata/ingestion/ometa/routes.py

@@ -90,6 +90,16 @@ from metadata.generated.schema.api.tests.createTestDefinition import (
 from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest
 from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart
 from metadata.generated.schema.dataInsight.kpi.kpi import Kpi
+from metadata.generated.schema.entity.applications.app import App
+from metadata.generated.schema.entity.applications.createAppRequest import (
+    CreateAppRequest,
+)
+from metadata.generated.schema.entity.applications.marketplace.appMarketPlaceDefinition import (
+    AppMarketPlaceDefinition,
+)
+from metadata.generated.schema.entity.applications.marketplace.createAppMarketPlaceDefinitionReq import (
+    CreateAppMarketPlaceDefinitionRequest,
+)
 from metadata.generated.schema.entity.automations.workflow import Workflow
 from metadata.generated.schema.entity.bot import Bot
 from metadata.generated.schema.entity.classification.classification import (
@@ -232,4 +242,9 @@ ROUTES = {
     # Suggestions
     Suggestion.__name__: "/suggestions",
     CreateSuggestionRequest.__name__: "/suggestions",
+    # Apps
+    App.__name__: "/apps",
+    CreateAppRequest.__name__: "/apps",
+    AppMarketPlaceDefinition.__name__: "/apps/marketplace",
+    CreateAppMarketPlaceDefinitionRequest.__name__: "/apps/marketplace",
 }

metadata/ingestion/source/dashboard/looker/utils.py

@@ -58,7 +58,7 @@ def _clone_repo(
         if isinstance(credential, GitHubCredentials):
             url = f"https://x-oauth-basic:{credential.token.__root__.get_secret_value()}@github.com/{repo_name}.git"
         elif isinstance(credential, BitBucketCredentials):
-            url = f"https://x-token-auth::{credential.token.__root__.get_secret_value()}@bitbucket.or/{repo_name}.git"
+            url = f"https://x-token-auth:{credential.token.__root__.get_secret_value()}@bitbucket.org/{repo_name}.git"
             allow_unsafe_protocols = True
 
         assert url is not None

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -19,6 +19,9 @@ from typing import List, Optional, Tuple
 
 import msal
 
+from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import (
+    PowerBIConnection,
+)
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.client import REST, ClientConfig
 from metadata.ingestion.source.dashboard.powerbi.models import (
@@ -52,7 +55,7 @@ class PowerBiApiClient:
 
     client: REST
 
-    def __init__(self, config):
+    def __init__(self, config: PowerBIConnection):
         self.config = config
         self.msal_client = msal.ConfidentialClientApplication(
             client_id=self.config.clientId,

metadata/ingestion/source/database/azuresql/connection.py

@@ -15,12 +15,13 @@ Source connection handler
 from typing import Optional, Union
 from urllib.parse import quote_plus
 
-from sqlalchemy.engine import Engine
+from sqlalchemy.engine import URL, Engine
 
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
 from metadata.generated.schema.entity.services.connections.database.azureSQLConnection import (
+    Authentication,
     AzureSQLConnection,
 )
 from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
@@ -40,13 +41,29 @@ def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) -
     Build the connection URL
     """
 
+    if connection.authenticationMode:
+        connection_string = f"Driver={connection.driver};Server={connection.hostPort};Database={connection.database};"
+        connection_string += f"Uid={connection.username};"
+        if (
+            connection.authenticationMode.authentication
+            == Authentication.ActiveDirectoryPassword
+        ):
+            connection_string += f"Pwd={connection.password.get_secret_value()};"
+
+        connection_string += f"Encrypt={'yes' if connection.authenticationMode.encrypt else 'no'};TrustServerCertificate={'yes' if connection.authenticationMode.trustServerCertificate else 'no'};"
+        connection_string += f"Connection Timeout={connection.authenticationMode.connectionTimeout or 30};Authentication={connection.authenticationMode.authentication.value};"
+
+        connection_url = URL.create(
+            "mssql+pyodbc", query={"odbc_connect": connection_string}
+        )
+        return connection_url
     url = f"{connection.scheme.value}://"
 
     if connection.username:
         url += f"{quote_plus(connection.username)}"
         url += (
             f":{quote_plus(connection.password.get_secret_value())}"
-            if connection
+            if connection.password
             else ""
         )
         url += "@"
@@ -54,12 +71,13 @@ def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) -
     url += f"{connection.hostPort}"
     url += f"/{quote_plus(connection.database)}" if connection.database else ""
     url += f"?driver={quote_plus(connection.driver)}"
+
     options = get_connection_options_dict(connection)
     if options:
         if not connection.database:
             url += "/"
         params = "&".join(
-            f"{key}={quote_plus(value)}" for (key, value) in options.items() if value
+            f"{key}={quote_plus(value)}" for key, value in options.items() if value
         )
         url = f"{url}?{params}"
 

metadata/ingestion/source/database/datalake/connection.py

@@ -20,6 +20,7 @@ from typing import Optional
 
 from google.cloud import storage
 
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
@@ -88,22 +89,9 @@ def _(config: GCSConfig):
 
 @get_datalake_client.register
 def _(config: AzureConfig):
-    from azure.identity import ClientSecretCredential
-    from azure.storage.blob import BlobServiceClient
 
     try:
-        credentials = ClientSecretCredential(
-            config.securityConfig.tenantId,
-            config.securityConfig.clientId,
-            config.securityConfig.clientSecret.get_secret_value(),
-        )
-
-        azure_client = BlobServiceClient(
-            f"https://{config.securityConfig.accountName}.blob.core.windows.net/",
-            credential=credentials,
-        )
-        return azure_client
-
+        return AzureClient(config.securityConfig).create_blob_client()
     except Exception as exc:
         raise RuntimeError(
             f"Unknown error connecting with {config.securityConfig}: {exc}."

metadata/ingestion/source/database/datalake/metadata.py

@@ -407,7 +407,7 @@ class DatalakeSource(DatabaseServiceSource):
         schema_name = self.context.database_schema
         try:
             table_constraints = None
-            data_frame = fetch_dataframe(
+            data_frame, raw_data = fetch_dataframe(
                 config_source=self.config_source,
                 client=self.client,
                 file_fqn=DatalakeTableSchemaWrapper(
@@ -415,10 +415,11 @@ class DatalakeSource(DatabaseServiceSource):
                     bucket_name=schema_name,
                     file_extension=table_extension,
                 ),
+                fetch_raw_data=True,
             )
             if data_frame:
                 column_parser = DataFrameColumnParser.create(
-                    data_frame[0], table_extension
+                    data_frame[0], table_extension, raw_data=raw_data
                 )
                 columns = column_parser.get_columns()
             else:

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -20,6 +20,7 @@ from typing import Dict, Iterable, List, Optional, Tuple
 import requests
 
 from metadata.clients.aws_client import AWSClient
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.metadataIngestion.dbtconfig.dbtAzureConfig import (
     DbtAzureConfig,
 )
@@ -172,7 +173,7 @@ def _(config: DbtCloudConfig):  # pylint: disable=too-many-locals
             params_data["job_definition_id"] = job_id
 
         response = client.get(f"/accounts/{account_id}/runs", data=params_data)
-        if not response and not response.get("data"):
+        if not response or not response.get("data"):
             raise DBTConfigException(
                 "Unable to get the dbt job runs information.\n"
                 "Please check if the auth token is correct and has the necessary scopes to fetch dbt runs"
@@ -357,21 +358,8 @@ def _(config: DbtGcsConfig):
 def _(config: DbtAzureConfig):
     try:
         bucket_name, prefix = get_dbt_prefix_config(config)
-        from azure.identity import (  # pylint: disable=import-outside-toplevel
-            ClientSecretCredential,
-        )
-        from azure.storage.blob import (  # pylint: disable=import-outside-toplevel
-            BlobServiceClient,
-        )
 
-        client = BlobServiceClient(
-            f"https://{config.dbtSecurityConfig.accountName}.blob.core.windows.net/",
-            credential=ClientSecretCredential(
-                config.dbtSecurityConfig.tenantId,
-                config.dbtSecurityConfig.clientId,
-                config.dbtSecurityConfig.clientSecret.get_secret_value(),
-            ),
-        )
+        client = AzureClient(config.dbtSecurityConfig).create_blob_client()
 
         if not bucket_name:
             container_dicts = client.list_containers()

metadata/ingestion/source/database/mssql/lineage.py

@@ -27,5 +27,6 @@ class MssqlLineageSource(MssqlQueryParserSource, LineageSource):
             OR lower(t.text) LIKE '%%merge%%'
         )
         AND lower(t.text) NOT LIKE '%%create%%procedure%%'
+        AND lower(t.text) NOT LIKE '%%create%%function%%'
         AND lower(t.text) NOT LIKE '%%declare%%'
     """

metadata/ingestion/source/database/mssql/usage.py

@@ -19,4 +19,8 @@ from metadata.ingestion.source.database.usage_source import UsageSource
 class MssqlUsageSource(MssqlQueryParserSource, UsageSource):
     sql_stmt = MSSQL_SQL_STATEMENT
 
-    filters = ""  # No filtering in the queries
+    filters = """
+        AND lower(t.text) NOT LIKE '%%create%%procedure%%'
+        AND lower(t.text) NOT LIKE '%%create%%function%%'
+        AND lower(t.text) NOT LIKE '%%declare%%'
+        """

metadata/ingestion/source/database/mysql/connection.py

@@ -16,9 +16,13 @@ from typing import Optional
 
 from sqlalchemy.engine import Engine
 
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
+from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
+    BasicAuth,
+)
 from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
     MysqlConnection,
 )
@@ -38,6 +42,16 @@ def get_connection(connection: MysqlConnection) -> Engine:
     """
     Create connection
     """
+    if hasattr(connection.authType, "azureConfig"):
+        azure_client = AzureClient(connection.authType.azureConfig).create_client()
+        if not connection.authType.azureConfig.scopes:
+            raise ValueError(
+                "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default"
+            )
+        access_token_obj = azure_client.get_token(
+            *connection.authType.azureConfig.scopes.split(",")
+        )
+        connection.authType = BasicAuth(password=access_token_obj.token)
     if connection.sslCA or connection.sslCert or connection.sslKey:
         if not connection.connectionOptions:
             connection.connectionOptions = init_empty_connection_options()

metadata/ingestion/source/database/oracle/connection.py

@@ -38,6 +38,7 @@ from metadata.ingestion.connections.builders import (
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.oracle.queries import CHECK_ACCESS_TO_DBA
 from metadata.utils.logger import ingestion_logger
 
 CX_ORACLE_LIB_VERSION = "8.3.0"
@@ -136,9 +137,13 @@ def test_connection(
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
+
+    test_conn_queries = {"CheckAccess": CHECK_ACCESS_TO_DBA}
+
     test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        queries=test_conn_queries,
     )

metadata/ingestion/source/database/oracle/queries.py

@@ -88,7 +88,7 @@ WHERE
     type = 'PROCEDURE' and owner = '{schema}'
 """
 )
-
+CHECK_ACCESS_TO_DBA = "SELECT table_name FROM DBA_TABLES where ROWNUM < 2"
 ORACLE_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent(
     """
 WITH SP_HISTORY AS (SELECT

metadata/ingestion/source/database/postgres/connection.py

@@ -17,9 +17,13 @@ from typing import Optional
 
 from sqlalchemy.engine import Engine
 
+from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
+from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
+    BasicAuth,
+)
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
     PostgresConnection,
     SslMode,
@@ -46,6 +50,17 @@ def get_connection(connection: PostgresConnection) -> Engine:
     """
     Create connection
     """
+
+    if hasattr(connection.authType, "azureConfig"):
+        azure_client = AzureClient(connection.authType.azureConfig).create_client()
+        if not connection.authType.azureConfig.scopes:
+            raise ValueError(
+                "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/postgresql/flexible-server/how-to-configure-sign-in-azure-ad-authentication#retrieve-the-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default"
+            )
+        access_token_obj = azure_client.get_token(
+            *connection.authType.azureConfig.scopes.split(",")
+        )
+        connection.authType = BasicAuth(password=access_token_obj.token)
     if connection.sslMode:
         if not connection.connectionArguments:
             connection.connectionArguments = init_empty_connection_arguments()

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -138,7 +138,7 @@ class StoredProcedureMixin(ABC):
             return True
 
         if query_type == "INSERT" and re.search(
-            "^.*insert.*into.*select.*$", query_text, re.IGNORECASE
+            "^.*insert.*into.*select.*$", query_text.replace("\n", " "), re.IGNORECASE
         ):
             return True
 

metadata/ingestion/source/database/unitycatalog/connection.py

@@ -68,16 +68,20 @@ def test_connection(
             break
 
     def get_schemas(connection: WorkspaceClient, table_obj: DatabricksTable):
-        for schema in connection.schemas.list(catalog_name=table_obj.catalog_name):
-            table_obj.schema_name = schema.name
-            break
+        for catalog in connection.catalogs.list():
+            for schema in connection.schemas.list(catalog_name=catalog.name):
+                if schema.name:
+                    table_obj.schema_name = schema.name
+                    table_obj.catalog_name = catalog.name
+                    return
 
     def get_tables(connection: WorkspaceClient, table_obj: DatabricksTable):
-        for table in connection.tables.list(
-            catalog_name=table_obj.catalog_name, schema_name=table_obj.schema_name
-        ):
-            table_obj.name = table.name
-            break
+        if table_obj.catalog_name and table_obj.schema_name:
+            for table in connection.tables.list(
+                catalog_name=table_obj.catalog_name, schema_name=table_obj.schema_name
+            ):
+                table_obj.name = table.name
+                break
 
     test_fn = {
         "CheckAccess": connection.catalogs.list,

metadata/ingestion/source/pipeline/dagster/metadata.py

@@ -214,7 +214,9 @@ class DagsterSource(PipelineServiceSource):
             service_name=self.context.pipeline_service,
             pipeline_name=self.context.pipeline,
         )
-        pipeline_entity = self.metadata.get_by_name(entity=Pipeline, fqn=pipeline_fqn)
+        pipeline_entity = self.metadata.get_by_name(
+            entity=Pipeline, fqn=pipeline_fqn, fields=["tasks"]
+        )
         for task in pipeline_entity.tasks or []:
             try:
                 runs = self.client.get_task_runs(