openmetadata-ingestion 1.7.0.0rc2__py3-none-any.whl → 1.7.1.0__py3-none-any.whl

metadata/ingestion/source/database/snowflake/metadata.py

@@ -116,7 +116,11 @@ from metadata.ingestion.source.database.snowflake.utils import (
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_database
 from metadata.utils.logger import ingestion_logger
-from metadata.utils.sqlalchemy_utils import get_all_table_comments, get_all_table_ddls
+from metadata.utils.sqlalchemy_utils import (
+    get_all_table_comments,
+    get_all_table_ddls,
+    get_all_view_definitions,
+)
 from metadata.utils.tag_utils import get_ometa_tag_and_classification
 
 ischema_names["VARIANT"] = VARIANT
@@ -134,6 +138,7 @@ SnowflakeDialect.get_stream_names = get_stream_names
 SnowflakeDialect.get_all_table_comments = get_all_table_comments
 SnowflakeDialect.normalize_name = normalize_names
 SnowflakeDialect.get_table_comment = get_table_comment
+SnowflakeDialect.get_all_view_definitions = get_all_view_definitions
 SnowflakeDialect.get_view_definition = get_view_definition
 SnowflakeDialect.get_unique_constraints = get_unique_constraints
 SnowflakeDialect._get_schema_columns = (  # pylint: disable=protected-access
@@ -164,6 +169,8 @@ class SnowflakeSource(
     Database metadata from Snowflake Source
     """
 
+    service_connection: SnowflakeConnection
+
     def __init__(
         self,
         config,
@@ -487,6 +494,7 @@ class SnowflakeSource(
         snowflake_tables = self.inspector.get_table_names(
             schema=schema_name,
             incremental=self.incremental,
+            account_usage=self.service_connection.accountUsageSchema,
             **table_type_to_params_map[table_type],
         )
 
@@ -635,6 +643,7 @@ class SnowflakeSource(
         snowflake_views = self.inspector.get_view_names(
             schema=schema_name,
             incremental=self.incremental,
+            account_usage=self.service_connection.accountUsageSchema,
             materialized_views=materialized_views,
         )
 
@@ -678,24 +687,28 @@ class SnowflakeSource(
     def _get_stored_procedures_internal(
         self, query: str
     ) -> Iterable[SnowflakeStoredProcedure]:
-        results = self.engine.execute(
-            query.format(
-                database_name=self.context.get().database,
-                schema_name=self.context.get().database_schema,
-                account_usage=self.service_connection.accountUsageSchema,
-            )
-        ).all()
-        for row in results:
-            stored_procedure = SnowflakeStoredProcedure.model_validate(dict(row))
-            if stored_procedure.definition is None:
-                logger.debug(
-                    f"Missing ownership permissions on procedure {stored_procedure.name}."
-                    " Trying to fetch description via DESCRIBE."
-                )
-                stored_procedure.definition = self.describe_procedure_definition(
-                    stored_procedure
+        try:
+            results = self.engine.execute(
+                query.format(
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
+                    account_usage=self.service_connection.accountUsageSchema,
                 )
-            yield stored_procedure
+            ).all()
+            for row in results:
+                stored_procedure = SnowflakeStoredProcedure.model_validate(dict(row))
+                if stored_procedure.definition is None:
+                    logger.debug(
+                        f"Missing ownership permissions on procedure {stored_procedure.name}."
+                        " Trying to fetch description via DESCRIBE."
+                    )
+                    stored_procedure.definition = self.describe_procedure_definition(
+                        stored_procedure
+                    )
+                yield stored_procedure
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.error(f"Error fetching stored procedures: {exc}")
 
     def get_stored_procedures(self) -> Iterable[SnowflakeStoredProcedure]:
         """List Snowflake stored procedures"""
@@ -715,19 +728,27 @@ class SnowflakeSource(
         Then, if the procedure is created with `EXECUTE AS CALLER`, we can still try to
         get the definition with a DESCRIBE.
         """
-        if stored_procedure.procedure_type == StoredProcedureType.StoredProcedure.value:
-            query = SNOWFLAKE_DESC_STORED_PROCEDURE
-        else:
-            query = SNOWFLAKE_DESC_FUNCTION
-        res = self.engine.execute(
-            query.format(
-                database_name=self.context.get().database,
-                schema_name=self.context.get().database_schema,
-                procedure_name=stored_procedure.name,
-                procedure_signature=stored_procedure.unquote_signature(),
+        try:
+            if (
+                stored_procedure.procedure_type
+                == StoredProcedureType.StoredProcedure.value
+            ):
+                query = SNOWFLAKE_DESC_STORED_PROCEDURE
+            else:
+                query = SNOWFLAKE_DESC_FUNCTION
+            res = self.engine.execute(
+                query.format(
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
+                    procedure_name=stored_procedure.name,
+                    procedure_signature=stored_procedure.unquote_signature(),
+                )
             )
-        )
-        return dict(res.all()).get("body", "")
+            return dict(res.all()).get("body", "")
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.error(f"Error fetching stored procedure definition: {exc}")
+            return ""
 
     def yield_stored_procedure(
         self, stored_procedure: SnowflakeStoredProcedure
@@ -835,6 +856,21 @@ class SnowflakeSource(
     ) -> Optional[str]:
         """
         Get the DDL statement, View Definition or Stream Definition for a table
+
+        To fetch the view definition, we have followed an optimised approach
+        i.e. fetching view definition of all the views in schema storing it
+        in cache and using the same cache to fetch the view definition.
+
+        To fetch defintion for other types of tables, we have used the
+        get_ddl method, since this method only accepts string literal as arguments
+        it is not possible to do something like this:
+
+        select table_name, schema, get_ddl('table', table_name) from information_schema.tables
+        so we have to fetch the ddl for each table individually.
+
+        Alternavies are executing an stroed procedure to automate this but
+        it requires additional permissions like execute which users may not be comfortable doing.
+        Or reconstruct the ddl from column types, which we can explore in the future.
         """
         try:
             schema_definition = None
@@ -846,7 +882,7 @@ class SnowflakeSource(
                 schema_definition = inspector.get_stream_definition(
                     self.connection, table_name, schema_name
                 )
-            elif hasattr(inspector, "get_table_ddl") and self.source_config.includeDDL:
+            elif self.source_config.includeDDL or table_type == TableType.Dynamic:
                 schema_definition = inspector.get_table_ddl(
                     self.connection, table_name, schema_name
                 )
@@ -862,3 +898,13 @@ class SnowflakeSource(
             logger.debug(f"Failed to fetch schema definition for {table_name}: {exc}")
 
         return None
+
+    def get_life_cycle_query(self):
+        """
+        Get the life cycle query
+        """
+        return self.life_cycle_query.format(
+            database_name=self.context.get().database,
+            schema_name=self.context.get().database_schema,
+            account_usage=self.service_connection.accountUsageSchema,
+        )

metadata/ingestion/source/database/snowflake/queries.py

@@ -60,7 +60,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where TABLE_CATALOG = '{database}'
       and TABLE_SCHEMA = '{schema}'
       and TABLE_TYPE = 'EXTERNAL TABLE'
@@ -86,7 +86,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where TABLE_CATALOG = '{database}'
     and TABLE_SCHEMA = '{schema}'
     and TABLE_TYPE = 'BASE TABLE'
@@ -111,7 +111,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where  TABLE_CATALOG = '{database}'
     and TABLE_SCHEMA = '{schema}'
     and TABLE_TYPE = 'VIEW'
@@ -134,7 +134,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where  TABLE_CATALOG = '{database}'
     and TABLE_SCHEMA = '{schema}'
     and TABLE_TYPE = 'MATERIALIZED VIEW'
@@ -171,7 +171,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where TABLE_CATALOG = '{database}'
     and TABLE_SCHEMA = '{schema}'
     and TABLE_TYPE = 'BASE TABLE'
@@ -197,7 +197,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where TABLE_CATALOG = '{database}'
     and TABLE_SCHEMA = '{schema}'
     and TABLE_TYPE = 'BASE TABLE'
@@ -297,7 +297,7 @@ SNOWFLAKE_LIFE_CYCLE_QUERY = textwrap.dedent(
 select
 table_name as table_name,
 created as created_at
-from snowflake.account_usage.tables
+from {account_usage}.tables
 where table_schema = '{schema_name}'
 and table_catalog = '{database_name}'
 """
@@ -407,6 +407,14 @@ SELECT GET_DDL('TABLE','{table_name}') AS \"text\"
 """
 
 SNOWFLAKE_GET_VIEW_DEFINITION = """
+SELECT table_name "view_name",
+    table_schema "schema",
+    view_definition "view_def"
+FROM information_schema.views
+WHERE view_definition is not null
+"""
+
+SNOWFLAKE_GET_VIEW_DDL = """
 SELECT GET_DDL('VIEW','{view_name}') AS \"text\"
 """
 

metadata/ingestion/source/database/snowflake/service_spec.py

@@ -1,6 +1,9 @@
 from metadata.data_quality.interface.sqlalchemy.snowflake.test_suite_interface import (
     SnowflakeTestSuiteInterface,
 )
+from metadata.ingestion.source.database.snowflake.data_diff.data_diff import (
+    SnowflakeTableParameter,
+)
 from metadata.ingestion.source.database.snowflake.lineage import SnowflakeLineageSource
 from metadata.ingestion.source.database.snowflake.metadata import SnowflakeSource
 from metadata.ingestion.source.database.snowflake.profiler.profiler import (
@@ -17,4 +20,5 @@ ServiceSpec = DefaultDatabaseSpec(
     profiler_class=SnowflakeProfiler,
     test_suite_class=SnowflakeTestSuiteInterface,
     sampler_class=SnowflakeSampler,
+    data_diff=SnowflakeTableParameter,
 )

metadata/ingestion/source/database/snowflake/utils.py

@@ -41,6 +41,7 @@ from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_GET_STREAM_NAMES,
     SNOWFLAKE_GET_TABLE_DDL,
     SNOWFLAKE_GET_TRANSIENT_NAMES,
+    SNOWFLAKE_GET_VIEW_DDL,
     SNOWFLAKE_GET_VIEW_DEFINITION,
     SNOWFLAKE_GET_VIEW_NAMES,
     SNOWFLAKE_GET_WITHOUT_TRANSIENT_TABLE_NAMES,
@@ -56,6 +57,7 @@ from metadata.utils import fqn
 from metadata.utils.sqlalchemy_utils import (
     get_display_datatype,
     get_table_comment_wrapper,
+    get_view_definition_wrapper,
 )
 
 dialect = SnowflakeDialect()
@@ -187,7 +189,11 @@ def _get_query_map(
 
 
 def _get_query_parameters(
-    self, connection, schema: str, incremental: Optional[IncrementalConfig]
+    self,
+    connection,
+    schema: str,
+    incremental: Optional[IncrementalConfig],
+    account_usage: Optional[str] = None,
 ):
     """Returns the proper query parameters depending if the extraction is Incremental or Full"""
     parameters = {"schema": fqn.unquote_name(schema)}
@@ -198,6 +204,7 @@ def _get_query_parameters(
             **parameters,
             "date": incremental.start_timestamp,
             "database": database,
+            "account_usage": account_usage or "SNOWFLAKE.ACCOUNT_USAGE",
         }
 
     return parameters
@@ -206,9 +213,12 @@ def _get_query_parameters(
 def get_table_names(self, connection, schema: str, **kw):
     """Return the Table names to process based on the incremental setup."""
     incremental = kw.get("incremental")
+    account_usage = kw.get("account_usage")
 
     queries = _get_query_map(incremental, TABLE_QUERY_MAPS)
-    parameters = _get_query_parameters(self, connection, schema, incremental)
+    parameters = _get_query_parameters(
+        self, connection, schema, incremental, account_usage
+    )
 
     query = queries["default"]
 
@@ -233,9 +243,12 @@ def get_table_names(self, connection, schema: str, **kw):
 
 def get_view_names(self, connection, schema, **kw):
     incremental = kw.get("incremental")
+    account_usage = kw.get("account_usage")
 
     queries = _get_query_map(incremental, VIEW_QUERY_MAPS)
-    parameters = _get_query_parameters(self, connection, schema, incremental)
+    parameters = _get_query_parameters(
+        self, connection, schema, incremental, account_usage
+    )
 
     if kw.get("materialized_views"):
         query = queries["materialized_views"]
@@ -271,22 +284,29 @@ def get_stream_names(self, connection, schema, **kw):
 
 
 @reflection.cache
-def get_view_definition(  # pylint: disable=unused-argument
-    self, connection, view_name, schema=None, **kw
-):
-    """
-    Gets the view definition
-    """
-    schema = schema or self.default_schema_name
-    view_name = f"{schema}.{view_name}" if schema else view_name
-    cursor = connection.execute(
-        SNOWFLAKE_GET_VIEW_DEFINITION.format(view_name=view_name)
+def get_view_definition(
+    self, connection, table_name, schema=None, **kw
+):  # pylint: disable=unused-argument
+    view_definition = get_view_definition_wrapper(
+        self,
+        connection,
+        table_name=table_name,
+        schema=schema,
+        query=SNOWFLAKE_GET_VIEW_DEFINITION,
     )
-    n2i = self.__class__._map_name_to_idx(cursor)  # pylint: disable=protected-access
+    if view_definition:
+        return view_definition
+
+    # If the view definition is not found via optimized query,
+    # we need to get the view definition from the view ddl
+
+    schema = schema or self.default_schema_name
+    view_name = f"{schema}.{table_name}" if schema else table_name
+    cursor = connection.execute(SNOWFLAKE_GET_VIEW_DDL.format(view_name=view_name))
     try:
-        ret = cursor.fetchone()
-        if ret:
-            return ret[n2i["text"]]
+        result = cursor.fetchone()
+        if result:
+            return result[0]
     except Exception:
         pass
     return None

metadata/ingestion/source/database/unitycatalog/metadata.py

@@ -72,7 +72,6 @@ from metadata.ingestion.source.database.unitycatalog.models import (
     ForeignConstrains,
     Type,
 )
-from metadata.ingestion.source.models import TableView
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_database, filter_by_schema, filter_by_table
 from metadata.utils.helpers import retry_with_docker_host
@@ -97,7 +96,6 @@ class UnitycatalogSource(
         self.source_config: DatabaseServiceMetadataPipeline = (
             self.config.sourceConfig.config
         )
-        self.context.get_global().table_views = []
         self.metadata = metadata
         self.service_connection: UnityCatalogConnection = (
             self.config.serviceConnection.root.config
@@ -346,19 +344,6 @@ class UnitycatalogSource(
             )
             yield Either(right=table_request)
 
-            if table_type == TableType.View or table.view_definition:
-                self.context.get_global().table_views.append(
-                    TableView(
-                        table_name=table_name,
-                        schema_name=schema_name,
-                        db_name=db_name,
-                        view_definition=(
-                            f'CREATE VIEW "{db_name}"."{schema_name}"'
-                            f'."{table_name}" AS {table.view_definition}'
-                        ),
-                    )
-                )
-
             self.register_record(table_request=table_request)
         except Exception as exc:
             yield Either(

metadata/ingestion/source/database/vertica/queries.py

@@ -28,20 +28,6 @@ import textwrap
 # v_catalog.comments with v_catalog.columns.
 VERTICA_GET_COLUMNS = textwrap.dedent(
     """
-        WITH column_projection as (
-          SELECT 
-            column_id,
-            proj.projection_name
-        FROM v_catalog.columns col,
-          v_catalog.projections proj
-        where lower(table_name) = '{table}' 
-            AND {schema_condition}
-            AND proj.projection_id in (
-              select 
-                min(projection_id) 
-              from v_catalog.projections sub_proj
-              where col.table_id=sub_proj.anchor_table_id
-        ))
         select
           column_name,
           data_type,
@@ -50,12 +36,11 @@ VERTICA_GET_COLUMNS = textwrap.dedent(
           comment 
         from 
           v_catalog.columns col
-          LEFT JOIN column_projection proj ON proj.column_id = col.column_id
-          LEFT JOIN v_catalog.comments com ON com.object_type = 'COLUMN'
-          AND com.object_name = CONCAT(
-            CONCAT(proj.projection_name, '.'),
-            col.column_name
-          )
+          LEFT JOIN v_catalog.comments cm
+          ON col.table_schema = cm.object_schema
+          AND col.table_name = cm.object_name
+          AND col.column_name = cm.child_object
+          AND cm.object_type = 'COLUMN'
         WHERE 
           lower(table_name) = '{table}'
           AND {schema_condition}

metadata/ingestion/source/messaging/kinesis/metadata.py

@@ -185,6 +185,9 @@ class KinesisSource(MessagingServiceSource):
         try:
             while has_more_partitions:
                 partitions = self.kinesis.list_shards(**args.dict())
+                # Handle the case when NextToken is not present
+                if "NextToken" not in partitions:
+                    partitions["NextToken"] = None
                 kinesis_partitions_model = KinesisPartitions(**partitions)
                 all_partitions.extend(
                     [

metadata/ingestion/source/pipeline/airbyte/constants.py

@@ -0,0 +1,29 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Constants for Airbyte
+"""
+
+from enum import Enum
+
+
+class AirbyteSource(Enum):
+    MYSQL = "MySQL"
+    POSTGRES = "Postgres"
+    MSSQL = "Microsoft SQL Server (MSSQL)"
+    MONGODB = "MongoDb"
+
+
+class AirbyteDestination(Enum):
+    MYSQL = "MySQL"
+    POSTGRES = "Postgres"
+    MSSQL = "MS SQL Server"

metadata/ingestion/source/pipeline/airbyte/metadata.py

@@ -29,7 +29,6 @@ from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.pipeline.airbyteConnection import (
     AirbyteConnection,
 )
-from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
@@ -46,12 +45,16 @@ from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.pipeline.openlineage.models import TableDetails
+from metadata.ingestion.source.pipeline.openlineage.utils import FQNNotFoundException
 from metadata.ingestion.source.pipeline.pipeline_service import PipelineServiceSource
 from metadata.utils import fqn
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.time_utils import convert_timestamp_to_milliseconds
 
+from .utils import get_destination_table_details, get_source_table_details
+
 logger = ingestion_logger()
 
 
@@ -189,6 +192,26 @@ class AirbyteSource(PipelineServiceSource):
                     )
                 )
 
+    def _get_table_fqn(self, table_details: TableDetails) -> Optional[str]:
+        """
+        Get the FQN of the table
+        """
+        try:
+            if self.get_db_service_names():
+                return self._get_table_fqn_from_om(table_details)
+
+            return fqn.build(
+                metadata=self.metadata,
+                entity_type=Table,
+                service_name="*",
+                database_name=table_details.database,
+                schema_name=table_details.schema,
+                table_name=table_details.name,
+            )
+        except FQNNotFoundException:
+            return None
+
+    # pylint: disable=too-many-locals
     def yield_pipeline_lineage_details(
         self, pipeline_details: AirbytePipelineDetails
     ) -> Iterable[Either[AddLineageRequest]]:
@@ -197,47 +220,65 @@ class AirbyteSource(PipelineServiceSource):
         :param pipeline_details: pipeline_details object from airbyte
         :return: Lineage from inlets and outlets
         """
+        pipeline_name = pipeline_details.connection.get("name")
         source_connection = self.client.get_source(
             pipeline_details.connection.get("sourceId")
         )
         destination_connection = self.client.get_destination(
             pipeline_details.connection.get("destinationId")
         )
-        source_service = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=source_connection.get("name")
-        )
-        destination_service = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=destination_connection.get("name")
-        )
-        if not source_service or not destination_service:
-            return
+        source_name = source_connection.get("sourceName")
+        destination_name = destination_connection.get("destinationName")
 
         for task in (
             pipeline_details.connection.get("syncCatalog", {}).get("streams") or []
         ):
             stream = task.get("stream")
-            from_fqn = fqn.build(
-                self.metadata,
-                Table,
-                table_name=stream.get("name"),
-                database_name=None,
-                schema_name=stream.get("namespace"),
-                service_name=source_connection.get("name"),
-            )
 
-            to_fqn = fqn.build(
-                self.metadata,
-                Table,
-                table_name=stream.get("name"),
-                database_name=None,
-                schema_name=stream.get("namespace"),
-                service_name=destination_connection.get("name"),
+            source_table_details = get_source_table_details(stream, source_connection)
+            destination_table_details = get_destination_table_details(
+                stream, destination_connection
             )
 
+            if not source_table_details or not destination_table_details:
+                continue
+
+            from_fqn = self._get_table_fqn(source_table_details)
+            to_fqn = self._get_table_fqn(destination_table_details)
+
+            if not from_fqn:
+                logger.warning(
+                    f"While extracting lineage: [{pipeline_name}],"
+                    f" source table: [{source_table_details.database or '*'}]"
+                    f".[{source_table_details.schema}].[{source_table_details.name}]"
+                    f" (type: {source_name}) not found in openmetadata"
+                )
+                continue
+            if not to_fqn:
+                logger.warning(
+                    f"While extracting lineage: [{pipeline_name}],"
+                    f" destination table: [{destination_table_details.database or '*'}]"
+                    f".[{destination_table_details.schema}].[{destination_table_details.name}]"
+                    f" (type: {destination_name}) not found in openmetadata"
+                )
+                continue
+
             from_entity = self.metadata.get_by_name(entity=Table, fqn=from_fqn)
             to_entity = self.metadata.get_by_name(entity=Table, fqn=to_fqn)
 
-            if not from_entity and not to_entity:
+            if not from_entity:
+                logger.warning(
+                    f"While extracting lineage: [{pipeline_name}],"
+                    f" source table (fqn: [{from_fqn}], type: {source_name}) not found"
+                    " in openmetadata"
+                )
+                continue
+            if not to_entity:
+                logger.warning(
+                    f"While extracting lineage: [{pipeline_name}],"
+                    f" destination table (fqn: [{to_fqn}], type: {destination_name}) not found"
+                    " in openmetadata"
+                )
                 continue
 
             pipeline_fqn = fqn.build(
@@ -279,4 +320,4 @@ class AirbyteSource(PipelineServiceSource):
         """
         Get Pipeline Name
         """
-        return pipeline_details.connection.get("connectionId")
+        return pipeline_details.connection.get("name")