openmetadata-ingestion 1.6.0.0rc1__py3-none-any.whl → 1.6.0.0rc3__py3-none-any.whl

metadata/ingestion/source/database/snowflake/lineage.py

@@ -12,8 +12,10 @@
 Snowflake lineage module
 """
 
-from typing import Dict, List
+import traceback
+from typing import Dict, Iterator, List
 
+from metadata.generated.schema.type.tableQuery import TableQuery
 from metadata.ingestion.source.database.lineage_source import LineageSource
 from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_GET_STORED_PROCEDURE_QUERIES,
@@ -27,6 +29,9 @@ from metadata.ingestion.source.database.stored_procedures_mixin import (
     StoredProcedureLineageMixin,
 )
 from metadata.utils.helpers import get_start_and_end
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
 
 
 class SnowflakeLineageSource(
@@ -61,3 +66,32 @@ class SnowflakeLineageSource(
         )
 
         return queries_dict
+
+    def yield_table_query(self) -> Iterator[TableQuery]:
+        """
+        Given an engine, iterate over the query results to
+        yield a TableQuery with query parsing info
+        """
+        for engine in self.get_engine():
+            rows = []
+            with engine.connect() as conn:
+                rows = conn.execution_options(
+                    stream_results=True, max_row_buffer=100
+                ).execute(
+                    self.get_sql_statement(start_time=self.start, end_time=self.end)
+                )
+            # exit from active connection after fetching rows & during
+            # further process of `yield_query_lineage`
+            for row in rows:
+                query_dict = dict(row)
+                try:
+                    yield TableQuery(
+                        dialect=self.dialect.value,
+                        query=query_dict["query_text"],
+                        databaseName=self.get_database_name(query_dict),
+                        serviceName=self.config.serviceName,
+                        databaseSchema=self.get_schema_name(query_dict),
+                    )
+                except Exception as exc:
+                    logger.debug(traceback.format_exc())
+                    logger.warning(f"Error processing query_dict {query_dict}: {exc}")

metadata/ingestion/source/database/snowflake/metadata.py

@@ -27,7 +27,10 @@ from metadata.generated.schema.api.data.createStoredProcedure import (
 )
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
-from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureCode
+from metadata.generated.schema.entity.data.storedProcedure import (
+    StoredProcedureCode,
+    StoredProcedureType,
+)
 from metadata.generated.schema.entity.data.table import (
     PartitionColumnDetails,
     PartitionIntervalTypes,
@@ -71,6 +74,7 @@ from metadata.ingestion.source.database.snowflake.models import (
     SnowflakeStoredProcedure,
 )
 from metadata.ingestion.source.database.snowflake.queries import (
+    SNOWFLAKE_DESC_FUNCTION,
     SNOWFLAKE_DESC_STORED_PROCEDURE,
     SNOWFLAKE_FETCH_ALL_TAGS,
     SNOWFLAKE_GET_CLUSTER_KEY,
@@ -78,6 +82,7 @@ from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_GET_DATABASE_COMMENTS,
     SNOWFLAKE_GET_DATABASES,
     SNOWFLAKE_GET_EXTERNAL_LOCATIONS,
+    SNOWFLAKE_GET_FUNCTIONS,
     SNOWFLAKE_GET_ORGANIZATION_NAME,
     SNOWFLAKE_GET_SCHEMA_COMMENTS,
     SNOWFLAKE_GET_STORED_PROCEDURES,
@@ -623,26 +628,34 @@ class SnowflakeSource(
 
         return views
 
+    def _get_stored_procedures_internal(
+        self, query: str
+    ) -> Iterable[SnowflakeStoredProcedure]:
+        results = self.engine.execute(
+            query.format(
+                database_name=self.context.get().database,
+                schema_name=self.context.get().database_schema,
+            )
+        ).all()
+        for row in results:
+            stored_procedure = SnowflakeStoredProcedure.model_validate(dict(row))
+            if stored_procedure.definition is None:
+                logger.debug(
+                    f"Missing ownership permissions on procedure {stored_procedure.name}."
+                    " Trying to fetch description via DESCRIBE."
+                )
+                stored_procedure.definition = self.describe_procedure_definition(
+                    stored_procedure
+                )
+            yield stored_procedure
+
     def get_stored_procedures(self) -> Iterable[SnowflakeStoredProcedure]:
         """List Snowflake stored procedures"""
         if self.source_config.includeStoredProcedures:
-            results = self.engine.execute(
-                SNOWFLAKE_GET_STORED_PROCEDURES.format(
-                    database_name=self.context.get().database,
-                    schema_name=self.context.get().database_schema,
-                )
-            ).all()
-            for row in results:
-                stored_procedure = SnowflakeStoredProcedure.model_validate(dict(row))
-                if stored_procedure.definition is None:
-                    logger.debug(
-                        f"Missing ownership permissions on procedure {stored_procedure.name}."
-                        " Trying to fetch description via DESCRIBE."
-                    )
-                    stored_procedure.definition = self.describe_procedure_definition(
-                        stored_procedure
-                    )
-                yield stored_procedure
+            yield from self._get_stored_procedures_internal(
+                SNOWFLAKE_GET_STORED_PROCEDURES
+            )
+            yield from self._get_stored_procedures_internal(SNOWFLAKE_GET_FUNCTIONS)
 
     def describe_procedure_definition(
         self, stored_procedure: SnowflakeStoredProcedure
@@ -654,8 +667,12 @@ class SnowflakeSource(
         Then, if the procedure is created with `EXECUTE AS CALLER`, we can still try to
         get the definition with a DESCRIBE.
         """
+        if stored_procedure.procedure_type == StoredProcedureType.StoredProcedure.value:
+            query = SNOWFLAKE_DESC_STORED_PROCEDURE
+        else:
+            query = SNOWFLAKE_DESC_FUNCTION
         res = self.engine.execute(
-            SNOWFLAKE_DESC_STORED_PROCEDURE.format(
+            query.format(
                 database_name=self.context.get().database,
                 schema_name=self.context.get().database_schema,
                 procedure_name=stored_procedure.name,
@@ -677,6 +694,8 @@ class SnowflakeSource(
                     language=STORED_PROC_LANGUAGE_MAP.get(stored_procedure.language),
                     code=stored_procedure.definition,
                 ),
+                storedProcedureType=stored_procedure.procedure_type
+                or StoredProcedureType.StoredProcedure.value,
                 databaseSchema=fqn.build(
                     metadata=self.metadata,
                     entity_type=DatabaseSchema,

metadata/ingestion/source/database/snowflake/models.py

@@ -50,6 +50,7 @@ class SnowflakeStoredProcedure(BaseModel):
         None, alias="SIGNATURE", description="Used to build the source URL"
     )
     comment: Optional[str] = Field(None, alias="COMMENT")
+    procedure_type: Optional[str] = Field(None, alias="PROCEDURE_TYPE")
 
     # Update the signature to clean it up on read
     @field_validator("signature")

metadata/ingestion/source/database/snowflake/profiler/system.py

@@ -20,6 +20,7 @@ from metadata.profiler.metrics.system.system import (
     SQASessionProvider,
     SystemMetricsComputer,
 )
+from metadata.profiler.processor.runner import QueryRunner
 from metadata.utils.collections import CaseInsensitiveString
 from metadata.utils.logger import profiler_logger
 from metadata.utils.lru_cache import LRU_CACHE_SIZE, LRUCache
@@ -294,11 +295,11 @@ class SnowflakeSystemMetricsSource(
         )
 
     def get_kwargs(self, **kwargs):
-        table = kwargs.get("table")
+        runner: QueryRunner = kwargs.get("runner")
         return {
-            "table": table.__table__.name,
-            "database": self.get_session().get_bind().url.database,
-            "schema": table.__table__.schema,
+            "table": runner.table_name,
+            "database": runner.session.get_bind().url.database,
+            "schema": runner.schema_name,
         }
 
     def get_inserts(self, **kwargs) -> List[SystemProfile]:

metadata/ingestion/source/database/snowflake/queries.py

@@ -294,17 +294,38 @@ SELECT
   PROCEDURE_LANGUAGE AS language,
   PROCEDURE_DEFINITION AS definition,
   ARGUMENT_SIGNATURE AS signature,
-  COMMENT as comment
+  COMMENT as comment,
+  'StoredProcedure' as procedure_type
 FROM INFORMATION_SCHEMA.PROCEDURES
 WHERE PROCEDURE_CATALOG = '{database_name}'
   AND PROCEDURE_SCHEMA = '{schema_name}'
     """
 )
 
+SNOWFLAKE_GET_FUNCTIONS = textwrap.dedent(
+    """
+SELECT
+  FUNCTION_NAME AS name,
+  FUNCTION_OWNER AS owner,
+  FUNCTION_LANGUAGE AS language,
+  FUNCTION_DEFINITION AS definition,
+  ARGUMENT_SIGNATURE AS signature,
+  COMMENT as comment,
+  'UDF' as procedure_type
+FROM INFORMATION_SCHEMA.FUNCTIONS
+WHERE FUNCTION_CATALOG = '{database_name}'
+  AND FUNCTION_SCHEMA = '{schema_name}'
+    """
+)
+
 SNOWFLAKE_DESC_STORED_PROCEDURE = (
     "DESC PROCEDURE {database_name}.{schema_name}.{procedure_name}{procedure_signature}"
 )
 
+SNOWFLAKE_DESC_FUNCTION = (
+    "DESC FUNCTION {database_name}.{schema_name}.{procedure_name}{procedure_signature}"
+)
+
 SNOWFLAKE_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent(
     """
 WITH SP_HISTORY AS (

metadata/ingestion/source/database/snowflake/service_spec.py

@@ -1,9 +1,13 @@
+from metadata.data_quality.interface.sqlalchemy.snowflake.test_suite_interface import (
+    SnowflakeTestSuiteInterface,
+)
 from metadata.ingestion.source.database.snowflake.lineage import SnowflakeLineageSource
 from metadata.ingestion.source.database.snowflake.metadata import SnowflakeSource
 from metadata.ingestion.source.database.snowflake.profiler.profiler import (
     SnowflakeProfiler,
 )
 from metadata.ingestion.source.database.snowflake.usage import SnowflakeUsageSource
+from metadata.sampler.sqlalchemy.snowflake.sampler import SnowflakeSampler
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
 ServiceSpec = DefaultDatabaseSpec(
@@ -11,4 +15,6 @@ ServiceSpec = DefaultDatabaseSpec(
     lineage_source_class=SnowflakeLineageSource,
     usage_source_class=SnowflakeUsageSource,
     profiler_class=SnowflakeProfiler,
+    test_suite_class=SnowflakeTestSuiteInterface,
+    sampler_class=SnowflakeSampler,
 )

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -217,14 +217,26 @@ class StoredProcedureLineageMixin(ABC):
         self, procedure_and_query: ProcedureAndQuery
     ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
 
-        yield from self._yield_procedure_lineage(
-            query_by_procedure=procedure_and_query.query_by_procedure,
-            procedure=procedure_and_query.procedure,
-        )
-        yield from self.yield_procedure_query(
-            query_by_procedure=procedure_and_query.query_by_procedure,
-            procedure=procedure_and_query.procedure,
-        )
+        try:
+            yield from self._yield_procedure_lineage(
+                query_by_procedure=procedure_and_query.query_by_procedure,
+                procedure=procedure_and_query.procedure,
+            )
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Could not get lineage for store procedure '{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]."
+            )
+        try:
+            yield from self.yield_procedure_query(
+                query_by_procedure=procedure_and_query.query_by_procedure,
+                procedure=procedure_and_query.procedure,
+            )
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Could not get query for store procedure '{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]."
+            )
 
     def procedure_lineage_generator(self) -> Iterable[ProcedureAndQuery]:
         query = {

metadata/ingestion/source/database/trino/service_spec.py

@@ -4,6 +4,7 @@ from metadata.ingestion.source.database.trino.usage import TrinoUsageSource
 from metadata.profiler.interface.sqlalchemy.trino.profiler_interface import (
     TrinoProfilerInterface,
 )
+from metadata.sampler.sqlalchemy.trino.sampler import TrinoSampler
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
 ServiceSpec = DefaultDatabaseSpec(
@@ -11,4 +12,5 @@ ServiceSpec = DefaultDatabaseSpec(
     lineage_source_class=TrinoLineageSource,
     usage_source_class=TrinoUsageSource,
     profiler_class=TrinoProfilerInterface,
+    sampler_class=TrinoSampler,
 )

metadata/ingestion/source/database/unitycatalog/service_spec.py

@@ -1,3 +1,6 @@
+from metadata.data_quality.interface.sqlalchemy.unity_catalog.test_suite_interface import (
+    UnityCatalogTestSuiteInterface,
+)
 from metadata.ingestion.source.database.unitycatalog.lineage import (
     UnitycatalogLineageSource,
 )
@@ -15,4 +18,5 @@ ServiceSpec = DefaultDatabaseSpec(
     lineage_source_class=UnitycatalogLineageSource,
     usage_source_class=UnitycatalogUsageSource,
     profiler_class=UnityCatalogProfilerInterface,
+    test_suite_class=UnityCatalogTestSuiteInterface,
 )

metadata/ingestion/source/pipeline/airflow/connection.py

@@ -16,7 +16,9 @@ from functools import partial, singledispatch
 from typing import Optional
 
 from airflow import settings
+from airflow.models.serialized_dag import SerializedDagModel
 from sqlalchemy.engine import Engine
+from sqlalchemy.orm import sessionmaker
 
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
@@ -102,6 +104,18 @@ def get_connection(connection: AirflowConnection) -> Engine:
         raise SourceConnectionException(msg) from exc
 
 
+class AirflowPipelineDetailsAccessError(Exception):
+    """
+    Raise when Pipeline information is not retrieved
+    """
+
+
+class AirflowTaskDetailsAccessError(Exception):
+    """
+    Raise when Task detail information is not retrieved
+    """
+
+
 def test_connection(
     metadata: OpenMetadata,
     engine: Engine,
@@ -114,8 +128,37 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
-    test_fn = {"CheckAccess": partial(test_connection_engine_step, engine)}
-
+    session_maker = sessionmaker(bind=engine)
+    session = session_maker()
+
+    def test_pipeline_details_access(session):
+        try:
+            result = session.query(SerializedDagModel).first()
+            return result
+        except Exception as e:
+            raise AirflowPipelineDetailsAccessError(
+                f"Pipeline details access error: {e}"
+            )
+
+    def test_task_detail_access(session):
+        try:
+            json_data_column = (
+                SerializedDagModel._data  # For 2.3.0 onwards # pylint: disable=protected-access
+                if hasattr(SerializedDagModel, "_data")
+                else SerializedDagModel.data  # For 2.2.5 and 2.1.4
+            )
+            result = session.query(json_data_column).first()
+
+            retrieved_tasks = result[0]["dag"]["tasks"]
+            return retrieved_tasks
+        except Exception as e:
+            raise AirflowTaskDetailsAccessError(f"Task details access error : {e}")
+
+    test_fn = {
+        "CheckAccess": partial(test_connection_engine_step, engine),
+        "PipelineDetailsAccess": partial(test_pipeline_details_access, session),
+        "TaskDetailAccess": partial(test_task_detail_access, session),
+    }
     return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,

metadata/ingestion/source/pipeline/airflow/metadata.py

@@ -357,32 +357,42 @@ class AirflowSource(PipelineServiceSource):
             ).filter(
                 DagModel.is_paused == False  # pylint: disable=singleton-comparison
             )
-        for serialized_dag in session_query.yield_per(100):
-            try:
-                data = serialized_dag[1]["dag"]
-                dag = AirflowDagDetails(
-                    dag_id=serialized_dag[0],
-                    fileloc=serialized_dag[2],
-                    data=AirflowDag.model_validate(serialized_dag[1]),
-                    max_active_runs=data.get("max_active_runs", None),
-                    description=data.get("_description", None),
-                    start_date=data.get("start_date", None),
-                    tasks=list(
-                        map(self._extract_serialized_task, data.get("tasks", []))
-                    ),
-                    schedule_interval=get_schedule_interval(data),
-                    owner=self.fetch_dag_owners(data),
-                )
+        limit = 100  # Number of records per batch
+        offset = 0  # Start
+
+        while True:
+            paginated_query = session_query.limit(limit).offset(offset)
+            results = paginated_query.all()
+            if not results:
+                break
+            for serialized_dag in results:
+                try:
+                    data = serialized_dag[1]["dag"]
+                    dag = AirflowDagDetails(
+                        dag_id=serialized_dag[0],
+                        fileloc=serialized_dag[2],
+                        data=AirflowDag.model_validate(serialized_dag[1]),
+                        max_active_runs=data.get("max_active_runs", None),
+                        description=data.get("_description", None),
+                        start_date=data.get("start_date", None),
+                        tasks=list(
+                            map(self._extract_serialized_task, data.get("tasks", []))
+                        ),
+                        schedule_interval=get_schedule_interval(data),
+                        owner=self.fetch_dag_owners(data),
+                    )
 
-                yield dag
-            except ValidationError as err:
-                logger.debug(traceback.format_exc())
-                logger.warning(
-                    f"Error building pydantic model for {serialized_dag} - {err}"
-                )
-            except Exception as err:
-                logger.debug(traceback.format_exc())
-                logger.warning(f"Wild error yielding dag {serialized_dag} - {err}")
+                    yield dag
+                except ValidationError as err:
+                    logger.debug(traceback.format_exc())
+                    logger.warning(
+                        f"Error building pydantic model for {serialized_dag} - {err}"
+                    )
+                except Exception as err:
+                    logger.debug(traceback.format_exc())
+                    logger.warning(f"Wild error yielding dag {serialized_dag} - {err}")
+
+            offset += limit
 
     def fetch_dag_owners(self, data) -> Optional[str]:
         """

metadata/ingestion/source/pipeline/dbtcloud/client.py

@@ -35,6 +35,7 @@ from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ometa_logger
 
 logger = ometa_logger()
+API_VERSION = "api/v2"
 
 
 class DBTCloudClient:
@@ -44,9 +45,13 @@ class DBTCloudClient:
 
     def __init__(self, config: DBTCloudConnection):
         self.config = config
+
+        self.job_ids = self.config.jobIds
+        self.project_ids = self.config.projectIds
+
         client_config: ClientConfig = ClientConfig(
             base_url=clean_uri(self.config.host),
-            api_version="api/v2",
+            api_version=API_VERSION,
             auth_header=AUTHORIZATION_HEADER,
             auth_token=lambda: (self.config.token.get_secret_value(), 0),
             allow_redirects=True,
@@ -63,27 +68,43 @@ class DBTCloudClient:
         self.client = REST(client_config)
         self.graphql_client = REST(graphql_client_config)
 
-    def test_get_jobs(self) -> Optional[List[DBTJob]]:
+    def _get_jobs(
+        self, job_id: str = None, project_id: str = None
+    ) -> Optional[List[DBTJob]]:
         """
-        test fetch jobs for an account in dbt cloud
+        fetch jobs for an account in dbt cloud
         """
-        job_path = f"{self.config.jobId}/" if self.config.jobId else ""
-        result = self.client.get(f"/accounts/{self.config.accountId}/jobs/{job_path}")
-        job_list = (
-            [DBTJob.model_validate(result["data"])]
-            if self.config.jobId
-            else DBTJobList.model_validate(result).Jobs
-        )
+        job_list = []
+        try:
+            job_path = f"{job_id}/" if job_id else ""
+            project_path = f"?project_id={project_id}" if project_id else ""
+            result = self.client.get(
+                f"/accounts/{self.config.accountId}/jobs/{job_path}{project_path}"
+            )
+            job_list = (
+                [DBTJob.model_validate(result["data"])]
+                if job_id
+                else DBTJobList.model_validate(result).Jobs
+            )
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.error(
+                f"Failed to get job info for project_id: `{project_id}` or job_id: `{job_id}` : {exc}"
+            )
         return job_list
 
-    def test_get_runs(self, job_id: int) -> Optional[List[DBTRun]]:
+    def test_get_jobs(self) -> List[DBTJob]:
+        """
+        test fetch jobs for an account in dbt cloud
+        """
+        job_list = self.client.get(f"/accounts/{self.config.accountId}/jobs/")
+        return DBTJobList.model_validate(job_list).Jobs
+
+    def test_get_runs(self) -> List[DBTRun]:
         """
         test fetch runs for a job in dbt cloud
         """
-        result = self.client.get(
-            f"/accounts/{self.config.accountId}/runs/",
-            data={"job_definition_id": job_id},
-        )
+        result = self.client.get(f"/accounts/{self.config.accountId}/runs/")
         run_list = DBTRunList.model_validate(result).Runs
         return run_list
 
@@ -92,17 +113,29 @@ class DBTCloudClient:
         list jobs for an account in dbt cloud
         """
         try:
-            job_path = f"{self.config.jobId}/" if self.config.jobId else ""
-            result = self.client.get(
-                f"/accounts/{self.config.accountId}/jobs/{job_path}"
-            )
-            if result:
-                job_list = (
-                    [DBTJob.model_validate(result.get("data"))]
-                    if self.config.jobId
-                    else DBTJobList.model_validate(result).Jobs
-                )
-                return job_list
+            jobs = []
+            # case when job_ids are specified and project_ids are not
+            if self.job_ids and not self.project_ids:
+                for job_id in self.job_ids:
+                    jobs.extend(self._get_jobs(job_id=job_id))
+            # case when project_ids are specified or both are specified
+            elif self.project_ids:
+                for project_id in self.project_ids:
+                    results = self._get_jobs(project_id=project_id)
+                    if self.job_ids:
+                        jobs.extend(
+                            [
+                                result
+                                for result in results
+                                if str(result.id) in self.job_ids
+                            ]
+                        )
+                    else:
+                        jobs.extend(results)
+            else:
+                results = self._get_jobs()
+                jobs.extend(results)
+            return jobs
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Unable to get job info :{exc}")
@@ -141,6 +174,9 @@ class DBTCloudClient:
 
             if result.get("data") and result["data"].get("job"):
                 model_list = DBTModelList.model_validate(result["data"]["job"]).models
+                logger.debug(
+                    f"Successfully fetched models from dbt for job_id:{job_id} run_id:{run_id}: {model_list}"
+                )
                 return model_list
 
         except Exception as exc:
@@ -150,7 +186,7 @@ class DBTCloudClient:
 
     def get_models_and_seeds_details(self, job_id: int, run_id: int):
         """
-        get model details for a job in dbt cloud for lineage
+        get parent model details for a job in dbt cloud for lineage
         """
         try:
             query_params = {
@@ -163,9 +199,12 @@ class DBTCloudClient:
             if result.get("data") and result["data"].get("job"):
                 result = DBTModelList.model_validate(result["data"]["job"])
                 parents_list = result.models + result.seeds
+                logger.debug(
+                    f"Successfully fetched parent models from dbt for job_id:{job_id} run_id:{run_id}: {parents_list}"
+                )
                 return parents_list
 
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.warning(f"Unable to get model info :{exc}")
+            logger.warning(f"Unable to get parents model info :{exc}")
         return None

metadata/ingestion/source/pipeline/dbtcloud/connection.py

@@ -50,11 +50,9 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
-    job_id = int(service_connection.jobId) if service_connection.jobId else 0
-
     test_fn = {
         "GetJobs": client.test_get_jobs,
-        "GetRuns": partial(client.test_get_runs, job_id=job_id),
+        "GetRuns": partial(client.test_get_runs),
     }
 
     return test_connection_steps(

metadata/ingestion/source/pipeline/dbtcloud/models.py

@@ -35,7 +35,7 @@ class DBTJob(BaseModel):
 
 
 class DBTJobList(BaseModel):
-    Jobs: Optional[List[DBTJob]] = Field([], alias="data")
+    Jobs: List[DBTJob] = Field(alias="data")
 
 
 class DBTRun(BaseModel):

metadata/ingestion/source/pipeline/fivetran/metadata.py

@@ -275,7 +275,7 @@ class FivetranSource(PipelineServiceSource):
                 )
 
     def get_pipeline_name(self, pipeline_details: FivetranPipelineDetails) -> str:
-        return pipeline_details.pipeline_name
+        return pipeline_details.pipeline_display_name or pipeline_details.pipeline_name
 
     def get_source_url(
         self,

metadata/ingestion/source/pipeline/kafkaconnect/client.py

@@ -54,7 +54,7 @@ class KafkaConnectClient:
         auth = None
         ssl_verify = config.verifySSL
         if config.KafkaConnectConfig:
-            auth = f"{config.KafkaConnectConfig.username}:{config.KafkaConnectConfig.password}"
+            auth = f"{config.KafkaConnectConfig.username}:{config.KafkaConnectConfig.password.get_secret_value()}"
         self.client = KafkaConnect(url=url, auth=auth, ssl_verify=ssl_verify)
 
     def get_cluster_info(self) -> Optional[dict]: