openmetadata-ingestion 1.2.4.0__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl

metadata/ingestion/source/mlmodel/mlmodel_service.py

@@ -39,8 +39,8 @@ from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.topology import (
     NodeStage,
     ServiceTopology,
+    TopologyContext,
     TopologyNode,
-    create_source_context,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
@@ -100,7 +100,7 @@ class MlModelServiceSource(TopologyRunnerMixin, Source, ABC):
     service_connection: MlModelConnection.__fields__["config"].type_
 
     topology = MlModelServiceTopology()
-    context = create_source_context(topology)
+    context = TopologyContext.create(topology)
     mlmodel_source_state: Set = set()
 
     def __init__(
@@ -123,6 +123,10 @@ class MlModelServiceSource(TopologyRunnerMixin, Source, ABC):
 
         self.client = self.connection
 
+    @property
+    def name(self) -> str:
+        return self.service_connection.type.name
+
     def get_services(self) -> Iterable[WorkflowSource]:
         yield self.config
 

metadata/ingestion/source/mlmodel/sagemaker/metadata.py

@@ -24,6 +24,9 @@ from metadata.generated.schema.entity.data.mlmodel import (
 from metadata.generated.schema.entity.services.connections.mlmodel.sageMakerConnection import (
     SageMakerConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
@@ -128,14 +131,23 @@ class SagemakerSource(MlModelServiceSource):
         """
         Prepare the Request model
         """
-        mlmodel_request = CreateMlModelRequest(
-            name=model.name,
-            algorithm=self._get_algorithm(),  # Setting this to a constant
-            mlStore=self._get_ml_store(model.name),
-            service=self.context.mlmodel_service,
-        )
-        yield mlmodel_request
-        self.register_record(mlmodel_request=mlmodel_request)
+        try:
+            mlmodel_request = CreateMlModelRequest(
+                name=model.name,
+                algorithm=self._get_algorithm(),  # Setting this to a constant
+                mlStore=self._get_ml_store(model.name),
+                service=self.context.mlmodel_service,
+            )
+            yield Either(right=mlmodel_request)
+            self.register_record(mlmodel_request=mlmodel_request)
+        except Exception as exc:  # pylint: disable=broad-except
+            yield Either(
+                left=StackTraceError(
+                    name=model.name,
+                    error=f"Error creating mlmodel: {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
 
     def _get_ml_store(  # pylint: disable=arguments-differ
         self,

metadata/ingestion/source/pipeline/airflow/connection.py

@@ -21,9 +21,6 @@ from sqlalchemy.engine import Engine
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
-from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
-    MssqlConnection,
-)
 from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
     MysqlConnection,
 )
@@ -81,15 +78,6 @@ def _(airflow_connection: PostgresConnection) -> Engine:
     return get_postgres_connection(airflow_connection)
 
 
-@_get_connection.register
-def _(airflow_connection: MssqlConnection) -> Engine:
-    from metadata.ingestion.source.database.mssql.connection import (
-        get_connection as get_mssql_connection,
-    )
-
-    return get_mssql_connection(airflow_connection)
-
-
 @_get_connection.register
 def _(airflow_connection: SQLiteConnection) -> Engine:
     from metadata.ingestion.source.database.sqlite.connection import (

metadata/ingestion/source/pipeline/airflow/lineage_parser.py

@@ -64,6 +64,7 @@ and we'll treat this as independent sets of lineage
 """
 import json
 import logging
+import textwrap
 import traceback
 from collections import defaultdict
 from copy import deepcopy
@@ -134,16 +135,16 @@ class XLets(BaseModel):
 
 
 def concat_dict_values(
-    d1: DefaultDict[str, List[Any]], d2: Optional[Dict[str, List[Any]]]
+    dict_1: DefaultDict[str, List[Any]], dict_2: Optional[Dict[str, List[Any]]]
 ) -> DefaultDict[str, List[Any]]:
     """
     Update d1 based on d2 values concatenating their results.
     """
-    if d2:
-        for key, value in d2.items():
-            d1[key] = d1[key] + value
+    if dict_2:
+        for key, value in dict_2.items():
+            dict_1[key] = dict_1[key] + value
 
-    return d1
+    return dict_1
 
 
 def parse_xlets(xlet: List[Any]) -> Optional[Dict[str, List[OMEntity]]]:
@@ -200,7 +201,12 @@ def _parse_xlets(xlet: Any) -> None:
 
 @_parse_xlets.register
 @deprecated(
-    message="Please update your inlets/outlets to follow <TODO DOCS>",
+    message=textwrap.dedent(
+        """
+    Please update your inlets/outlets to follow 
+    https://docs.open-metadata.org/connectors/pipeline/airflow/configuring-lineage
+    """
+    ),
     release="1.4.0",
 )
 def dictionary_lineage_annotation(xlet: dict) -> Dict[str, List[OMEntity]]:

metadata/ingestion/source/pipeline/airflow/metadata.py

@@ -12,6 +12,7 @@
 Airflow source to extract metadata from OM UI
 """
 import traceback
+from collections import Counter
 from datetime import datetime
 from enum import Enum
 from typing import Iterable, List, Optional, cast
@@ -34,13 +35,16 @@ from metadata.generated.schema.entity.data.pipeline import (
 from metadata.generated.schema.entity.services.connections.pipeline.airflowConnection import (
     AirflowConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
 from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.connections.session import create_and_bind_session
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
@@ -52,6 +56,7 @@ from metadata.ingestion.source.pipeline.airflow.lineage_parser import (
 from metadata.ingestion.source.pipeline.airflow.models import (
     AirflowDag,
     AirflowDagDetails,
+    AirflowTask,
 )
 from metadata.ingestion.source.pipeline.airflow.utils import get_schedule_interval
 from metadata.ingestion.source.pipeline.pipeline_service import PipelineServiceSource
@@ -68,12 +73,14 @@ class AirflowTaskStatus(Enum):
     FAILED = "failed"
     QUEUED = "queued"
     REMOVED = "removed"
+    SKIPPED = "skipped"
 
 
 STATUS_MAP = {
     AirflowTaskStatus.SUCCESS.value: StatusType.Successful.value,
     AirflowTaskStatus.FAILED.value: StatusType.Failed.value,
     AirflowTaskStatus.QUEUED.value: StatusType.Pending.value,
+    AirflowTaskStatus.SKIPPED.value: StatusType.Skipped.value,
 }
 
 
@@ -104,7 +111,7 @@ class AirflowSource(PipelineServiceSource):
         self._session = None
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(cls, config_dict, metadata: OpenMetadata) -> "AirflowSource":
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: AirflowConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, AirflowConnection):
@@ -158,7 +165,9 @@ class AirflowSource(PipelineServiceSource):
             for elem in dag_run_dict
         ]
 
-    def get_task_instances(self, dag_id: str, run_id: str) -> List[OMTaskInstance]:
+    def get_task_instances(
+        self, dag_id: str, run_id: str, serialized_tasks: List[AirflowTask]
+    ) -> List[OMTaskInstance]:
         """
         We are building our own scoped TaskInstance
         class to only focus on core properties required
@@ -168,6 +177,7 @@ class AirflowSource(PipelineServiceSource):
         sources we support.
         """
         task_instance_list = None
+        serialized_tasks_ids = {task.task_id for task in serialized_tasks}
 
         try:
             task_instance_list = (
@@ -182,7 +192,7 @@ class AirflowSource(PipelineServiceSource):
                     TaskInstance.dag_id == dag_id,
                     TaskInstance.run_id == run_id,
                     # updating old runs flag deleted tasks as `removed`
-                    TaskInstance.state != AirflowTaskStatus.REMOVED,
+                    TaskInstance.state != AirflowTaskStatus.REMOVED.value,
                 )
                 .all()
             )
@@ -204,6 +214,7 @@ class AirflowSource(PipelineServiceSource):
                 end_date=elem.get("end_date"),
             )
             for elem in task_instance_dict
+            if elem.get("task_id") in serialized_tasks_ids
         ]
 
     def yield_pipeline_status(
@@ -217,7 +228,9 @@ class AirflowSource(PipelineServiceSource):
                     dag_run.run_id and self.context.task_names
                 ):  # Airflow dags can have old task which are turned off/commented out in code
                     tasks = self.get_task_instances(
-                        dag_id=dag_run.dag_id, run_id=dag_run.run_id
+                        dag_id=dag_run.dag_id,
+                        run_id=dag_run.run_id,
+                        serialized_tasks=pipeline_details.tasks,
                     )
 
                     task_statuses = [
@@ -259,7 +272,7 @@ class AirflowSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=f"{pipeline_details.dag_id} Pipeline Status",
                     error=f"Wild error trying to extract status from DAG {pipeline_details.dag_id} - {exc}.",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -292,7 +305,7 @@ class AirflowSource(PipelineServiceSource):
                     start_date=data.get("start_date", None),
                     tasks=data.get("tasks", []),
                     schedule_interval=get_schedule_interval(data),
-                    owners=self.fetch_owners(data),
+                    owner=self.fetch_dag_owners(data),
                 )
 
                 yield dag
@@ -305,12 +318,29 @@ class AirflowSource(PipelineServiceSource):
                 logger.debug(traceback.format_exc())
                 logger.warning(f"Wild error yielding dag {serialized_dag} - {err}")
 
-    def fetch_owners(self, data) -> Optional[str]:
+    def fetch_dag_owners(self, data) -> Optional[str]:
+        """
+        In Airflow, ownership is defined as:
+        - `default_args`: Applied to all tasks and available on the DAG payload
+        - `owners`: Applied at the tasks. In Airflow's source code, DAG ownership is then a
+          list joined with the owners of all the tasks.
+
+        We will pick the owner from the tasks that appears in most tasks.
+        """
         try:
-            if self.source_config.includeOwners and data.get("default_args"):
-                return data.get("default_args", [])["__var"].get("email", [])
-        except TypeError:
-            pass
+            if self.source_config.includeOwners:
+                task_owners = [
+                    task.get("owner")
+                    for task in data.get("tasks", [])
+                    if task.get("owner") is not None
+                ]
+                if task_owners:
+                    most_common_owner, _ = Counter(task_owners).most_common(1)[0]
+                    return most_common_owner
+        except Exception as exc:
+            self.status.warning(
+                data.get("dag_id"), f"Could not extract owner information due to {exc}"
+            )
         return None
 
     def get_pipeline_name(self, pipeline_details: SerializedDAG) -> str:
@@ -319,8 +349,7 @@ class AirflowSource(PipelineServiceSource):
         """
         return pipeline_details.dag_id
 
-    @staticmethod
-    def get_tasks_from_dag(dag: AirflowDagDetails, host_port: str) -> List[Task]:
+    def get_tasks_from_dag(self, dag: AirflowDagDetails, host_port: str) -> List[Task]:
         """
         Obtain the tasks from a SerializedDAG
         :param dag: AirflowDagDetails
@@ -341,28 +370,26 @@ class AirflowSource(PipelineServiceSource):
                 startDate=task.start_date.isoformat() if task.start_date else None,
                 endDate=task.end_date.isoformat() if task.end_date else None,
                 taskType=task.task_type,
+                owner=self.get_owner(task.owner),
             )
             for task in cast(Iterable[BaseOperator], dag.tasks)
         ]
 
-    def get_user_details(self, email) -> Optional[EntityReference]:
-        user = self.metadata.get_user_by_email(email=email)
-        if user:
-            return EntityReference(id=user.id.__root__, type="user")
-        return None
-
-    def get_owner(self, owners) -> Optional[EntityReference]:
-        try:
-            if isinstance(owners, str) and owners:
-                return self.get_user_details(email=owners)
+    def get_owner(self, owner) -> Optional[EntityReference]:
+        """
+        Fetching users by name via ES to keep things as fast as possible.
 
-            if isinstance(owners, List) and owners:
-                for owner in owners or []:
-                    return self.get_user_details(email=owner)
+        We use the `owner` field since it's the onw used by Airflow to showcase
+        the info in its UI. In other connectors we might use the mail (e.g., in Looker),
+        but we use name here to be consistent with Airflow itself.
 
-            logger.debug(f"No user found with email [{owners}] in OMD")
+        If data is not indexed, we can live without this information
+        until the next run.
+        """
+        try:
+            return self.metadata.get_reference_by_name(name=owner)
         except Exception as exc:
-            logger.warning(f"Error while getting details of user {owners} - {exc}")
+            logger.warning(f"Error while getting details of user {owner} - {exc}")
         return None
 
     def yield_pipeline(
@@ -375,11 +402,13 @@ class AirflowSource(PipelineServiceSource):
         """
 
         try:
+            # Airflow uses /dags/dag_id/grid to show pipeline / dag
+            source_url = f"{clean_uri(self.service_connection.hostPort)}/dags/{pipeline_details.dag_id}/grid"
 
             pipeline_request = CreatePipelineRequest(
                 name=pipeline_details.dag_id,
                 description=pipeline_details.description,
-                sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/tree?dag_id={pipeline_details.dag_id}",
+                sourceUrl=source_url,
                 concurrency=pipeline_details.max_active_runs,
                 pipelineLocation=pipeline_details.fileloc,
                 startDate=pipeline_details.start_date.isoformat()
@@ -389,7 +418,7 @@ class AirflowSource(PipelineServiceSource):
                     pipeline_details, self.service_connection.hostPort
                 ),
                 service=self.context.pipeline_service,
-                owner=self.get_owner(pipeline_details.owners),
+                owner=self.get_owner(pipeline_details.owner),
                 scheduleInterval=pipeline_details.schedule_interval,
             )
             yield Either(right=pipeline_request)
@@ -406,7 +435,7 @@ class AirflowSource(PipelineServiceSource):
                         f"Error building DAG information from {pipeline_details}. There might be Airflow version"
                         f" incompatibilities - {err}"
                     ),
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except ValidationError as err:
@@ -415,7 +444,7 @@ class AirflowSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_details.dag_id,
                     error=f"Error building pydantic model for {pipeline_details} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -425,7 +454,7 @@ class AirflowSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_details.dag_id,
                     error=f"Wild error ingesting pipeline {pipeline_details} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 

metadata/ingestion/source/pipeline/airflow/models.py

@@ -31,7 +31,7 @@ class AirflowBaseModel(BaseModel):
     dag_id: str
 
 
-class Task(BaseModel):
+class AirflowTask(BaseModel):
     pool: Optional[str]
     doc_md: Optional[str]
     inlets: Optional[List[Any]] = Field(alias="_inlets")
@@ -41,6 +41,7 @@ class Task(BaseModel):
     downstream_task_ids: Optional[List[str]]
     start_date: Optional[datetime]
     end_date: Optional[datetime]
+    owner: Optional[str]
 
     # Allow picking up data from key `inlets` and `_inlets`
     class Config:
@@ -48,7 +49,7 @@ class Task(BaseModel):
 
 
 class TaskList(BaseModel):
-    __root__: List[Task]
+    __root__: List[AirflowTask]
 
 
 class Dag(BaseModel):
@@ -68,6 +69,6 @@ class AirflowDagDetails(AirflowBaseModel):
     max_active_runs: Optional[int]
     description: Optional[str]
     start_date: Optional[datetime]
-    tasks: List[Task]
-    owners: Optional[Any]
+    tasks: List[AirflowTask]
+    owner: Optional[str]
     schedule_interval: Optional[str]

metadata/ingestion/source/pipeline/dagster/metadata.py

@@ -26,10 +26,13 @@ from metadata.generated.schema.entity.data.pipeline import (
 from metadata.generated.schema.entity.services.connections.pipeline.dagsterConnection import (
     DagsterConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.step import WorkflowFatalError
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -141,7 +144,7 @@ class DagsterSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_details.name,
                     error=f"Error to yield pipeline for {pipeline_details}: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -197,7 +200,7 @@ class DagsterSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=run.runId,
                     error=f"Error to yield run status for {run}: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -227,7 +230,7 @@ class DagsterSource(PipelineServiceSource):
                     left=StackTraceError(
                         name=f"{pipeline_details.name} Pipeline Status",
                         error=f"Error to yield pipeline status for {pipeline_details}: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 

metadata/ingestion/source/pipeline/databrickspipeline/connection.py

@@ -44,11 +44,7 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
-    def custom_executor_for_pipeline():
-        result = client.list_jobs()
-        return list(result)
-
-    test_fn = {"GetPipelines": custom_executor_for_pipeline}
+    test_fn = {"GetPipelines": client.list_jobs_test_connection}
 
     test_connection_steps(
         metadata=metadata,

metadata/ingestion/source/pipeline/databrickspipeline/metadata.py

@@ -30,10 +30,13 @@ from metadata.generated.schema.entity.data.pipeline import (
 from metadata.generated.schema.entity.services.connections.pipeline.databricksPipelineConnection import (
     DatabricksPipelineConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -78,7 +81,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
         return cls(config, metadata)
 
     def get_pipelines_list(self) -> Iterable[dict]:
-        for workflow in self.client.list_jobs():
+        for workflow in self.client.list_jobs() or []:
             yield workflow
 
     def get_pipeline_name(self, pipeline_details: dict) -> str:
@@ -108,7 +111,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
                         f"Error building Databricks Pipeline information from {pipeline_details}."
                         f" There might be Databricks Jobs API version incompatibilities - {err}"
                     ),
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except ValidationError as err:
@@ -116,7 +119,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
                 left=StackTraceError(
                     name="Pipeline",
                     error=f"Error building pydantic model for {pipeline_details} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except Exception as err:
@@ -124,18 +127,18 @@ class DatabrickspipelineSource(PipelineServiceSource):
                 left=StackTraceError(
                     name="Pipeline",
                     error=f"Wild error ingesting pipeline {pipeline_details} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
     def get_tasks(self, pipeline_details: dict) -> List[Task]:
         task_list = []
-        self._append_context(key="job_id_list", value=pipeline_details["job_id"])
+        self.context.append(key="job_id_list", value=pipeline_details["job_id"])
 
         downstream_tasks = self.get_downstream_tasks(
             pipeline_details["settings"].get("tasks")
         )
-        for task in pipeline_details["settings"].get("tasks"):
+        for task in pipeline_details["settings"].get("tasks", []):
             task_list.append(
                 Task(
                     name=task["task_key"],
@@ -156,7 +159,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
         return task_key
 
     def get_downstream_tasks(self, workflow):
-        task_key_list = [task["task_key"] for task in workflow]
+        task_key_list = [task["task_key"] for task in workflow or []]
 
         dependent_tasks = self.get_dependent_tasks(workflow)
 
@@ -179,7 +182,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
     def get_dependent_tasks(self, workflow):
         dependent_tasks = {}
 
-        for task in workflow:
+        for task in workflow or []:
             depends_on = task.get("depends_on")
             if depends_on:
                 dependent_tasks[task["task_key"]] = [v["task_key"] for v in depends_on]
@@ -192,7 +195,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
         for job_id in self.context.job_id_list:
             try:
                 runs = self.client.get_job_runs(job_id=job_id)
-                for attempt in runs:
+                for attempt in runs or []:
                     for task_run in attempt["tasks"]:
                         task_status = []
                         task_status.append(
@@ -238,7 +241,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
                     left=StackTraceError(
                         name=pipeline_fqn,
                         error=f"Failed to yield pipeline status: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 

metadata/ingestion/source/pipeline/domopipeline/metadata.py

@@ -27,10 +27,13 @@ from metadata.generated.schema.entity.data.pipeline import (
 from metadata.generated.schema.entity.services.connections.pipeline.domoPipelineConnection import (
     DomoPipelineConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -103,7 +106,7 @@ class DomopipelineSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_details.get("name", "unknown"),
                     error=f"Error extracting data from {pipeline_details.get('name', 'unknown')} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except Exception as err:
@@ -111,7 +114,7 @@ class DomopipelineSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_details.get("name", "unknown"),
                     error=f"Wild error ingesting pipeline {pipeline_details.get('name', 'unknown')} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -175,7 +178,7 @@ class DomopipelineSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_fqn,
                     error=f"Error extracting status for {pipeline_id} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 

metadata/ingestion/source/pipeline/gluepipeline/metadata.py

@@ -28,10 +28,13 @@ from metadata.generated.schema.entity.data.pipeline import (
 from metadata.generated.schema.entity.services.connections.pipeline.gluePipelineConnection import (
     GluePipelineConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -183,7 +186,7 @@ class GluepipelineSource(PipelineServiceSource):
                     left=StackTraceError(
                         name=pipeline_fqn,
                         error=f"Failed to yield pipeline status: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )