openmetadata-ingestion 1.2.3.1__py3-none-any.whl → 1.2.4.0__py3-none-any.whl

metadata/ingestion/source/dashboard/superset/db_source.py

@@ -71,15 +71,27 @@ class SupersetDBSource(SupersetSourceMixin):
         this step is done because fetch_total_charts api fetches all
         the required information which is not available in fetch_charts_with_id api
         """
-        charts = self.engine.execute(FETCH_ALL_CHARTS)
-        for chart in charts:
-            chart_detail = FetchChart(**chart)
-            self.all_charts[chart_detail.id] = chart_detail
+        try:
+            charts = self.engine.execute(FETCH_ALL_CHARTS)
+            for chart in charts:
+                chart_detail = FetchChart(**chart)
+                self.all_charts[chart_detail.id] = chart_detail
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch chart list due to - {err}]")
 
     def get_column_list(self, table_name: str) -> Iterable[FetchChart]:
-        sql_query = sql.text(FETCH_COLUMN.format(table_name=table_name.lower()))
-        col_list = self.engine.execute(sql_query)
-        return [FetchColumn(**col) for col in col_list]
+        try:
+            if table_name:
+                sql_query = sql.text(FETCH_COLUMN.format(table_name=table_name.lower()))
+                col_list = self.engine.execute(sql_query)
+                return [FetchColumn(**col) for col in col_list]
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Failed to fetch column name list for table: [{table_name} due to - {err}]"
+            )
+        return []
 
     def get_dashboards_list(self) -> Iterable[FetchDashboard]:
         """
@@ -93,23 +105,35 @@ class SupersetDBSource(SupersetSourceMixin):
         self, dashboard_details: FetchDashboard
     ) -> Iterable[Either[CreateDashboardRequest]]:
         """Method to Get Dashboard Entity"""
-        dashboard_request = CreateDashboardRequest(
-            name=dashboard_details.id,
-            displayName=dashboard_details.dashboard_title,
-            sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/superset/dashboard/{dashboard_details.id}/",
-            charts=[
-                fqn.build(
-                    self.metadata,
-                    entity_type=Chart,
-                    service_name=self.context.dashboard_service,
-                    chart_name=chart,
+        try:
+            dashboard_request = CreateDashboardRequest(
+                name=dashboard_details.id,
+                displayName=dashboard_details.dashboard_title,
+                sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/superset/dashboard/{dashboard_details.id}/",
+                charts=[
+                    fqn.build(
+                        self.metadata,
+                        entity_type=Chart,
+                        service_name=self.context.dashboard_service,
+                        chart_name=chart,
+                    )
+                    for chart in self.context.charts
+                ],
+                service=self.context.dashboard_service,
+            )
+            yield Either(right=dashboard_request)
+            self.register_record(dashboard_request=dashboard_request)
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=dashboard_details.id,
+                    error=(
+                        f"Error yielding Dashboard [{dashboard_details.id} "
+                        f"- {dashboard_details.dashboard_title}]: {exc}"
+                    ),
+                    stack_trace=traceback.format_exc(),
                 )
-                for chart in self.context.charts
-            ],
-            service=self.context.dashboard_service,
-        )
-        yield Either(right=dashboard_request)
-        self.register_record(dashboard_request=dashboard_request)
+            )
 
     def _get_datasource_fqn_for_lineage(
         self, chart_json: FetchChart, db_service_entity: DatabaseService
@@ -127,19 +151,31 @@ class SupersetDBSource(SupersetSourceMixin):
         Metod to fetch charts linked to dashboard
         """
         for chart_id in self._get_charts_of_dashboard(dashboard_details):
-            chart_json = self.all_charts.get(chart_id)
-            if not chart_json:
-                logger.warning(f"chart details for id: {chart_id} not found, skipped")
-                continue
-            chart = CreateChartRequest(
-                name=chart_json.id,
-                displayName=chart_json.slice_name,
-                description=chart_json.description,
-                chartType=get_standard_chart_type(chart_json.viz_type),
-                sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/explore/?slice_id={chart_json.id}",
-                service=self.context.dashboard_service,
-            )
-            yield Either(right=chart)
+            try:
+                chart_json = self.all_charts.get(chart_id)
+                if not chart_json:
+                    logger.warning(
+                        f"chart details for id: {chart_id} not found, skipped"
+                    )
+
+                    continue
+                chart = CreateChartRequest(
+                    name=chart_json.id,
+                    displayName=chart_json.slice_name,
+                    description=chart_json.description,
+                    chartType=get_standard_chart_type(chart_json.viz_type),
+                    sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/explore/?slice_id={chart_json.id}",
+                    service=self.context.dashboard_service,
+                )
+                yield Either(right=chart)
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name=chart_json.id,
+                        error=f"Error yielding Chart [{chart_json.id} - {chart_json.slice_name}]: {exc}",
+                        stack_trace=traceback.format_exc(),
+                    )
+                )
 
     def _get_database_name(
         self, sqa_str: str, db_service_entity: DatabaseService

metadata/ingestion/source/dashboard/superset/mixin.py

@@ -121,14 +121,20 @@ class SupersetSourceMixin(DashboardServiceSource):
         """
         Method to fetch chart ids linked to dashboard
         """
-        raw_position_data = dashboard_details.position_json
-        if raw_position_data:
-            position_data = json.loads(raw_position_data)
-            return [
-                value.get("meta", {}).get("chartId")
-                for key, value in position_data.items()
-                if key.startswith("CHART-") and value.get("meta", {}).get("chartId")
-            ]
+        try:
+            raw_position_data = dashboard_details.position_json
+            if raw_position_data:
+                position_data = json.loads(raw_position_data)
+                return [
+                    value.get("meta", {}).get("chartId")
+                    for key, value in position_data.items()
+                    if key.startswith("CHART-") and value.get("meta", {}).get("chartId")
+                ]
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Failed to charts of dashboard {dashboard_details.id} due to {err}"
+            )
         return []
 
     def yield_dashboard_lineage_details(
@@ -146,16 +152,16 @@ class SupersetSourceMixin(DashboardServiceSource):
             for chart_id in self._get_charts_of_dashboard(dashboard_details):
                 chart_json = self.all_charts.get(chart_id)
                 if chart_json:
-                    datasource_fqn = self._get_datasource_fqn_for_lineage(
-                        chart_json, db_service_entity
-                    )
-                    if not datasource_fqn:
-                        continue
-                    from_entity = self.metadata.get_by_name(
-                        entity=Table,
-                        fqn=datasource_fqn,
-                    )
                     try:
+                        datasource_fqn = self._get_datasource_fqn_for_lineage(
+                            chart_json, db_service_entity
+                        )
+                        if not datasource_fqn:
+                            continue
+                        from_entity = self.metadata.get_by_name(
+                            entity=Table,
+                            fqn=datasource_fqn,
+                        )
                         datamodel_fqn = fqn.build(
                             self.metadata,
                             entity_type=DashboardDataModel,

metadata/ingestion/source/database/dbt/constants.py

@@ -14,6 +14,8 @@ Constants required for dbt
 
 from enum import Enum
 
+DBT_RUN_RESULT_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+
 # Based on https://schemas.getdbt.com/dbt/manifest/v7/index.html
 REQUIRED_MANIFEST_KEYS = ["name", "schema", "resource_type"]
 

metadata/ingestion/source/database/dbt/metadata.py

@@ -59,6 +59,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.ingestion.source.database.database_service import DataModelLink
 from metadata.ingestion.source.database.dbt.constants import (
+    DBT_RUN_RESULT_DATE_FORMAT,
     REQUIRED_CATALOG_KEYS,
     REQUIRED_MANIFEST_KEYS,
     DbtCommonEnum,
@@ -870,12 +871,21 @@ class DbtSource(DbtServiceSource):
                         dbt_test_completed_at = dbt_test_timing.completed_at
                 dbt_timestamp = None
                 if dbt_test_completed_at:
-                    dbt_timestamp = dbt_test_completed_at.timestamp()
+                    dbt_timestamp = dbt_test_completed_at
                 elif self.context.run_results_generate_time:
-                    dbt_timestamp = self.context.run_results_generate_time.timestamp()
+                    dbt_timestamp = self.context.run_results_generate_time
+
+                # check if the timestamp is a str type and convert accordingly
+                if isinstance(dbt_timestamp, str):
+                    dbt_timestamp = datetime.strptime(
+                        dbt_timestamp, DBT_RUN_RESULT_DATE_FORMAT
+                    )
+
                 # Create the test case result object
                 test_case_result = TestCaseResult(
-                    timestamp=convert_timestamp_to_milliseconds(dbt_timestamp),
+                    timestamp=convert_timestamp_to_milliseconds(
+                        dbt_timestamp.timestamp()
+                    ),
                     testCaseStatus=test_case_status,
                     testResultValue=[
                         TestResultValue(

metadata/ingestion/source/database/salesforce/metadata.py

@@ -213,7 +213,7 @@ class SalesforceSource(DatabaseServiceSource):
                     entity_type=DatabaseSchema,
                     service_name=self.context.database_service,
                     database_name=self.context.database,
-                    schema_name=self.context.database.database_schema,
+                    schema_name=self.context.database_schema,
                 ),
                 sourceUrl=self.get_source_url(
                     table_name=table_name,

metadata/ingestion/source/pipeline/airflow/metadata.py

@@ -13,6 +13,7 @@ Airflow source to extract metadata from OM UI
 """
 import traceback
 from datetime import datetime
+from enum import Enum
 from typing import Iterable, List, Optional, cast
 
 from airflow.models import BaseOperator, DagRun, TaskInstance
@@ -61,10 +62,18 @@ from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
+
+class AirflowTaskStatus(Enum):
+    SUCCESS = "success"
+    FAILED = "failed"
+    QUEUED = "queued"
+    REMOVED = "removed"
+
+
 STATUS_MAP = {
-    "success": StatusType.Successful.value,
-    "failed": StatusType.Failed.value,
-    "queued": StatusType.Pending.value,
+    AirflowTaskStatus.SUCCESS.value: StatusType.Successful.value,
+    AirflowTaskStatus.FAILED.value: StatusType.Failed.value,
+    AirflowTaskStatus.QUEUED.value: StatusType.Pending.value,
 }
 
 
@@ -169,12 +178,15 @@ class AirflowSource(PipelineServiceSource):
                     TaskInstance.end_date,
                     TaskInstance.run_id,
                 )
-                .filter(TaskInstance.dag_id == dag_id, TaskInstance.run_id == run_id)
+                .filter(
+                    TaskInstance.dag_id == dag_id,
+                    TaskInstance.run_id == run_id,
+                    # updating old runs flag deleted tasks as `removed`
+                    TaskInstance.state != AirflowTaskStatus.REMOVED,
+                )
                 .all()
             )
-        except Exception as exc:  # pylint: disable=broad-except
-            # Using a broad Exception here as the backend can come in many flavours (pymysql, pyodbc...)
-            # And we don't want to force all imports
+        except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(
                 f"Tried to get TaskInstances with run_id. It might not be available in older Airflow versions - {exc}."
@@ -202,7 +214,7 @@ class AirflowSource(PipelineServiceSource):
 
             for dag_run in dag_run_list:
                 if (
-                    dag_run.run_id
+                    dag_run.run_id and self.context.task_names
                 ):  # Airflow dags can have old task which are turned off/commented out in code
                     tasks = self.get_task_instances(
                         dag_id=dag_run.dag_id, run_id=dag_run.run_id
@@ -220,6 +232,7 @@ class AirflowSource(PipelineServiceSource):
                             ),  # Might be None for running tasks
                         )  # Log link might not be present in all Airflow versions
                         for task in tasks
+                        if task.task_id in self.context.task_names
                     ]
 
                     pipeline_status = PipelineStatus(
@@ -381,7 +394,11 @@ class AirflowSource(PipelineServiceSource):
             )
             yield Either(right=pipeline_request)
             self.register_record(pipeline_request=pipeline_request)
+            self.context.task_names = {
+                task.name for task in pipeline_request.tasks or []
+            }
         except TypeError as err:
+            self.context.task_names = set()
             yield Either(
                 left=StackTraceError(
                     name=pipeline_details.dag_id,
@@ -393,6 +410,7 @@ class AirflowSource(PipelineServiceSource):
                 )
             )
         except ValidationError as err:
+            self.context.task_names = set()
             yield Either(
                 left=StackTraceError(
                     name=pipeline_details.dag_id,
@@ -402,6 +420,7 @@ class AirflowSource(PipelineServiceSource):
             )
 
         except Exception as err:
+            self.context.task_names = set()
             yield Either(
                 left=StackTraceError(
                     name=pipeline_details.dag_id,

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -399,6 +399,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                 table,
                 sample,
             )
+            row = None
 
             try:
                 row = self._get_metric_fn[metric_type.value](
@@ -412,7 +413,6 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                 error = f"{column if column is not None else runner.table.__tablename__} metric_type.value: {exc}"
                 logger.error(error)
                 self.status.failed_profiler(error, traceback.format_exc())
-                row = None
 
             if column is not None:
                 column = column.name

metadata/utils/logger.py

@@ -28,6 +28,8 @@ from metadata.ingestion.api.models import Entity
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.life_cycle import OMetaLifeCycleData
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
+from metadata.ingestion.models.patch_request import PatchRequest
+from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 
 METADATA_LOGGER = "metadata"
 BASE_LOGGING_FORMAT = (
@@ -251,3 +253,14 @@ def _(_: TestCaseResults) -> Optional[str]:
 @get_log_name.register
 def _(record: TestCaseResultResponse) -> str:
     return record.testCase.fullyQualifiedName.__root__
+
+
+@get_log_name.register
+def _(record: OMetaPipelineStatus) -> str:
+    return f"Pipeline Status [{record.pipeline_fqn}]"
+
+
+@get_log_name.register
+def _(record: PatchRequest) -> str:
+    """Get the log of the new entity"""
+    return get_log_name(record.new_entity)

metadata/utils/tag_utils.py

@@ -40,8 +40,8 @@ logger = ingestion_logger()
 def get_ometa_tag_and_classification(
     tags: List[str],
     classification_name: str,
-    tag_description: Optional[str],
-    classification_description: Optional[str],
+    tag_description: Optional[str] = None,
+    classification_description: Optional[str] = None,
     include_tags: bool = True,
     tag_fqn: Optional[FullyQualifiedEntityName] = None,
 ) -> Iterable[Either[OMetaTagAndClassification]]: