openmetadata-ingestion 1.4.8.0__py3-none-any.whl → 1.5.0.0rc2__py3-none-any.whl

metadata/workflow/workflow_init_error_handler.py

@@ -0,0 +1,154 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Module handles the init error messages from different workflows
+"""
+import traceback
+from pathlib import Path
+from typing import Any, Dict, Optional, Type, Union
+
+from metadata.config.common import ConfigurationError
+from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
+    PipelineType,
+)
+from metadata.ingestion.api.parser import (
+    InvalidWorkflowException,
+    ParsingConfigurationError,
+)
+from metadata.utils.constants import UTF_8
+from metadata.utils.logger import ANSI, log_ansi_encoded_string
+
+EXAMPLES_WORKFLOW_PATH: Path = Path(__file__).parent / "../examples" / "workflows"
+
+
+URLS: Dict[PipelineType, str] = {
+    PipelineType.metadata: "https://docs.open-metadata.org/connectors/ingestion/workflows/metadata",
+    PipelineType.profiler: "https://docs.open-metadata.org/connectors/ingestion/workflows/profiler",
+    PipelineType.TestSuite: "https://docs.open-metadata.org/connectors/ingestion/workflows/data-quality",
+    PipelineType.lineage: "https://docs.open-metadata.org/connectors/ingestion/workflows/lineage",
+    PipelineType.usage: "https://docs.open-metadata.org/connectors/ingestion/workflows/usage",
+    PipelineType.dbt: "https://docs.open-metadata.org/connectors/ingestion/workflows/dbt",
+}
+
+
+DEFAULT_EXAMPLE_FILE: Dict[PipelineType, str] = {
+    PipelineType.metadata: "bigquery",
+    PipelineType.profiler: "bigquery_profiler",
+    PipelineType.TestSuite: "test_suite",
+    PipelineType.lineage: "bigquery_lineage",
+    PipelineType.usage: "bigquery_usage",
+}
+
+
+class WorkflowInitErrorHandler:
+    """Resonsible for handling the InitError flow, when a Workflow errors before even initializing."""
+
+    @staticmethod
+    def print_init_error(
+        exc: Union[Exception, Type[Exception]],
+        config: Dict[str, Any],
+        pipeline_type: PipelineType = PipelineType.metadata,
+    ):
+        """
+        Print a workflow initialization error
+        """
+        source_type_name = WorkflowInitErrorHandler._get_source_type_name(config)
+
+        if isinstance(
+            exc,
+            (ParsingConfigurationError, ConfigurationError, InvalidWorkflowException),
+        ):
+            WorkflowInitErrorHandler._print_error_msg(
+                f"Error loading {pipeline_type.name} configuration: {exc}"
+            )
+            WorkflowInitErrorHandler._print_file_example(
+                source_type_name, pipeline_type
+            )
+        else:
+            WorkflowInitErrorHandler._print_error_msg(
+                f"\nError initializing {pipeline_type.name}: {exc}"
+            )
+            WorkflowInitErrorHandler._print_error_msg(traceback.format_exc())
+
+        WorkflowInitErrorHandler._print_more_info(pipeline_type)
+
+    @staticmethod
+    def _get_source_type_name(config: Dict[str, Any]) -> Optional[str]:
+        """Returns the Source Type Name based on the Configuration passed."""
+        source_type_name = None
+
+        if (
+            config
+            and config.get("source", None) is not None
+            and config["source"].get("type", None) is not None
+        ):
+            source_type_name = config["source"].get("type")
+            source_type_name = source_type_name.replace("-", "-")
+
+        return source_type_name
+
+    @staticmethod
+    def _print_file_example(
+        source_type_name: Optional[str], pipeline_type: PipelineType
+    ):
+        """
+        Print an example file for a given configuration
+        """
+        if source_type_name is not None:
+            example_file = WorkflowInitErrorHandler._calculate_example_file(
+                source_type_name, pipeline_type
+            )
+            example_path = EXAMPLES_WORKFLOW_PATH / f"{example_file}.yaml"
+            if not example_path.exists():
+                example_file = DEFAULT_EXAMPLE_FILE[pipeline_type]
+                example_path = EXAMPLES_WORKFLOW_PATH / f"{example_file}.yaml"
+            log_ansi_encoded_string(
+                message=f"\nMake sure you are following the following format e.g. '{example_file}':"
+            )
+            log_ansi_encoded_string(message="------------")
+            with open(example_path, encoding=UTF_8) as file:
+                log_ansi_encoded_string(message=file.read())
+            log_ansi_encoded_string(message="------------")
+
+    @staticmethod
+    def _calculate_example_file(
+        source_type_name: str, pipeline_type: PipelineType
+    ) -> str:
+        """
+        Calculates the ingestion type depending on the source type name and workflow_type
+        """
+        if pipeline_type == PipelineType.usage:
+            return f"{source_type_name}_usage"
+        if pipeline_type == PipelineType.lineage:
+            return f"{source_type_name}_lineage"
+        if pipeline_type == PipelineType.profiler:
+            return f"{source_type_name}_profiler"
+        if pipeline_type == PipelineType.TestSuite:
+            return DEFAULT_EXAMPLE_FILE[pipeline_type]
+        return source_type_name
+
+    @staticmethod
+    def _print_more_info(pipeline_type: PipelineType) -> None:
+        """
+        Print more information message
+        """
+        log_ansi_encoded_string(
+            message=f"\nFor more information, please visit: {URLS[pipeline_type]}"
+            + "\nOr join us in Slack: https://slack.open-metadata.org/"
+        )
+
+    @staticmethod
+    def _print_error_msg(msg: str) -> None:
+        """
+        Print message with error style
+        """
+        log_ansi_encoded_string(color=ANSI.BRIGHT_RED, bold=False, message=f"{msg}")

metadata/workflow/workflow_output_handler.py

@@ -14,127 +14,218 @@ Module handles the output messages from different workflows
 """
 
 import time
-import traceback
-from typing import Type, Union
+from typing import Any, Dict, List, Optional, Type, Union
 
-from metadata.config.common import ConfigurationError
-from metadata.ingestion.api.parser import (
-    InvalidWorkflowException,
-    ParsingConfigurationError,
-)
-from metadata.utils.constants import UTF_8
+from pydantic import BaseModel
+from tabulate import tabulate
+
+from metadata.ingestion.api.status import TruncatedStackTraceError
+from metadata.ingestion.api.step import Step, Summary
+from metadata.ingestion.lineage.models import QueryParsingFailures
+from metadata.utils.deprecation import deprecated
+from metadata.utils.execution_time_tracker import ExecutionTimeTracker
 from metadata.utils.helpers import pretty_print_time_duration
 from metadata.utils.logger import ANSI, log_ansi_encoded_string
 from metadata.workflow.output_handler import (
-    DEFAULT_EXAMPLE_FILE,
-    EXAMPLES_WORKFLOW_PATH,
-    WORKFLOW_FAILURE_MESSAGE,
     WorkflowType,
-    print_error_msg,
-    print_more_info,
-    print_workflow_summary,
+    workflow_type_to_pipeline_type,
 )
+from metadata.workflow.workflow_init_error_handler import WorkflowInitErrorHandler
+from metadata.workflow.workflow_status_mixin import WorkflowResultStatus
+
+WORKFLOW_FAILURE_MESSAGE = "Workflow finished with failures"
+WORKFLOW_WARNING_MESSAGE = "Workflow finished with warnings"
+WORKFLOW_SUCCESS_MESSAGE = "Workflow finished successfully"
 
 
-def calculate_ingestion_type(source_type_name: str) -> WorkflowType:
+class Failure(BaseModel):
     """
-    Calculates the ingestion type depending on the source type name
+    Auxiliary class to print the error per status
     """
-    if source_type_name.endswith("lineage"):
-        return WorkflowType.LINEAGE
-    if source_type_name.endswith("usage"):
-        return WorkflowType.USAGE
-    return WorkflowType.INGEST
 
+    name: str
+    failures: List[TruncatedStackTraceError]
 
-def calculate_example_file(source_type_name: str, workflow_type: WorkflowType) -> str:
-    """
-    Calculates the ingestion type depending on the source type name and workflow_type
-    """
-    if workflow_type == WorkflowType.USAGE:
-        return f"{source_type_name}_usage"
-    if workflow_type == WorkflowType.LINEAGE:
-        return f"{source_type_name}_lineage"
-    if workflow_type == WorkflowType.PROFILE:
-        return f"{source_type_name}_profiler"
-    if workflow_type == WorkflowType.TEST:
-        return DEFAULT_EXAMPLE_FILE[workflow_type]
-    return source_type_name
-
-
-def print_file_example(source_type_name: str, workflow_type: WorkflowType):
-    """
-    Print an example file for a given configuration
-    """
-    if source_type_name is not None:
-        example_file = calculate_example_file(source_type_name, workflow_type)
-        example_path = EXAMPLES_WORKFLOW_PATH / f"{example_file}.yaml"
-        if not example_path.exists():
-            example_file = DEFAULT_EXAMPLE_FILE[workflow_type]
-            example_path = EXAMPLES_WORKFLOW_PATH / f"{example_file}.yaml"
-        log_ansi_encoded_string(
-            message=f"\nMake sure you are following the following format e.g. '{example_file}':"
-        )
-        log_ansi_encoded_string(message="------------")
-        with open(example_path, encoding=UTF_8) as file:
-            log_ansi_encoded_string(message=file.read())
-        log_ansi_encoded_string(message="------------")
 
+@deprecated(message="Use 'workflow.print_status()' instead.", release="1.6")
+def print_status(
+    workflow: "BaseWorkflow",  # pyright: ignore[reportUndefinedVariable,reportUnknownParameterType]
+):
+    workflow.print_status()  # pyright: ignore[reportUnknownMemberType]
 
+
+@deprecated(
+    message=(
+        "Use 'WorkflowInitErrorHandler.print_init_error(exc, config, workflow_type)'"
+        " from 'metadata.workflow.workflow_init_error_handler'"
+    ),
+    release="1.6",
+)
 def print_init_error(
     exc: Union[Exception, Type[Exception]],
-    config: dict,
+    config: Dict[str, Any],
     workflow_type: WorkflowType = WorkflowType.INGEST,
-) -> None:
-    """
-    Print a workflow initialization error
-    """
-    source_type_name = None
-    if (
+):
+    # pylint: disable=W0212
+    source_type_name = WorkflowInitErrorHandler._get_source_type_name(  # pyright: ignore[reportPrivateUsage]
         config
-        and config.get("source", None) is not None
-        and config["source"].get("type", None) is not None
-    ):
-        source_type_name = config["source"].get("type")
-        source_type_name = source_type_name.replace("-", "-")
-        workflow_type = (
-            calculate_ingestion_type(source_type_name)
-            if workflow_type == WorkflowType.INGEST
-            else workflow_type
-        )
-
-    if isinstance(
-        exc, (ParsingConfigurationError, ConfigurationError, InvalidWorkflowException)
-    ):
-        print_error_msg(f"Error loading {workflow_type.name} configuration: {exc}")
-        print_file_example(source_type_name, workflow_type)
-        print_more_info(workflow_type)
-    else:
-        print_error_msg(f"\nError initializing {workflow_type.name}: {exc}")
-        print_error_msg(traceback.format_exc())
-        print_more_info(workflow_type)
-
+    )
+    WorkflowInitErrorHandler.print_init_error(
+        exc, config, workflow_type_to_pipeline_type(workflow_type, source_type_name)
+    )
 
-def print_status(workflow: "IngestionWorkflow") -> None:
-    """
-    Print the workflow results
-    """
 
-    print_workflow_summary(workflow)
+class WorkflowOutputHandler:
+    """Responsible for dealing with the Workflow Outputs"""
 
-    # Get the time to execute the first step
-    first_step = workflow.workflow_steps()[0]
-    if first_step.get_status().source_start_time:
+    def print_status(
+        self,
+        result_status: WorkflowResultStatus,
+        steps: List[Step],
+        start_time: Optional[Any] = None,
+        debug: bool = False,
+    ):
+        """
+        Print the workflow results
+        """
+        self.print_summary(steps, debug)
+
+        if start_time:
+            log_ansi_encoded_string(
+                color=ANSI.BRIGHT_CYAN,
+                bold=True,
+                message="Workflow finished in time: "
+                + f"{pretty_print_time_duration(time.time() - start_time)}",
+            )
+
+        if result_status == WorkflowResultStatus.FAILURE:
+            log_ansi_encoded_string(
+                color=ANSI.BRIGHT_RED,
+                bold=True,
+                message=WORKFLOW_FAILURE_MESSAGE,
+            )
+
+    def print_summary(self, steps: List[Step], debug: bool = False):
+        """Prints the summary information for a Workflow Execution."""
+        if debug:
+            self._print_debug_summary(steps)
+            self._print_execution_time_summary()
+            self._print_query_parsing_issues()
+
+        self._print_summary(steps)
+
+    def _print_summary(self, steps: List[Step]):
+        failures: List[Failure] = []
+        total_records: int = 0
+        total_errors: int = 0
+
+        for step in steps:
+            step_summary = Summary.from_step(step)
+
+            total_records += step_summary.records or 0
+            total_errors += step_summary.errors or 0
+            failures.append(
+                Failure(name=step.name, failures=step.get_status().failures)
+            )
+
+            log_ansi_encoded_string(bold=True, message=f"Workflow {step.name} Summary:")
+            log_ansi_encoded_string(
+                message=f"Processed records: {step_summary.records}"
+            )
+            log_ansi_encoded_string(
+                message=f"Updated records: {step_summary.updated_records}"
+            )
+            log_ansi_encoded_string(message=f"Warnings: {step_summary.warnings}")
+
+            if step_summary.filtered:
+                log_ansi_encoded_string(message=f"Filtered: {step_summary.filtered}")
+
+            log_ansi_encoded_string(message=f"Errors: {step_summary.errors}")
+
+        self._print_failures_if_apply(failures)
+
+        total_success = max(total_records, 1)
         log_ansi_encoded_string(
             color=ANSI.BRIGHT_CYAN,
             bold=True,
-            message="Workflow finished in time: "
-            f"{pretty_print_time_duration(time.time()-first_step.get_status().source_start_time)}",
+            message="Success %: "
+            + f"{round(total_success * 100 / (total_success + total_errors), 2)}",
         )
 
-    if workflow.result_status() == 1:
-        log_ansi_encoded_string(
-            color=ANSI.BRIGHT_RED,
-            bold=True,
-            message=WORKFLOW_FAILURE_MESSAGE,
-        )
+    def _print_debug_summary(self, steps: List[Step]):
+        log_ansi_encoded_string(bold=True, message="Statuses detailed info:")
+
+        for step in steps:
+            log_ansi_encoded_string(bold=True, message=f"{step.name} Status:")
+            log_ansi_encoded_string(message=step.get_status().as_string())
+
+    def _print_execution_time_summary(self):
+        """Log the ExecutionTimeTracker Summary."""
+        tracker = ExecutionTimeTracker()
+
+        summary_table: Dict[str, List[Union[str, float]]] = {
+            "Context": [],
+            "Execution Time Aggregate": [],
+        }
+
+        for key in sorted(tracker.state.state.keys()):
+            summary_table["Context"].append(key)
+            summary_table["Execution Time Aggregate"].append(
+                pretty_print_time_duration(tracker.state.state[key])
+            )
+
+        log_ansi_encoded_string(bold=True, message="Execution Time Summary")
+        log_ansi_encoded_string(message=f"\n{tabulate(summary_table, tablefmt='grid')}")
+
+    def _print_query_parsing_issues(self):
+        """Log the QueryParsingFailures Summary."""
+        query_failures = QueryParsingFailures()
+
+        summary_table: Dict[str, List[Optional[str]]] = {
+            "Query": [],
+            "Error": [],
+        }
+
+        for failure in query_failures:
+            summary_table["Query"].append(failure.query)
+            summary_table["Error"].append(failure.error)
+
+        if summary_table["Query"]:
+            log_ansi_encoded_string(bold=True, message="Query Parsing Error Summary")
+            log_ansi_encoded_string(
+                message=f"\n{tabulate(summary_table, tablefmt='grid', headers=list(summary_table.keys()))}"
+            )
+
+    def _get_failures(self, failure: Failure) -> List[Dict[str, Optional[str]]]:
+        return [
+            {
+                "From": failure.name,
+                "Entity Name": f.name,
+                "Message": f.error,
+                "Stack Trace": f.stackTrace,
+            }
+            for f in failure.failures
+        ]
+
+    def _print_failures_if_apply(self, failures: List[Failure]) -> None:
+        # take only the ones that contain failures
+        failures = [f for f in failures if f.failures]
+        if failures:
+            # create a list of dictionaries' list
+            all_data = [self._get_failures(failure) for failure in failures]
+            # create a single of dictionaries
+            data = [f for fs in all_data for f in fs]
+            # create a dictionary with a key and a list of values from the list
+            error_table = {k: [dic[k] for dic in data] for k in data[0]}
+            if len(list(error_table.items())[0][1]) > 100:
+                log_ansi_encoded_string(
+                    bold=True, message="Showing only the first 100 failures:"
+                )
+                # truncate list if number of values are over 100
+                error_table = {k: v[:100] for k, v in error_table.items()}
+            else:
+                log_ansi_encoded_string(bold=True, message="List of failures:")
+
+            log_ansi_encoded_string(
+                message=f"\n{tabulate(error_table, headers='keys', tablefmt='grid')}"
+            )

metadata/workflow/workflow_status_mixin.py

@@ -14,6 +14,7 @@ Add methods to the workflows for updating the IngestionPipeline status
 import traceback
 import uuid
 from datetime import datetime
+from enum import Enum
 from typing import Optional, Tuple
 
 from metadata.config.common import WorkflowExecutionError
@@ -29,6 +30,7 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import
 from metadata.generated.schema.metadataIngestion.workflow import (
     OpenMetadataWorkflowConfig,
 )
+from metadata.generated.schema.type.basic import Timestamp
 from metadata.ingestion.api.step import Step, Summary
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils.logger import ometa_logger
@@ -38,6 +40,11 @@ logger = ometa_logger()
 SUCCESS_THRESHOLD_VALUE = 90
 
 
+class WorkflowResultStatus(Enum):
+    SUCCESS = 0
+    FAILURE = 1
+
+
 class WorkflowStatusMixin:
     """
     Helper methods to manage IngestionPipeline status
@@ -63,7 +70,7 @@ class WorkflowStatusMixin:
         """
         if not self._run_id:
             if self.config.pipelineRunId:
-                self._run_id = str(self.config.pipelineRunId.__root__)
+                self._run_id = str(self.config.pipelineRunId.root)
             else:
                 self._run_id = str(uuid.uuid4())
 
@@ -74,8 +81,8 @@ class WorkflowStatusMixin:
         return PipelineStatus(
             runId=self.run_id,
             pipelineState=state,
-            startDate=self._start_ts,
-            timestamp=self._start_ts,
+            startDate=Timestamp(self._start_ts),
+            timestamp=Timestamp(self._start_ts),
         )
 
     def set_ingestion_pipeline_status(
@@ -91,21 +98,23 @@ class WorkflowStatusMixin:
             # if we don't have a related Ingestion Pipeline FQN, no status is set.
             if self.config.ingestionPipelineFQN and self.ingestion_pipeline:
                 pipeline_status = self.metadata.get_pipeline_status(
-                    self.ingestion_pipeline.fullyQualifiedName.__root__, self.run_id
+                    self.ingestion_pipeline.fullyQualifiedName.root, self.run_id
                 )
                 if not pipeline_status:
                     # We need to crete the status
                     pipeline_status = self._new_pipeline_status(state)
                 else:
                     # if workflow is ended then update the end date in status
-                    pipeline_status.endDate = datetime.now().timestamp() * 1000
+                    pipeline_status.endDate = Timestamp(
+                        int(datetime.now().timestamp() * 1000)
+                    )
                     pipeline_status.pipelineState = state
 
                 pipeline_status.status = (
                     ingestion_status if ingestion_status else pipeline_status.status
                 )
                 self.metadata.create_or_update_pipeline_status(
-                    self.ingestion_pipeline.fullyQualifiedName.__root__, pipeline_status
+                    self.ingestion_pipeline.fullyQualifiedName.root, pipeline_status
                 )
         except Exception as err:
             logger.debug(traceback.format_exc())
@@ -124,13 +133,13 @@ class WorkflowStatusMixin:
             self.set_ingestion_pipeline_status(PipelineState.failed)
             raise err
 
-    def result_status(self) -> int:
+    def result_status(self) -> WorkflowResultStatus:
         """
         Returns 1 if source status is failed, 0 otherwise.
         """
         if self.get_failures():
-            return 1
-        return 0
+            return WorkflowResultStatus.FAILURE
+        return WorkflowResultStatus.SUCCESS
 
     def build_ingestion_status(self) -> Optional[IngestionStatus]:
         """
@@ -139,8 +148,8 @@ class WorkflowStatusMixin:
         """
 
         return IngestionStatus(
-            __root__=[
-                StepSummary.parse_obj(Summary.from_step(step).dict())
+            [
+                StepSummary.model_validate(Summary.from_step(step).model_dump())
                 for step in self.workflow_steps()
             ]
         )