openmetadata-ingestion 1.8.6.0__py3-none-any.whl → 1.8.7.0__py3-none-any.whl

metadata/generated/schema/type/entityHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHierarchy.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-07-17T22:32:01+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/great_expectations/action.py

@@ -322,6 +322,16 @@ class OpenMetadataValidationAction(ValidationAction):
         fqn_ = cast(str, fqn_)
         return fqn_
 
+    def _get_test_case_description(self, result: dict) -> str:
+        """Get test case description from GE test result"""
+        if self.expectation_suite:
+            expectation = self._get_expectation_config(result)
+            if expectation:
+                meta: Optional[Dict] = expectation.get("meta")
+                if meta:
+                    return meta.get("description", "")
+        return ""
+
     def _get_test_case_params_value(self, result: dict) -> List[TestCaseParameterValue]:
         """Build test case parameter value from GE test result"""
         if self.expectation_suite:
@@ -545,6 +555,7 @@ class OpenMetadataValidationAction(ValidationAction):
                 ),
                 test_definition_fqn=test_definition.fullyQualifiedName.root,
                 test_case_parameter_values=self._get_test_case_params_value(result),
+                description=self._get_test_case_description(result),
             )
 
             self.ometa_conn.add_test_case_results(

metadata/ingestion/lineage/sql_lineage.py

@@ -14,7 +14,7 @@ Helper functions to handle SQL lineage operations
 import itertools
 import traceback
 from collections import defaultdict
-from typing import Any, Iterable, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
 
 import networkx as nx
 from collate_sqllineage.core.models import Column, DataFunction
@@ -894,7 +894,7 @@ def _get_paths_from_subtree(subtree: DiGraph) -> List[List[Any]]:
 
 
 def get_lineage_by_graph(
-    graph: DiGraph,
+    graph: Optional[DiGraph],
     metadata: OpenMetadata,
 ) -> Iterable[Either[AddLineageRequest]]:
     """
@@ -922,3 +922,27 @@ def get_lineage_by_graph(
         subtree = graph.subgraph(component).copy()
         for path in _get_paths_from_subtree(subtree):
             yield from _process_sequence(path, subtree, metadata)
+
+
+def get_lineage_by_procedure_graph(
+    procedure_graph_map: Optional[Dict],
+    metadata: OpenMetadata,
+) -> Iterable[Either[AddLineageRequest]]:
+    """
+    Generate lineage information from a directed graph.
+    """
+    if procedure_graph_map is None:
+        return
+
+    for procedure_and_procedure_graph in procedure_graph_map.values():
+        for either_lineage in get_lineage_by_graph(
+            graph=procedure_and_procedure_graph.graph,
+            metadata=metadata,
+        ):
+            if either_lineage.left is None and either_lineage.right.edge.lineageDetails:
+                either_lineage.right.edge.lineageDetails.pipeline = EntityReference(
+                    id=procedure_and_procedure_graph.procedure.id,
+                    type="storedProcedure",
+                )
+
+            yield either_lineage

metadata/ingestion/ometa/mixins/tests_mixin.py

@@ -169,6 +169,7 @@ class OMetaTestsMixin:
         entity_link: Optional[str] = None,
         test_definition_fqn: Optional[str] = None,
         test_case_parameter_values: Optional[List[TestCaseParameterValue]] = None,
+        description: Optional[str] = None,
     ):
         """Get or create a test case
 
@@ -197,6 +198,7 @@ class OMetaTestsMixin:
                 entityLink=entity_link,
                 testDefinition=test_definition_fqn,
                 parameterValues=test_case_parameter_values,
+                description=description,
             )  # type: ignore
         )
         return test_case

metadata/ingestion/ometa/ometa_api.py

@@ -15,7 +15,7 @@ models from the JSON schemas and provides a typed approach to
 working with OpenMetadata entities.
 """
 import traceback
-from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union
+from typing import Any, Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union
 
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -145,6 +145,7 @@ class OpenMetadata(
         self,
         config: OpenMetadataConnection,
         raw_data: bool = False,
+        additional_client_config_arguments: Optional[Dict[str, Any]] = None,
     ):
         self.config = config
 
@@ -161,14 +162,17 @@ class OpenMetadata(
         extra_headers: Optional[dict[str, str]] = None
         if self.config.extraHeaders:
             extra_headers = self.config.extraHeaders.root
-        client_config: ClientConfig = ClientConfig(
+
+        client_config = ClientConfig(
             base_url=self.config.hostPort,
             api_version=self.config.apiVersion,
             auth_header="Authorization",
             extra_headers=extra_headers,
             auth_token=self._auth_provider.get_access_token,
             verify=get_verify_ssl(self.config.sslConfig),
+            **(additional_client_config_arguments or {}),
         )
+
         self.client = REST(client_config)
         self._use_raw_data = raw_data
         if self.config.enableVersionValidation:

metadata/ingestion/source/database/lineage_source.py

@@ -41,6 +41,7 @@ from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Diale
 from metadata.ingestion.lineage.sql_lineage import (
     get_column_fqn,
     get_lineage_by_graph,
+    get_lineage_by_procedure_graph,
     get_lineage_by_query,
 )
 from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
@@ -49,7 +50,7 @@ from metadata.ingestion.source.database.query_parser_source import QueryParserSo
 from metadata.ingestion.source.models import TableView
 from metadata.utils import fqn
 from metadata.utils.db_utils import get_view_lineage
-from metadata.utils.filters import filter_by_table
+from metadata.utils.filters import filter_by_database, filter_by_schema, filter_by_table
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -314,9 +315,19 @@ class LineageSource(QueryParserSource, ABC):
     ) -> Iterable[Either[AddLineageRequest]]:
         try:
             for view in views:
-                if filter_by_table(
-                    self.source_config.tableFilterPattern,
-                    view.table_name,
+                if (
+                    filter_by_database(
+                        self.source_config.databaseFilterPattern,
+                        view.db_name,
+                    )
+                    or filter_by_schema(
+                        self.source_config.schemaFilterPattern,
+                        view.schema_name,
+                    )
+                    or filter_by_table(
+                        self.source_config.tableFilterPattern,
+                        view.table_name,
+                    )
                 ):
                     self.status.filter(
                         view.table_name,
@@ -441,10 +452,21 @@ class LineageSource(QueryParserSource, ABC):
         Based on the query logs, prepare the lineage
         and send it to the sink
         """
+        if (
+            self.procedure_graph_map is None
+            and self.source_config.enableTempTableLineage
+        ):
+            # Create a dictionary to store the directed graph for each procedure
+            self.procedure_graph_map = {}
+
         if self.source_config.processViewLineage:
             yield from self.yield_view_lineage() or []
         if self.source_config.processStoredProcedureLineage:
             yield from self.yield_procedure_lineage() or []
+            yield from get_lineage_by_procedure_graph(
+                procedure_graph_map=self.procedure_graph_map,
+                metadata=self.metadata,
+            )
         if self.source_config.processQueryLineage:
             if hasattr(self.service_connection, "supportsLineageExtraction"):
                 yield from self.yield_query_lineage() or []

metadata/ingestion/source/database/query_parser_source.py

@@ -64,6 +64,7 @@ class QueryParserSource(Source, ABC):
         self.source_config = self.config.sourceConfig.config
         self.start, self.end = get_start_and_end(self.source_config.queryLogDuration)
         self.graph = None
+        self.procedure_graph_map = None
 
         self.engine = None
         if get_engine:

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -19,6 +19,7 @@ from collections import defaultdict
 from datetime import datetime
 from typing import Dict, Iterable, List, Optional, Union
 
+from networkx import DiGraph
 from pydantic import BaseModel, ConfigDict, Field
 from sqlalchemy.engine import Engine
 
@@ -41,7 +42,12 @@ from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query
 from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
 from metadata.ingestion.models.topology import Queue
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.utils.filters import filter_by_stored_procedure
+from metadata.utils.filters import (
+    filter_by_database,
+    filter_by_schema,
+    filter_by_stored_procedure,
+)
+from metadata.utils.helpers import pprint_format_object
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.stored_procedures import get_procedure_name_from_call
 from metadata.utils.time_utils import datetime_to_timestamp
@@ -80,6 +86,17 @@ class ProcedureAndQuery(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
 
+class ProcedureAndProcedureGraph(BaseModel):
+    """
+    Model to hold the procedure and its graph
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    procedure: StoredProcedure
+    graph: DiGraph
+
+
 class StoredProcedureLineageMixin(ABC):
     """
     The full flow is:
@@ -137,6 +154,17 @@ class StoredProcedureLineageMixin(ABC):
                     )
                 )
 
+        queries_count_per_procedure = {
+            procedure_name: len(queries)
+            for procedure_name, queries in queries_dict.items()
+        }
+        logger.info(
+            f"Count of queries executed for stored procedures: {sum(queries_count_per_procedure.values())}"
+        )
+        logger.info(
+            f"Count of queries per stored procedure: {pprint_format_object(queries_count_per_procedure)}"
+        )
+
         return queries_dict
 
     @staticmethod
@@ -166,6 +194,18 @@ class StoredProcedureLineageMixin(ABC):
             query_type=query_by_procedure.query_type,
             query_text=query_by_procedure.query_text,
         ):
+            graph = None
+            if self.source_config.enableTempTableLineage:
+                if not self.procedure_graph_map.get(procedure.fullyQualifiedName.root):
+                    # Map to store the directed graph for each procedure with its FQN as key
+                    self.procedure_graph_map[
+                        procedure.fullyQualifiedName.root
+                    ] = ProcedureAndProcedureGraph(procedure=procedure, graph=DiGraph())
+
+                graph = self.procedure_graph_map.get(
+                    procedure.fullyQualifiedName.root
+                ).graph
+
             self.stored_procedure_query_lineage = True
             for either_lineage in get_lineage_by_query(
                 self.metadata,
@@ -178,6 +218,7 @@ class StoredProcedureLineageMixin(ABC):
                 ),
                 timeout_seconds=self.source_config.parsingTimeoutLimit,
                 lineage_source=LineageSource.QueryLineage,
+                graph=graph,
             ):
                 if (
                     either_lineage.left is None
@@ -290,9 +331,19 @@ class StoredProcedureLineageMixin(ABC):
             or []
         ):
             if procedure:
-                if filter_by_stored_procedure(
-                    self.source_config.storedProcedureFilterPattern,
-                    procedure.name.root,
+                if (
+                    filter_by_database(
+                        self.source_config.databaseFilterPattern,
+                        procedure.database.name,
+                    )
+                    or filter_by_schema(
+                        self.source_config.schemaFilterPattern,
+                        procedure.databaseSchema.name,
+                    )
+                    or filter_by_stored_procedure(
+                        self.source_config.storedProcedureFilterPattern,
+                        procedure.name.root,
+                    )
                 ):
                     self.status.filter(
                         procedure.name.root,

metadata/utils/helpers.py

@@ -17,6 +17,7 @@ from __future__ import annotations
 
 import hashlib
 import itertools
+import pprint
 import re
 import shutil
 import sys
@@ -568,3 +569,10 @@ def evaluate_threshold(threshold: int, operator: str, result: int) -> bool:
         f"Invalid threshold: {threshold}, "
         "Allowed format: <, >, <=, >=, ==, !=. Example: >5"
     )
+
+
+def pprint_format_object(data: Any) -> str:
+    """
+    Pretty print an object in a format that is easy to read
+    """
+    return pprint.pformat(data, width=150)