openmetadata-ingestion 1.6.3.2.dev1__py3-none-any.whl → 1.6.4.0__py3-none-any.whl

metadata/generated/schema/type/assetCertification.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/assetCertification.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/auditLog.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/auditLog.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/basic.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/basic.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 
@@ -227,6 +227,13 @@ class ComponentConfig(RootModel[Optional[Dict[str, Any]]]):
     root: Optional[Dict[str, Any]] = None
 
 
+class Map(BaseModel):
+    pass
+    model_config = ConfigDict(
+        extra='allow',
+    )
+
+
 class Status(Enum):
     success = 'success'
     failure = 'failure'

metadata/generated/schema/type/bulkOperationResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/bulkOperationResult.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/changeEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/changeEvent.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/changeEventType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/changeEventType.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/collectionDescriptor.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/collectionDescriptor.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvDocumentation.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvDocumentation.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvErrorType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvErrorType.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvFile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvFile.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvImportResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvImportResult.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperties/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00

metadata/generated/schema/type/customProperties/complexTypes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/customProperties/complexTypes.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperties/enumConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/customProperties/enumConfig.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperties/tableConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/customProperties/tableConfig.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperty.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/customProperty.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/dailyCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/dailyCount.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/databaseConnectionConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/databaseConnectionConfig.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHierarchy.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-02-07T08:43:03+00:00
+#   timestamp: 2025-02-19T13:24:04+00:00
 
 from __future__ import annotations
 

metadata/ingestion/lineage/masker.py

@@ -127,4 +127,4 @@ def mask_query(
     except Exception as exc:
         logger.debug(f"Failed to mask query with sqlfluff: {exc}")
         logger.debug(traceback.format_exc())
-    return query
+    return None

metadata/ingestion/lineage/models.py

@@ -96,7 +96,6 @@ class Dialect(Enum):
     IMPALA = "impala"
     MATERIALIZE = "materialize"
     MYSQL = "mysql"
-    MARIADB = "mariadb"
     ORACLE = "oracle"
     POSTGRES = "postgres"
     REDSHIFT = "redshift"
@@ -127,7 +126,7 @@ MAP_CONNECTION_TYPE_DIALECT: Dict[str, Dialect] = {
     str(MssqlType.Mssql.value): Dialect.TSQL,
     str(AzureSQLType.AzureSQL.value): Dialect.TSQL,
     str(TeradataType.Teradata.value): Dialect.TERADATA,
-    str(MariaDBType.MariaDB.value): Dialect.MARIADB,
+    str(MariaDBType.MariaDB.value): Dialect.MYSQL,
     str(SingleStoreType.SingleStore.value): Dialect.MYSQL,
 }
 

metadata/ingestion/lineage/parser.py

@@ -338,7 +338,7 @@ class LineageParser:
                     logger.debug(
                         f"Can't extract table names when parsing JOIN information from {comparison}"
                     )
-                    logger.debug(f"Query: {self.masked_query}")
+                    logger.debug(f"Query: {self.masked_query or self.query}")
                     continue
 
                 left_table_column = TableColumn(table=table_left, column=column_left)
@@ -463,7 +463,7 @@ class LineageParser:
 
         self.masked_query = mask_query(self._clean_query, parser=lr_sqlparser)
         logger.debug(
-            f"Using sqlparse for lineage parsing for query: {self.masked_query}"
+            f"Using sqlparse for lineage parsing for query: {self.masked_query or self.query}"
         )
         return lr_sqlparser
 

metadata/ingestion/lineage/sql_lineage.py

@@ -16,8 +16,10 @@ import traceback
 from collections import defaultdict
 from typing import Any, Iterable, List, Optional, Tuple, Union
 
+import networkx as nx
 from collate_sqllineage.core.models import Column, DataFunction
 from collate_sqllineage.core.models import Table as LineageTable
+from networkx import DiGraph
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.storedProcedure import (
@@ -509,6 +511,7 @@ def _create_lineage_by_table_name(
     column_lineage_map: dict,
     lineage_source: LineageSource = LineageSource.QueryLineage,
     procedure: Optional[EntityReference] = None,
+    graph: DiGraph = None,
 ) -> Iterable[Either[AddLineageRequest]]:
     """
     This method is to create a lineage between two tables
@@ -539,6 +542,11 @@ def _create_lineage_by_table_name(
                 logger.debug(
                     f"WARNING: Table entity [{table_name}] not found in OpenMetadata"
                 )
+        if graph is not None and (not from_table_entities or not to_table_entities):
+            graph.add_node(from_table, entity=from_table_entities)
+            graph.add_node(to_table, entity=to_table_entities)
+            graph.add_edge(from_table, to_table, query=masked_query)
+            return
 
         for from_entity, to_entity in itertools.product(
             from_table_entities or [], to_table_entities or []
@@ -606,6 +614,7 @@ def get_lineage_by_query(
     dialect: Dialect,
     timeout_seconds: int = LINEAGE_PARSING_TIMEOUT,
     lineage_source: LineageSource = LineageSource.QueryLineage,
+    graph: DiGraph = None,
 ) -> Iterable[Either[AddLineageRequest]]:
     """
     This method parses the query to get source, target and intermediate table names to create lineage,
@@ -616,8 +625,8 @@ def get_lineage_by_query(
 
     try:
         lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds)
-        masked_query = lineage_parser.masked_query or query
-        logger.debug(f"Running lineage with query: {masked_query}")
+        masked_query = lineage_parser.masked_query
+        logger.debug(f"Running lineage with query: {masked_query or query}")
 
         raw_column_lineage = lineage_parser.column_lineage
         column_lineage.update(populate_column_lineage_map(raw_column_lineage))
@@ -645,6 +654,7 @@ def get_lineage_by_query(
                         column_lineage_map=column_lineage,
                         lineage_source=lineage_source,
                         procedure=procedure,
+                        graph=graph,
                     )
             for target_table in lineage_parser.target_tables:
                 yield from _create_lineage_by_table_name(
@@ -682,11 +692,12 @@ def get_lineage_by_query(
                             column_lineage_map=column_lineage,
                             lineage_source=lineage_source,
                             procedure=procedure,
+                            graph=graph,
                         )
         if not lineage_parser.query_parsing_success:
             query_parsing_failures.add(
                 QueryParsingError(
-                    query=masked_query,
+                    query=masked_query or query,
                     error=lineage_parser.query_parsing_failure_reason,
                 )
             )
@@ -710,6 +721,7 @@ def get_lineage_via_table_entity(
     dialect: Dialect,
     timeout_seconds: int = LINEAGE_PARSING_TIMEOUT,
     lineage_source: LineageSource = LineageSource.QueryLineage,
+    graph: DiGraph = None,
 ) -> Iterable[Either[AddLineageRequest]]:
     """Get lineage from table entity"""
     column_lineage = {}
@@ -717,8 +729,10 @@ def get_lineage_via_table_entity(
 
     try:
         lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds)
-        masked_query = lineage_parser.masked_query or query
-        logger.debug(f"Getting lineage via table entity using query: {masked_query}")
+        masked_query = lineage_parser.masked_query
+        logger.debug(
+            f"Getting lineage via table entity using query: {masked_query or query}"
+        )
         to_table_name = table_entity.name.root
 
         for from_table_name in lineage_parser.source_tables:
@@ -743,6 +757,7 @@ def get_lineage_via_table_entity(
                     column_lineage_map=column_lineage,
                     lineage_source=lineage_source,
                     procedure=procedure,
+                    graph=graph,
                 ) or []
         if not lineage_parser.query_parsing_success:
             query_parsing_failures.add(
@@ -759,3 +774,94 @@ def get_lineage_via_table_entity(
                 stackTrace=traceback.format_exc(),
             )
         )
+
+
+def _process_sequence(
+    sequence: List[Any], graph: DiGraph
+) -> Iterable[Either[AddLineageRequest]]:
+    """
+    Process a sequence of nodes to generate lineage information.
+    """
+    from_node = None
+    queries = set()
+    clean_queries = False
+    previous_node = None
+    for node in sequence:
+        try:
+            if clean_queries:
+                queries.clear()
+                clean_queries = False
+            current_node = graph.nodes[node]
+            current_entity = current_node.get("entity")
+
+            if (
+                previous_node is not None
+                and graph.edges[(previous_node, node)].get("query") is not None
+            ):
+                queries.add(graph.edges[(previous_node, node)].get("query"))
+
+            if current_entity and from_node is not None:
+                for from_entity, to_entity in itertools.product(
+                    from_node.get("entity") or [], current_entity or []
+                ):
+                    if to_entity and from_entity:
+                        yield _build_table_lineage(
+                            to_entity=to_entity,
+                            from_entity=from_entity,
+                            to_table_raw_name=str(node),
+                            from_table_raw_name=str(from_node),
+                            masked_query="\n--------\n".join(queries),
+                            column_lineage_map={},
+                            lineage_source=LineageSource.QueryLineage,
+                            procedure=None,
+                        )
+                        clean_queries = True
+
+            if current_entity:
+                from_node = graph.nodes[node]
+            previous_node = node
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.error(f"Error creating lineage for node [{node}]: {exc}")
+
+
+def get_lineage_by_graph(
+    graph: DiGraph,
+) -> Iterable[Either[AddLineageRequest]]:
+    """
+    Generate lineage information from a directed graph.
+    This method processes a directed graph to extract lineage information by identifying
+    weakly connected components and traversing each component to generate sequences of nodes.
+    It then yields lineage information for each sequence.
+    Args:
+        graph (DiGraph): A directed graph representing the lineage.
+    Raises:
+        Exception: If an error occurs during the lineage creation process, it logs the error.
+    """
+    if graph is None:
+        return
+
+    # Get all weakly connected components
+    components = list(nx.weakly_connected_components(graph))
+
+    # Extract each component as an independent subgraph
+    independent_subtrees = [
+        graph.subgraph(component).copy() for component in components
+    ]
+
+    # Print results in the desired format
+    for subtree in independent_subtrees:
+        # Find a root node (node with no incoming edges)
+        root = [node for node in subtree if subtree.in_degree(node) == 0][0]
+
+        # Traverse from the root to get the sequence of nodes
+        current = root
+        sequence = [current]
+        while True:
+            successors = list(subtree.successors(current))
+            if not successors:
+                break
+            current = successors[0]
+            sequence.append(current)
+
+        yield from _process_sequence(sequence, subtree)

metadata/ingestion/ometa/mixins/query_mixin.py

@@ -42,6 +42,8 @@ class OMetaQueryMixin:
         return str(result.hexdigest())
 
     def _get_or_create_query(self, query: CreateQueryRequest) -> Optional[Query]:
+        if query.query.root is None:
+            return None
         query_hash = self._get_query_hash(query=query.query.root)
         query_entity = self.get_by_name(entity=Query, fqn=query_hash)
         if query_entity is None: