openmetadata-ingestion 1.6.5.0__py3-none-any.whl → 1.6.5.2__py3-none-any.whl

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-02-28T14:54:55+00:00
+#   timestamp: 2025-03-10T12:22:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-02-28T14:54:55+00:00
+#   timestamp: 2025-03-10T12:22:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-02-28T14:54:55+00:00
+#   timestamp: 2025-03-10T12:22:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-02-28T14:54:55+00:00
+#   timestamp: 2025-03-10T12:22:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-02-28T14:54:55+00:00
+#   timestamp: 2025-03-10T12:22:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-02-28T14:54:55+00:00
+#   timestamp: 2025-03-10T12:22:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-02-28T14:54:55+00:00
+#   timestamp: 2025-03-10T12:22:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-02-28T14:54:55+00:00
+#   timestamp: 2025-03-10T12:22:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-02-28T14:54:55+00:00
+#   timestamp: 2025-03-10T12:22:04+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-02-28T14:54:55+00:00
+#   timestamp: 2025-03-10T12:22:04+00:00
 
 from __future__ import annotations
 

metadata/ingestion/lineage/sql_lineage.py

@@ -47,6 +47,7 @@ from metadata.ingestion.lineage.models import (
 from metadata.ingestion.lineage.parser import LINEAGE_PARSING_TIMEOUT, LineageParser
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils import fqn
+from metadata.utils.elasticsearch import get_entity_from_es_result
 from metadata.utils.fqn import build_es_fqn_search_string
 from metadata.utils.logger import utils_logger
 from metadata.utils.lru_cache import LRU_CACHE_SIZE, LRUCache
@@ -516,7 +517,6 @@ def _create_lineage_by_table_name(
     """
     This method is to create a lineage between two tables
     """
-
     try:
         from_table_entities = get_table_entities_from_query(
             metadata=metadata,
@@ -543,9 +543,20 @@ def _create_lineage_by_table_name(
                     f"WARNING: Table entity [{table_name}] not found in OpenMetadata"
                 )
         if graph is not None and (not from_table_entities or not to_table_entities):
-            graph.add_node(from_table, entity=from_table_entities)
-            graph.add_node(to_table, entity=to_table_entities)
-            graph.add_edge(from_table, to_table, query=masked_query)
+            # Add nodes and edges with minimal data
+            graph.add_node(
+                from_table,
+                fqns=[table.fullyQualifiedName.root for table in from_table_entities]
+                if from_table_entities
+                else [],
+            )
+            graph.add_node(
+                to_table,
+                fqns=[table.fullyQualifiedName.root for table in to_table_entities]
+                if to_table_entities
+                else [],
+            )
+            graph.add_edge(from_table, to_table)
             return
 
         for from_entity, to_entity in itertools.product(
@@ -776,57 +787,107 @@ def get_lineage_via_table_entity(
         )
 
 
+def _get_lineage_for_path(
+    from_fqn: str,
+    to_fqn: str,
+    from_node: Any,
+    current_node: Any,
+    table_chain: List[str],
+    metadata: OpenMetadata,
+) -> Optional[Either[AddLineageRequest]]:
+    """
+    Get lineage for a pair of FQNs in the path
+    """
+    try:
+        to_entity = get_entity_from_es_result(
+            entity_list=metadata.es_search_from_fqn(
+                entity_type=Table,
+                fqn_search_string=to_fqn,
+            ),
+        )
+        from_entity = get_entity_from_es_result(
+            entity_list=metadata.es_search_from_fqn(
+                entity_type=Table,
+                fqn_search_string=from_fqn,
+            ),
+        )
+        if to_entity and from_entity:
+            # Create the table chain string
+            table_relationship = "--- TEMPT TABLE LINEAGE \n--- "
+            table_relationship += " > ".join(table_chain)
+            return _build_table_lineage(
+                to_entity=to_entity,
+                from_entity=from_entity,
+                to_table_raw_name=str(current_node),
+                from_table_raw_name=str(from_node),
+                masked_query=table_relationship,  # Using table chain as the query
+                column_lineage_map={},
+                lineage_source=LineageSource.QueryLineage,
+                procedure=None,
+            )
+    except Exception as exc:
+        logger.debug(traceback.format_exc())
+        logger.error(f"Error fetching table entities [{from_fqn} -> {to_fqn}]: {exc}")
+    return None
+
+
 def _process_sequence(
-    sequence: List[Any], graph: DiGraph
+    sequence: List[Any], graph: DiGraph, metadata: OpenMetadata
 ) -> Iterable[Either[AddLineageRequest]]:
     """
     Process a sequence of nodes to generate lineage information.
     """
     from_node = None
-    queries = set()
-    clean_queries = False
-    previous_node = None
+    table_chain = []
     for node in sequence:
         try:
-            if clean_queries:
-                queries.clear()
-                clean_queries = False
             current_node = graph.nodes[node]
-            current_entity = current_node.get("entity")
-
-            if (
-                previous_node is not None
-                and graph.edges[(previous_node, node)].get("query") is not None
-            ):
-                queries.add(graph.edges[(previous_node, node)].get("query"))
-
-            if current_entity and from_node is not None:
-                for from_entity, to_entity in itertools.product(
-                    from_node.get("entity") or [], current_entity or []
-                ):
-                    if to_entity and from_entity:
-                        yield _build_table_lineage(
-                            to_entity=to_entity,
-                            from_entity=from_entity,
-                            to_table_raw_name=str(node),
-                            from_table_raw_name=str(from_node),
-                            masked_query="\n--------\n".join(queries),
-                            column_lineage_map={},
-                            lineage_source=LineageSource.QueryLineage,
-                            procedure=None,
-                        )
-                        clean_queries = True
+            current_fqns = current_node.get("fqns", [])
+
+            # Add the current node name to the chain
+            table_chain.append(str(node).replace(f"{DEFAULT_SCHEMA_NAME}.", ""))
+
+            if current_fqns and from_node is not None:
+                from_fqns = from_node.get("fqns", [])
+                for from_fqn, to_fqn in itertools.product(from_fqns, current_fqns):
+                    lineage = _get_lineage_for_path(
+                        from_fqn=from_fqn,
+                        to_fqn=to_fqn,
+                        from_node=from_node,
+                        current_node=node,
+                        table_chain=table_chain,
+                        metadata=metadata,
+                    )
+                    if lineage:
+                        yield lineage
 
-            if current_entity:
+            if current_fqns:
                 from_node = graph.nodes[node]
-            previous_node = node
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Error creating lineage for node [{node}]: {exc}")
 
 
+def _get_paths_from_subtree(subtree: DiGraph) -> List[List[Any]]:
+    """
+    Get all paths from root nodes to leaf nodes in a subtree
+    """
+    paths = []
+    # Find all root nodes (nodes with no incoming edges)
+    root_nodes = [node for node in subtree if subtree.in_degree(node) == 0]
+    # Find all leaf nodes (nodes with no outgoing edges)
+    leaf_nodes = [node for node in subtree if subtree.out_degree(node) == 0]
+
+    # Find all simple paths from each root to each leaf
+    for root in root_nodes:
+        for leaf in leaf_nodes:
+            paths.extend(nx.all_simple_paths(subtree, root, leaf))
+    return paths
+
+
 def get_lineage_by_graph(
     graph: DiGraph,
+    metadata: OpenMetadata,
 ) -> Iterable[Either[AddLineageRequest]]:
     """
     Generate lineage information from a directed graph.
@@ -835,6 +896,7 @@ def get_lineage_by_graph(
     It then yields lineage information for each sequence.
     Args:
         graph (DiGraph): A directed graph representing the lineage.
+        metadata (OpenMetadata): OpenMetadata client instance to fetch table entities
     Raises:
         Exception: If an error occurs during the lineage creation process, it logs the error.
     """
@@ -844,24 +906,8 @@ def get_lineage_by_graph(
     # Get all weakly connected components
     components = list(nx.weakly_connected_components(graph))
 
-    # Extract each component as an independent subgraph
-    independent_subtrees = [
-        graph.subgraph(component).copy() for component in components
-    ]
-
-    # Print results in the desired format
-    for subtree in independent_subtrees:
-        # Find a root node (node with no incoming edges)
-        root = [node for node in subtree if subtree.in_degree(node) == 0][0]
-
-        # Traverse from the root to get the sequence of nodes
-        current = root
-        sequence = [current]
-        while True:
-            successors = list(subtree.successors(current))
-            if not successors:
-                break
-            current = successors[0]
-            sequence.append(current)
-
-        yield from _process_sequence(sequence, subtree)
+    # Extract each component as an independent subgraph and process paths
+    for component in components:
+        subtree = graph.subgraph(component).copy()
+        for path in _get_paths_from_subtree(subtree):
+            yield from _process_sequence(path, subtree, metadata)

metadata/ingestion/source/dashboard/sigma/client.py

@@ -94,14 +94,39 @@ class SigmaApiClient:
         )
         return result.access_token, result.expires_in
 
-    def get_dashboards(self) -> Optional[List[Workbook]]:
+    def test_get_dashboards(self) -> Optional[List[Workbook]]:
         """
-        method to fetch dashboards from api
+        method to test fetch dashboards from api
         """
+        result = self.client.get("/workbooks")
         result = WorkBookResponseDetails.model_validate(self.client.get("/workbooks"))
         if result:
             return result.entries
 
+    def get_dashboards(self) -> Optional[List[Workbook]]:
+        """
+        method to fetch dashboards from api
+        """
+        workbooks = []
+        try:
+            result = self.client.get("/workbooks")
+            result = WorkBookResponseDetails.model_validate(
+                self.client.get("/workbooks")
+            )
+            if result:
+                workbooks.extend(result.entries)
+                while result.nextPage:
+                    data = {"page": int(result.nextPage)}
+                    result = WorkBookResponseDetails.model_validate(
+                        self.client.get("/workbooks", data=data)
+                    )
+                    if result:
+                        workbooks.extend(result.entries)
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.error(f"Error fetching Dashboards: {exc}")
+        return workbooks
+
     def get_dashboard_detail(self, workbook_id: str) -> Optional[WorkbookDetails]:
         """
         method to fetch dashboard details from api
@@ -134,6 +159,36 @@ class SigmaApiClient:
             logger.warning(f"Failed to fetch owner details for owner {owner_id}: {exc}")
         return None
 
+    def get_page_elements(
+        self, workbook_id: str, page_id: str
+    ) -> Optional[List[Elements]]:
+        """
+        method to fetch dashboards page elements from api
+        """
+        elements = []
+        try:
+            result = ElementsResponse.model_validate(
+                self.client.get(f"/workbooks/{workbook_id}/pages/{page_id}/elements")
+            )
+            if result:
+                elements.extend(result.entries)
+                while result.nextPage:
+                    data = {"page": int(result.nextPage)}
+                    result = ElementsResponse.model_validate(
+                        self.client.get(
+                            f"/workbooks/{workbook_id}/pages/{page_id}/elements",
+                            data=data,
+                        )
+                    )
+                    if result:
+                        elements.extend(result.entries)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Failed to fetch page elements for workbook {workbook_id}: {exc}"
+            )
+        return elements
+
     def get_chart_details(self, workbook_id: str) -> Optional[List[Elements]]:
         """
         method to fetch dashboards chart details from api
@@ -143,13 +198,21 @@ class SigmaApiClient:
             pages = WorkBookPageResponse.model_validate(
                 self.client.get(f"/workbooks/{workbook_id}/pages")
             )
-            for page in pages.entries:
-                elements = ElementsResponse.model_validate(
+            if not pages.entries:
+                return None
+            while pages.nextPage:
+                pages = WorkBookPageResponse.model_validate(
                     self.client.get(
-                        f"/workbooks/{workbook_id}/pages/{page.pageId}/elements"
+                        f"/workbooks/{workbook_id}/pages",
+                        data={"page": int(pages.nextPage)},
                     )
                 )
-                elements_list.extend(elements.entries or [])
+                if not pages.entries:
+                    break
+                for page in pages.entries:
+                    elements_list.extend(
+                        self.get_page_elements(workbook_id, page.pageId)
+                    )
             return elements_list
         except Exception as exc:  # pylint: disable=broad-except
             logger.debug(traceback.format_exc())

metadata/ingestion/source/dashboard/sigma/connection.py

@@ -56,7 +56,10 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
-    test_fn = {"GetToken": client.get_auth_token, "GetWorkbooks": client.get_dashboards}
+    test_fn = {
+        "GetToken": client.get_auth_token,
+        "GetWorkbooks": client.test_get_dashboards,
+    }
 
     return test_connection_steps(
         metadata=metadata,

metadata/ingestion/source/dashboard/sigma/models.py

@@ -42,6 +42,8 @@ class WorkbookDetails(BaseModel):
 
 class WorkBookResponseDetails(BaseModel):
     entries: Optional[List[Workbook]] = []
+    total: int
+    nextPage: Optional[str] = None
 
 
 class OwnerDetails(BaseModel):
@@ -55,6 +57,8 @@ class WorkBookPage(BaseModel):
 
 class WorkBookPageResponse(BaseModel):
     entries: Optional[List[WorkBookPage]] = []
+    total: int
+    nextPage: Optional[str] = None
 
 
 class Elements(BaseModel):
@@ -66,6 +70,8 @@ class Elements(BaseModel):
 
 class ElementsResponse(BaseModel):
     entries: Optional[List[Elements]] = []
+    total: int
+    nextPage: Optional[str] = None
 
 
 class EdgeSource(BaseModel):

metadata/ingestion/source/database/lineage_source.py

@@ -309,7 +309,9 @@ class LineageSource(QueryParserSource, ABC):
         if self.source_config.processQueryLineage:
             if hasattr(self.service_connection, "supportsLineageExtraction"):
                 yield from self.yield_query_lineage() or []
-                yield from get_lineage_by_graph(graph=self.graph)
+                yield from get_lineage_by_graph(
+                    graph=self.graph, metadata=self.metadata
+                )
             else:
                 logger.warning(
                     f"Lineage extraction is not supported for {str(self.service_connection.type.value)} connection"

metadata/ingestion/source/database/snowflake/metadata.py

@@ -115,6 +115,7 @@ from metadata.utils.tag_utils import get_ometa_tag_and_classification
 ischema_names["VARIANT"] = VARIANT
 ischema_names["GEOGRAPHY"] = create_sqlalchemy_type("GEOGRAPHY")
 ischema_names["GEOMETRY"] = create_sqlalchemy_type("GEOMETRY")
+ischema_names["VECTOR"] = create_sqlalchemy_type("VECTOR")
 
 logger = ingestion_logger()
 

metadata/ingestion/source/database/snowflake/utils.py

@@ -300,7 +300,8 @@ def get_schema_columns(self, connection, schema, **kw):
             sa_util.warn(
                 f"Did not recognize type '{coltype}' of column '{column_name}'"
             )
-            col_type = sqltypes.NULLTYPE
+            col_type = sqltypes.NullType
+            type_instance = col_type()
         else:
             if issubclass(col_type, FLOAT):
                 col_type_kw["precision"] = numeric_precision
@@ -310,8 +311,7 @@ def get_schema_columns(self, connection, schema, **kw):
                 col_type_kw["scale"] = numeric_scale
             elif issubclass(col_type, (sqltypes.String, sqltypes.BINARY)):
                 col_type_kw["length"] = character_maximum_length
-
-        type_instance = col_type(**col_type_kw)
+            type_instance = col_type(**col_type_kw)
 
         current_table_pks = schema_primary_keys.get(table_name)