openmetadata-ingestion 1.6.2.2__py3-none-any.whl → 1.6.3.0__py3-none-any.whl

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-01-28T05:29:02+00:00
+#   timestamp: 2025-01-29T15:21:35+00:00
 
 from __future__ import annotations
 

metadata/ingestion/lineage/masker.py

@@ -14,8 +14,7 @@ Query masking utilities
 
 import traceback
 
-import sqlparse
-from sqlfluff.core import Linter
+from collate_sqllineage.runner import SQLPARSE_DIALECT, LineageRunner
 from sqlparse.sql import Comparison
 from sqlparse.tokens import Literal, Number, String
 
@@ -24,6 +23,7 @@ from metadata.ingestion.lineage.models import Dialect
 MASK_TOKEN = "?"
 
 
+# pylint: disable=protected-access
 def get_logger():
     # pylint: disable=import-outside-toplevel
     from metadata.utils.logger import utils_logger
@@ -31,18 +31,14 @@ def get_logger():
     return utils_logger()
 
 
-def mask_literals_with_sqlparse(query: str):
+def mask_literals_with_sqlparse(query: str, parser: LineageRunner):
     """
     Mask literals in a query using sqlparse.
     """
     logger = get_logger()
 
     try:
-        parsed = sqlparse.parse(query)  # Parse the query
-
-        if not parsed:
-            return query
-        parsed = parsed[0]
+        parsed = parser._parsed_result
 
         def mask_token(token):
             # Mask all literals: strings, numbers, or other literal values
@@ -79,17 +75,16 @@ def mask_literals_with_sqlparse(query: str):
     return query
 
 
-def mask_literals_with_sqlfluff(query: str, dialect: str = Dialect.ANSI.value) -> str:
+def mask_literals_with_sqlfluff(query: str, parser: LineageRunner) -> str:
     """
     Mask literals in a query using SQLFluff.
     """
     logger = get_logger()
     try:
-        # Initialize SQLFluff linter
-        linter = Linter(dialect=dialect)
+        if not parser._evaluated:
+            parser._eval()
 
-        # Parse the query
-        parsed = linter.parse_string(query)
+        parsed = parser._parsed_result
 
         def replace_literals(segment):
             """Recursively replace literals with placeholders."""
@@ -114,17 +109,21 @@ def mask_literals_with_sqlfluff(query: str, dialect: str = Dialect.ANSI.value) -
     return query
 
 
-def mask_query(query: str, dialect: str = Dialect.ANSI.value) -> str:
+def mask_query(
+    query: str, dialect: str = Dialect.ANSI.value, parser: LineageRunner = None
+) -> str:
     logger = get_logger()
     try:
-        sqlfluff_masked_query = mask_literals_with_sqlfluff(query, dialect)
-        sqlparse_masked_query = mask_literals_with_sqlparse(query)
-        # compare both masked queries and return the one with more masked tokens
-        if sqlfluff_masked_query.count(MASK_TOKEN) >= sqlparse_masked_query.count(
-            MASK_TOKEN
-        ):
-            return sqlfluff_masked_query
-        return sqlparse_masked_query
+        if not parser:
+            try:
+                parser = LineageRunner(query, dialect=dialect)
+                len(parser.source_tables)
+            except Exception:
+                parser = LineageRunner(query)
+                len(parser.source_tables)
+        if parser._dialect == SQLPARSE_DIALECT:
+            return mask_literals_with_sqlparse(query, parser)
+        return mask_literals_with_sqlfluff(query, parser)
     except Exception as exc:
         logger.debug(f"Failed to mask query with sqlfluff: {exc}")
         logger.debug(traceback.format_exc())

metadata/ingestion/lineage/parser.py

@@ -71,12 +71,13 @@ class LineageParser:
         self.query_parsing_success = True
         self.query_parsing_failure_reason = None
         self.dialect = dialect
-        self._masked_query = mask_query(self.query, dialect.value)
+        self.masked_query = None
         self._clean_query = self.clean_raw_query(query)
-        self._masked_clean_query = mask_query(self._clean_query, dialect.value)
         self.parser = self._evaluate_best_parser(
             self._clean_query, dialect=dialect, timeout_seconds=timeout_seconds
         )
+        if self.masked_query is None:
+            self.masked_query = mask_query(self._clean_query, parser=self.parser)
 
     @cached_property
     def involved_tables(self) -> Optional[List[Table]]:
@@ -95,7 +96,7 @@ class LineageParser:
         except SQLLineageException as exc:
             logger.debug(traceback.format_exc())
             logger.warning(
-                f"Cannot extract source table information from query [{self._masked_query}]: {exc}"
+                f"Cannot extract source table information from query [{self.masked_query or self.query}]: {exc}"
             )
             return None
 
@@ -334,12 +335,10 @@ class LineageParser:
                 )
 
                 if not table_left or not table_right:
-                    logger.warning(
-                        f"Can't extract table names when parsing JOIN information from {comparison}"
-                    )
                     logger.debug(
-                        f"Query: {mask_query(sql_statement, self.dialect.value)}"
+                        f"Can't extract table names when parsing JOIN information from {comparison}"
                     )
+                    logger.debug(f"Query: {self.masked_query}")
                     continue
 
                 left_table_column = TableColumn(table=table_left, column=column_left)
@@ -422,10 +421,9 @@ class LineageParser:
             lr_dialect.get_column_lineage()
             return lr_dialect
 
-        sqlfluff_count = 0
         try:
             lr_sqlfluff = get_sqlfluff_lineage_runner(query, dialect.value)
-            sqlfluff_count = len(lr_sqlfluff.get_column_lineage()) + len(
+            _ = len(lr_sqlfluff.get_column_lineage()) + len(
                 set(lr_sqlfluff.source_tables).union(
                     set(lr_sqlfluff.target_tables).union(
                         set(lr_sqlfluff.intermediate_tables)
@@ -438,23 +436,20 @@ class LineageParser:
                 f"Lineage with SqlFluff failed for the [{dialect.value}]. "
                 f"Parser has been running for more than {timeout_seconds} seconds."
             )
-            logger.debug(
-                f"{self.query_parsing_failure_reason}] query: [{self._masked_clean_query}]"
-            )
             lr_sqlfluff = None
         except Exception:
             self.query_parsing_success = False
             self.query_parsing_failure_reason = (
                 f"Lineage with SqlFluff failed for the [{dialect.value}]"
             )
-            logger.debug(
-                f"{self.query_parsing_failure_reason} query: [{self._masked_clean_query}]"
-            )
             lr_sqlfluff = None
 
+        if lr_sqlfluff:
+            return lr_sqlfluff
+
         lr_sqlparser = LineageRunner(query)
         try:
-            sqlparser_count = len(lr_sqlparser.get_column_lineage()) + len(
+            _ = len(lr_sqlparser.get_column_lineage()) + len(
                 set(lr_sqlparser.source_tables).union(
                     set(lr_sqlparser.target_tables).union(
                         set(lr_sqlparser.intermediate_tables)
@@ -463,21 +458,13 @@ class LineageParser:
             )
         except Exception:
             # if both runner have failed we return the usual one
+            logger.debug(f"Failed to parse query with sqlparse & sqlfluff: {query}")
             return lr_sqlfluff if lr_sqlfluff else lr_sqlparser
 
-        if lr_sqlfluff:
-            # if sqlparser retrieve more lineage info that sqlfluff
-            if sqlparser_count > sqlfluff_count:
-                self.query_parsing_success = False
-                self.query_parsing_failure_reason = (
-                    "Lineage computed with SqlFluff did not perform as expected "
-                    f"for the [{dialect.value}]"
-                )
-                logger.debug(
-                    f"{self.query_parsing_failure_reason} query: [{self._masked_clean_query}]"
-                )
-                return lr_sqlparser
-            return lr_sqlfluff
+        self.masked_query = mask_query(self._clean_query, parser=lr_sqlparser)
+        logger.debug(
+            f"Using sqlparse for lineage parsing for query: {self.masked_query}"
+        )
         return lr_sqlparser
 
     @staticmethod

metadata/ingestion/lineage/sql_lineage.py

@@ -37,7 +37,6 @@ from metadata.generated.schema.type.entityLineage import (
 from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either
-from metadata.ingestion.lineage.masker import mask_query
 from metadata.ingestion.lineage.models import (
     Dialect,
     QueryParsingError,
@@ -614,11 +613,11 @@ def get_lineage_by_query(
     """
     column_lineage = {}
     query_parsing_failures = QueryParsingFailures()
-    masked_query = mask_query(query, dialect.value)
 
     try:
-        logger.debug(f"Running lineage with query: {masked_query}")
         lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds)
+        masked_query = lineage_parser.masked_query or query
+        logger.debug(f"Running lineage with query: {masked_query}")
 
         raw_column_lineage = lineage_parser.column_lineage
         column_lineage.update(populate_column_lineage_map(raw_column_lineage))
@@ -715,11 +714,11 @@ def get_lineage_via_table_entity(
     """Get lineage from table entity"""
     column_lineage = {}
     query_parsing_failures = QueryParsingFailures()
-    masked_query = mask_query(query, dialect.value)
 
     try:
-        logger.debug(f"Getting lineage via table entity using query: {masked_query}")
         lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds)
+        masked_query = lineage_parser.masked_query or query
+        logger.debug(f"Getting lineage via table entity using query: {masked_query}")
         to_table_name = table_entity.name.root
 
         for from_table_name in lineage_parser.source_tables:

metadata/ingestion/ometa/mixins/es_mixin.py

@@ -344,7 +344,7 @@ class ESMixin(Generic[T]):
             # Get next page
             last_hit = response.hits.hits[-1] if response.hits.hits else None
             if not last_hit or not last_hit.sort:
-                logger.info("No more pages to fetch")
+                logger.debug("No more pages to fetch")
                 break
 
             after = ",".join(last_hit.sort)
@@ -402,19 +402,51 @@ class ESMixin(Generic[T]):
             "query": {
                 "bool": {
                     "must": [
-                        {"term": {"service.name.keyword": service_name}},
                         {
-                            "term": {
-                                "tableType": [
-                                    TableType.View.value,
-                                    TableType.MaterializedView.value,
-                                    TableType.SecureView.value,
-                                    TableType.Dynamic.value,
+                            "bool": {
+                                "should": [
+                                    {"term": {"service.name.keyword": service_name}}
+                                ]
+                            }
+                        },
+                        {
+                            "bool": {
+                                "must": [
+                                    {
+                                        "bool": {
+                                            "should": [
+                                                {
+                                                    "term": {
+                                                        "tableType": TableType.View.value
+                                                    }
+                                                },
+                                                {
+                                                    "term": {
+                                                        "tableType": TableType.MaterializedView.value
+                                                    }
+                                                },
+                                                {
+                                                    "term": {
+                                                        "tableType": TableType.SecureView.value
+                                                    }
+                                                },
+                                                {
+                                                    "term": {
+                                                        "tableType": TableType.Dynamic.value
+                                                    }
+                                                },
+                                            ]
+                                        }
+                                    }
                                 ]
                             }
                         },
-                        {"term": {"deleted": False}},
-                        {"exists": {"field": "schemaDefinition"}},
+                        {"bool": {"should": [{"term": {"deleted": False}}]}},
+                        {
+                            "bool": {
+                                "should": [{"exists": {"field": "schemaDefinition"}}]
+                            }
+                        },
                     ]
                 }
             }
@@ -429,10 +461,11 @@ class ESMixin(Generic[T]):
                 _, database_name, schema_name, table_name = fqn.split(
                     hit.source["fullyQualifiedName"]
                 )
-                yield TableView(
-                    view_definition=hit.source["schemaDefinition"],
-                    service_name=service_name,
-                    db_name=database_name,
-                    schema_name=schema_name,
-                    table_name=table_name,
-                )
+                if hit.source.get("schemaDefinition"):
+                    yield TableView(
+                        view_definition=hit.source["schemaDefinition"],
+                        service_name=service_name,
+                        db_name=database_name,
+                        schema_name=schema_name,
+                        table_name=table_name,
+                    )

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -63,7 +63,7 @@ class OMetaLineageMixin(Generic[T]):
                     )
             for column in updated or []:
                 if not isinstance(column, dict):
-                    data = column.dict()
+                    data = column.model_dump()
                 else:
                     data = column
                 if data.get("toColumn") and data.get("fromColumns"):

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -24,7 +24,18 @@ import os
 import traceback
 from datetime import datetime
 from pathlib import Path
-from typing import Dict, Iterable, List, Optional, Sequence, Set, Type, Union, cast
+from typing import (
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Set,
+    Type,
+    Union,
+    cast,
+    get_args,
+)
 
 import giturlparse
 import lkml
@@ -91,7 +102,7 @@ from metadata.generated.schema.type.entityReferenceList import EntityReferenceLi
 from metadata.generated.schema.type.usageRequest import UsageRequest
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
-from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
+from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
 from metadata.ingestion.lineage.parser import LineageParser
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import (
@@ -108,7 +119,6 @@ from metadata.ingestion.source.dashboard.looker.models import (
     LookMlView,
     ViewName,
 )
-from metadata.ingestion.source.dashboard.looker.parser import LkmlParser
 from metadata.ingestion.source.dashboard.looker.utils import _clone_repo
 from metadata.readers.file.api_reader import ReadersCredentials
 from metadata.readers.file.base import Reader
@@ -121,7 +131,6 @@ from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
-
 LIST_DASHBOARD_FIELDS = ["id", "title"]
 IMPORTED_PROJECTS_DIR = "imported_projects"
 
@@ -178,7 +187,7 @@ class LookerSource(DashboardServiceSource):
         self._explores_cache = {}
         self._repo_credentials: Optional[ReadersCredentials] = None
         self._reader_class: Optional[Type[Reader]] = None
-        self._project_parsers: Optional[Dict[str, LkmlParser]] = None
+        self._project_parsers: Optional[Dict[str, BulkLkmlParser]] = None
         self._main_lookml_repo: Optional[LookMLRepo] = None
         self._main__lookml_manifest: Optional[LookMLManifest] = None
         self._view_data_model: Optional[DashboardDataModel] = None
@@ -260,7 +269,7 @@ class LookerSource(DashboardServiceSource):
             self._main__lookml_manifest = self.__read_manifest(credentials)
 
     @property
-    def parser(self) -> Optional[Dict[str, LkmlParser]]:
+    def parser(self) -> Optional[Dict[str, BulkLkmlParser]]:
         if self.repository_credentials:
             return self._project_parsers
         return None
@@ -282,7 +291,7 @@ class LookerSource(DashboardServiceSource):
         """
         if self.repository_credentials:
             all_projects: Set[str] = {model.project_name for model in all_lookml_models}
-            self._project_parsers: Dict[str, LkmlParser] = {
+            self._project_parsers: Dict[str, BulkLkmlParser] = {
                 project_name: BulkLkmlParser(
                     reader=self.reader(Path(self._main_lookml_repo.path))
                 )
@@ -325,7 +334,7 @@ class LookerSource(DashboardServiceSource):
         """
         if not self._repo_credentials:
             if self.service_connection.gitCredentials and isinstance(
-                self.service_connection.gitCredentials, ReadersCredentials
+                self.service_connection.gitCredentials, get_args(ReadersCredentials)
             ):
                 self._repo_credentials = self.service_connection.gitCredentials
 
@@ -572,10 +581,12 @@ class LookerSource(DashboardServiceSource):
 
             if view.sql_table_name:
                 sql_table_name = self._render_table_name(view.sql_table_name)
-                source_table_name = self._clean_table_name(sql_table_name)
 
-                # View to the source is only there if we are informing the dbServiceNames
                 for db_service_name in db_service_names or []:
+                    dialect = self._get_db_dialect(db_service_name)
+                    source_table_name = self._clean_table_name(sql_table_name, dialect)
+
+                    # View to the source is only there if we are informing the dbServiceNames
                     yield self.build_lineage_request(
                         source=source_table_name,
                         db_service_name=db_service_name,
@@ -587,15 +598,9 @@ class LookerSource(DashboardServiceSource):
                 if not sql_query:
                     return
                 for db_service_name in db_service_names or []:
-                    db_service = self.metadata.get_by_name(
-                        DatabaseService, db_service_name
-                    )
-
                     lineage_parser = LineageParser(
                         sql_query,
-                        ConnectionTypeDialectMapper.dialect_of(
-                            db_service.connection.config.type.value
-                        ),
+                        self._get_db_dialect(db_service_name),
                         timeout_seconds=30,
                     )
                     if lineage_parser.source_tables:
@@ -615,6 +620,12 @@ class LookerSource(DashboardServiceSource):
                 )
             )
 
+    def _get_db_dialect(self, db_service_name) -> Dialect:
+        db_service = self.metadata.get_by_name(DatabaseService, db_service_name)
+        return ConnectionTypeDialectMapper.dialect_of(
+            db_service.connection.config.type.value
+        )
+
     def get_dashboards_list(self) -> List[DashboardBase]:
         """
         Get List of all dashboards
@@ -718,7 +729,7 @@ class LookerSource(DashboardServiceSource):
         return None
 
     @staticmethod
-    def _clean_table_name(table_name: str) -> str:
+    def _clean_table_name(table_name: str, dialect: Dialect = Dialect.ANSI) -> str:
         """
         sql_table_names might be renamed when defining
         an explore. E.g., customers as cust
@@ -726,7 +737,10 @@ class LookerSource(DashboardServiceSource):
         :return: clean table name
         """
 
-        return table_name.lower().split(" as ")[0].strip()
+        clean_table_name = table_name.lower().split(" as ")[0].strip()
+        if dialect == Dialect.BIGQUERY:
+            clean_table_name = clean_table_name.strip("`")
+        return clean_table_name
 
     @staticmethod
     def _render_table_name(table_name: str) -> str: