openmetadata-ingestion 1.8.1.0__py3-none-any.whl → 1.8.3.0__py3-none-any.whl

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-07-01T14:19:32+00:00
+#   timestamp: 2025-07-10T07:55:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-07-01T14:19:32+00:00
+#   timestamp: 2025-07-10T07:55:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-07-01T14:19:32+00:00
+#   timestamp: 2025-07-10T07:55:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-07-01T14:19:32+00:00
+#   timestamp: 2025-07-10T07:55:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-07-01T14:19:32+00:00
+#   timestamp: 2025-07-10T07:55:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-07-01T14:19:32+00:00
+#   timestamp: 2025-07-10T07:55:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-07-01T14:19:32+00:00
+#   timestamp: 2025-07-10T07:55:35+00:00
 
 from __future__ import annotations
 

metadata/ingestion/lineage/masker.py

@@ -13,6 +13,7 @@ Query masking utilities
 """
 
 import traceback
+from typing import Optional
 
 from cachetools import LRUCache
 from collate_sqllineage.runner import SQLPARSE_DIALECT, LineageRunner
@@ -26,6 +27,7 @@ MASK_TOKEN = "?"
 # Cache size is 128 to avoid memory issues
 masked_query_cache = LRUCache(maxsize=128)
 
+
 # pylint: disable=protected-access
 def get_logger():
     # pylint: disable=import-outside-toplevel
@@ -52,7 +54,6 @@ def mask_literals_with_sqlparse(query: str, parser: LineageRunner):
                 Literal.Number.Integer,
                 Literal.Number.Float,
                 Literal.String.Single,
-                Literal.String.Symbol,
             ):
                 token.value = MASK_TOKEN
             elif token.is_group:
@@ -113,7 +114,9 @@ def mask_literals_with_sqlfluff(query: str, parser: LineageRunner) -> str:
 
 
 def mask_query(
-    query: str, dialect: str = Dialect.ANSI.value, parser: LineageRunner = None
+    query: str,
+    dialect: str = Dialect.ANSI.value,
+    parser: Optional[LineageRunner] = None,
 ) -> str:
     """
     Mask a query using sqlparse or sqlfluff.

metadata/ingestion/lineage/sql_lineage.py

@@ -333,9 +333,11 @@ def get_source_table_names(
     """
     try:
         if not isinstance(source_table, DataFunction):
-            yield EntityReference(
-                id=procedure.id.root, type="storedProcedure"
-            ) if procedure else None, str(source_table)
+            yield (
+                EntityReference(id=procedure.id.root, type="storedProcedure")
+                if procedure
+                else None
+            ), str(source_table)
         else:
             yield from __process_udf_table_names(
                 metadata,
@@ -547,15 +549,19 @@ def _create_lineage_by_table_name(
             # Add nodes and edges with minimal data
             graph.add_node(
                 from_table,
-                fqns=[table.fullyQualifiedName.root for table in from_table_entities]
-                if from_table_entities
-                else [],
+                fqns=(
+                    [table.fullyQualifiedName.root for table in from_table_entities]
+                    if from_table_entities
+                    else []
+                ),
             )
             graph.add_node(
                 to_table,
-                fqns=[table.fullyQualifiedName.root for table in to_table_entities]
-                if to_table_entities
-                else [],
+                fqns=(
+                    [table.fullyQualifiedName.root for table in to_table_entities]
+                    if to_table_entities
+                    else []
+                ),
             )
             graph.add_edge(from_table, to_table)
             return

metadata/ingestion/source/dashboard/dashboard_service.py

@@ -13,7 +13,7 @@ Base class for ingesting dashboard services
 """
 import traceback
 from abc import ABC, abstractmethod
-from typing import Any, Iterable, List, Optional, Set, Union
+from typing import Any, Iterable, List, Optional, Set, Tuple, Union
 
 from pydantic import BaseModel, Field
 from typing_extensions import Annotated
@@ -251,7 +251,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: Any,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
@@ -340,16 +340,27 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                         f"Error to yield dashboard lineage details for data model name [{str(datamodel)}]: {err}"
                     )
 
-    def get_db_service_names(self) -> List[str]:
+    def get_db_service_prefixes(self) -> List[str]:
         """
-        Get the list of db service names
+        Get the list of db service prefixes
         """
         return (
-            self.source_config.lineageInformation.dbServiceNames or []
+            self.source_config.lineageInformation.dbServicePrefixes or []
             if self.source_config.lineageInformation
             else []
         )
 
+    def parse_db_service_prefix(
+        self, db_service_prefix: Optional[str]
+    ) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
+        """
+        Parse the db service prefix
+        Returns:
+            Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: service, database, schema, table
+        """
+        prefix_parts = (db_service_prefix or "").split(".")
+        return prefix_parts + ([None] * (4 - len(prefix_parts)))
+
     def yield_dashboard_lineage(
         self, dashboard_details: Any
     ) -> Iterable[Either[OMetaLineageRequest]]:
@@ -364,15 +375,12 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
             yield from self.yield_lineage_request(lineage)
 
         # yield datamodel lineage with tables from db services
-        db_service_names = self.get_db_service_names()
-        if not db_service_names:
+        db_service_prefixes = self.get_db_service_prefixes()
+        for db_service_prefix in db_service_prefixes or [None]:
             for lineage in (
-                self.yield_dashboard_lineage_details(dashboard_details) or []
-            ):
-                yield from self.yield_lineage_request(lineage)
-        for db_service_name in db_service_names or []:
-            for lineage in (
-                self.yield_dashboard_lineage_details(dashboard_details, db_service_name)
+                self.yield_dashboard_lineage_details(
+                    dashboard_details, db_service_prefix
+                )
                 or []
             ):
                 yield from self.yield_lineage_request(lineage)

metadata/ingestion/source/dashboard/domodashboard/metadata.py

@@ -127,9 +127,11 @@ class DomodashboardSource(DashboardServiceSource):
                 name=EntityName(dashboard_details.id),
                 sourceUrl=SourceUrl(dashboard_url),
                 displayName=dashboard_details.name,
-                description=Markdown(dashboard_details.description)
-                if dashboard_details.description
-                else None,
+                description=(
+                    Markdown(dashboard_details.description)
+                    if dashboard_details.description
+                    else None
+                ),
                 charts=[
                     FullyQualifiedEntityName(
                         fqn.build(
@@ -228,9 +230,11 @@ class DomodashboardSource(DashboardServiceSource):
                     yield Either(
                         right=CreateChartRequest(
                             name=EntityName(str(chart_id)),
-                            description=Markdown(chart.description)
-                            if chart.description
-                            else None,
+                            description=(
+                                Markdown(chart.description)
+                                if chart.description
+                                else None
+                            ),
                             displayName=chart.name,
                             sourceUrl=SourceUrl(chart_url),
                             service=self.context.get().dashboard_service,
@@ -250,6 +254,6 @@ class DomodashboardSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: dict,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """No lineage implemented"""

metadata/ingestion/source/dashboard/lightdash/metadata.py

@@ -124,9 +124,11 @@ class LightdashSource(DashboardServiceSource):
                 sourceUrl=SourceUrl(dashboard_url),
                 project=self.client.get_project_name(dashboard_details.projectUuid),
                 displayName=dashboard_details.name,
-                description=Markdown(dashboard_details.description)
-                if dashboard_details.description
-                else None,
+                description=(
+                    Markdown(dashboard_details.description)
+                    if dashboard_details.description
+                    else None
+                ),
                 charts=[
                     FullyQualifiedEntityName(
                         fqn.build(
@@ -178,9 +180,9 @@ class LightdashSource(DashboardServiceSource):
                     right=CreateChartRequest(
                         name=EntityName(chart.uuid),
                         displayName=chart.name,
-                        description=Markdown(chart.description)
-                        if chart.description
-                        else None,
+                        description=(
+                            Markdown(chart.description) if chart.description else None
+                        ),
                         sourceUrl=SourceUrl(chart_url),
                         service=self.context.get().dashboard_service,
                         chartType=chart_type,
@@ -194,7 +196,8 @@ class LightdashSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: LightdashDashboard,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
+        _: Optional[str] = None,
     ) -> Optional[Iterable[AddLineageRequest]]:
         """Get lineage method
 

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -247,7 +247,7 @@ class LookerSource(DashboardServiceSource):
                 BitBucketCredentials,
                 GitlabCredentials,
             ]
-        ]
+        ],
     ) -> "LookMLRepo":
         repo_name = (
             f"{credentials.repositoryOwner.root}/{credentials.repositoryName.root}"
@@ -471,9 +471,9 @@ class LookerSource(DashboardServiceSource):
                 explore_datamodel = CreateDashboardDataModelRequest(
                     name=EntityName(datamodel_name),
                     displayName=model.name,
-                    description=Markdown(model.description)
-                    if model.description
-                    else None,
+                    description=(
+                        Markdown(model.description) if model.description else None
+                    ),
                     service=self.context.get().dashboard_service,
                     tags=get_tag_labels(
                         metadata=self.metadata,
@@ -586,9 +586,9 @@ class LookerSource(DashboardServiceSource):
                 data_model_request = CreateDashboardDataModelRequest(
                     name=EntityName(datamodel_view_name),
                     displayName=view.name,
-                    description=Markdown(view.description)
-                    if view.description
-                    else None,
+                    description=(
+                        Markdown(view.description) if view.description else None
+                    ),
                     service=self.context.get().dashboard_service,
                     tags=get_tag_labels(
                         metadata=self.metadata,
@@ -816,12 +816,15 @@ class LookerSource(DashboardServiceSource):
                     " while processing view lineage."
                 )
 
-            db_service_names = self.get_db_service_names()
+            db_service_prefixes = self.get_db_service_prefixes()
 
             if view.sql_table_name:
                 sql_table_name = self._render_table_name(view.sql_table_name)
 
-                for db_service_name in db_service_names or []:
+                for db_service_prefix in db_service_prefixes or []:
+                    db_service_name, *_ = self.parse_db_service_prefix(
+                        db_service_prefix
+                    )
                     dialect = self._get_db_dialect(db_service_name)
                     source_table_name = self._clean_table_name(sql_table_name, dialect)
                     self._parsed_views[view.name] = source_table_name
@@ -832,7 +835,7 @@ class LookerSource(DashboardServiceSource):
                     # View to the source is only there if we are informing the dbServiceNames
                     yield self.build_lineage_request(
                         source=source_table_name,
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         to_entity=self._view_data_model,
                         column_lineage=column_lineage,
                     )
@@ -870,7 +873,8 @@ class LookerSource(DashboardServiceSource):
         """
         Parse the SQL query and build lineage for the view.
         """
-        for db_service_name in self.get_db_service_names() or []:
+        for db_service_prefix in self.get_db_service_prefixes() or []:
+            db_service_name, *_ = self.parse_db_service_prefix(db_service_prefix)
             lineage_parser = LineageParser(
                 f"create view {view_name} as {sql_query}",
                 self._get_db_dialect(db_service_name),
@@ -889,17 +893,21 @@ class LookerSource(DashboardServiceSource):
                         ):
                             column_lineage.append(
                                 (
-                                    column_tuple[0].raw_name
-                                    if hasattr(column_tuple[0], "raw_name")
-                                    else column_tuple[0],
-                                    column_tuple[-1].raw_name
-                                    if hasattr(column_tuple[-1], "raw_name")
-                                    else column_tuple[-1],
+                                    (
+                                        column_tuple[0].raw_name
+                                        if hasattr(column_tuple[0], "raw_name")
+                                        else column_tuple[0]
+                                    ),
+                                    (
+                                        column_tuple[-1].raw_name
+                                        if hasattr(column_tuple[-1], "raw_name")
+                                        else column_tuple[-1]
+                                    ),
                                 )
                             )
                     yield self.build_lineage_request(
                         source=str(from_table_name),
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         to_entity=self._view_data_model,
                         column_lineage=column_lineage if column_lineage else None,
                     )
@@ -976,9 +984,11 @@ class LookerSource(DashboardServiceSource):
         dashboard_request = CreateDashboardRequest(
             name=EntityName(clean_dashboard_name(dashboard_details.id)),
             displayName=dashboard_details.title,
-            description=Markdown(dashboard_details.description)
-            if dashboard_details.description
-            else None,
+            description=(
+                Markdown(dashboard_details.description)
+                if dashboard_details.description
+                else None
+            ),
             charts=[
                 FullyQualifiedEntityName(
                     fqn.build(
@@ -1103,7 +1113,7 @@ class LookerSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: LookerDashboard,
-        _: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between charts and data sources.
@@ -1207,7 +1217,7 @@ class LookerSource(DashboardServiceSource):
     def build_lineage_request(
         self,
         source: str,
-        db_service_name: str,
+        db_service_prefix: str,
         to_entity: Union[Dashboard, DashboardDataModel],
         column_lineage: Optional[List[Tuple[Column, Column]]] = None,
     ) -> Optional[Either[AddLineageRequest]]:
@@ -1219,21 +1229,48 @@ class LookerSource(DashboardServiceSource):
 
         Args:
             source: table name from the source list
-            db_service_name: name of the service from the config
+            db_service_prefix: db service prefix from the config
             to_entity: Dashboard Entity being used
         """
         logger.debug(f"Building lineage request for {source} to {to_entity.name}")
 
         source_elements = fqn.split_table_name(table_name=source)
 
+        (
+            db_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         for database_name in [source_elements["database"], None]:
+            if (
+                (
+                    prefix_database_name
+                    and database_name
+                    and prefix_database_name.lower() != database_name.lower()
+                )
+                or (
+                    prefix_schema_name
+                    and source_elements["database_schema"]
+                    and prefix_schema_name.lower()
+                    != source_elements["database_schema"].lower()
+                )
+                or (
+                    prefix_table_name
+                    and source_elements["table"]
+                    and prefix_table_name.lower() != source_elements["table"].lower()
+                )
+            ):
+                continue
+
             from_fqn = fqn.build(
                 self.metadata,
                 entity_type=Table,
                 service_name=db_service_name,
-                database_name=database_name,
-                schema_name=source_elements["database_schema"],
-                table_name=source_elements["table"],
+                database_name=prefix_database_name or database_name,
+                schema_name=prefix_schema_name or source_elements["database_schema"],
+                table_name=prefix_table_name or source_elements["table"],
             )
 
             from_entity: Table = self.metadata.get_by_name(

metadata/ingestion/source/dashboard/metabase/metadata.py

@@ -210,9 +210,11 @@ class MetabaseSource(DashboardServiceSource):
                 name=EntityName(str(dashboard_details.id)),
                 sourceUrl=SourceUrl(dashboard_url),
                 displayName=dashboard_details.name,
-                description=Markdown(dashboard_details.description)
-                if dashboard_details.description
-                else None,
+                description=(
+                    Markdown(dashboard_details.description)
+                    if dashboard_details.description
+                    else None
+                ),
                 project=self.context.get().project_name,
                 charts=[
                     FullyQualifiedEntityName(
@@ -286,7 +288,7 @@ class MetabaseSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: MetabaseDashboardDetails,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """Get lineage method
 
@@ -308,7 +310,7 @@ class MetabaseSource(DashboardServiceSource):
                 if chart_details.dataset_query.type == "native":
                     yield from self._yield_lineage_from_query(
                         chart_details=chart_details,
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         dashboard_name=dashboard_name,
                     ) or []
 
@@ -319,7 +321,7 @@ class MetabaseSource(DashboardServiceSource):
                         continue
                     yield from self._yield_lineage_from_api(
                         chart_details=chart_details,
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         dashboard_name=dashboard_name,
                     ) or []
 
@@ -341,11 +343,18 @@ class MetabaseSource(DashboardServiceSource):
     def _yield_lineage_from_query(
         self,
         chart_details: MetabaseChart,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
         dashboard_name: str,
     ) -> Iterable[Either[AddLineageRequest]]:
         database = self.client.get_database(chart_details.database_id)
 
+        (
+            prefix_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         query = None
         if (
             chart_details.dataset_query
@@ -359,23 +368,54 @@ class MetabaseSource(DashboardServiceSource):
 
         database_name = database.details.db if database and database.details else None
 
-        db_service = self._get_database_service(db_service_name)
+        db_service = self._get_database_service(prefix_service_name)
 
         lineage_parser = LineageParser(
             query,
-            ConnectionTypeDialectMapper.dialect_of(db_service.serviceType.value)
-            if db_service
-            else Dialect.ANSI,
+            (
+                ConnectionTypeDialectMapper.dialect_of(db_service.serviceType.value)
+                if db_service
+                else Dialect.ANSI
+            ),
         )
 
+        if (
+            prefix_database_name
+            and database_name
+            and prefix_database_name.lower() != database_name.lower()
+        ):
+            logger.debug(
+                f"Database {database_name} does not match prefix {prefix_database_name}"
+            )
+            return
+
         for table in lineage_parser.source_tables:
             database_schema_name, table = fqn.split(str(table))[-2:]
             database_schema_name = self.check_database_schema_name(database_schema_name)
+
+            if (
+                prefix_table_name
+                and table
+                and prefix_table_name.lower() != table.lower()
+            ):
+                logger.debug(f"Table {table} does not match prefix {prefix_table_name}")
+                continue
+
+            if (
+                prefix_schema_name
+                and database_schema_name
+                and prefix_schema_name.lower() != database_schema_name.lower()
+            ):
+                logger.debug(
+                    f"Schema {database_schema_name} does not match prefix {prefix_schema_name}"
+                )
+                continue
+
             fqn_search_string = build_es_fqn_search_string(
-                database_name=database_name,
-                schema_name=database_schema_name,
-                service_name=db_service_name or "*",
-                table_name=table,
+                database_name=prefix_database_name or database_name,
+                schema_name=prefix_schema_name or database_schema_name,
+                service_name=prefix_service_name or "*",
+                table_name=prefix_table_name or table,
             )
             from_entities = self.metadata.search_in_any_service(
                 entity_type=Table,
@@ -401,7 +441,7 @@ class MetabaseSource(DashboardServiceSource):
     def _yield_lineage_from_api(
         self,
         chart_details: MetabaseChart,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
         dashboard_name: str,
     ) -> Iterable[Either[AddLineageRequest]]:
         table = self.client.get_table(chart_details.table_id)
@@ -410,12 +450,50 @@ class MetabaseSource(DashboardServiceSource):
         if table is None or table_name is None:
             return
 
+        (
+            prefix_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         database_name = table.db.details.db if table.db and table.db.details else None
+
+        if (
+            prefix_table_name
+            and table_name
+            and prefix_table_name.lower() != table_name.lower()
+        ):
+            logger.debug(
+                f"Table {table_name} does not match prefix {prefix_table_name}"
+            )
+            return
+
+        if (
+            prefix_schema_name
+            and table.table_schema
+            and prefix_schema_name.lower() != table.table_schema.lower()
+        ):
+            logger.debug(
+                f"Schema {table.table_schema} does not match prefix {prefix_schema_name}"
+            )
+            return
+
+        if (
+            prefix_database_name
+            and database_name
+            and prefix_database_name.lower() != database_name.lower()
+        ):
+            logger.debug(
+                f"Database {database_name} does not match prefix {prefix_database_name}"
+            )
+            return
+
         fqn_search_string = build_es_fqn_search_string(
-            database_name=database_name,
-            schema_name=table.table_schema,
-            service_name=db_service_name or "*",
-            table_name=table_name,
+            service_name=prefix_service_name or "*",
+            database_name=prefix_database_name or database_name,
+            schema_name=prefix_schema_name or table.table_schema,
+            table_name=prefix_table_name or table_name,
         )
         from_entities = self.metadata.search_in_any_service(
             entity_type=Table,