openmetadata-ingestion 1.8.0.0__py3-none-any.whl → 1.8.2.0__py3-none-any.whl

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-09T12:59:02+00:00
 
 from __future__ import annotations
 

metadata/ingestion/lineage/masker.py

@@ -13,6 +13,7 @@ Query masking utilities
 """
 
 import traceback
+from typing import Optional
 
 from cachetools import LRUCache
 from collate_sqllineage.runner import SQLPARSE_DIALECT, LineageRunner
@@ -26,6 +27,7 @@ MASK_TOKEN = "?"
 # Cache size is 128 to avoid memory issues
 masked_query_cache = LRUCache(maxsize=128)
 
+
 # pylint: disable=protected-access
 def get_logger():
     # pylint: disable=import-outside-toplevel
@@ -52,7 +54,6 @@ def mask_literals_with_sqlparse(query: str, parser: LineageRunner):
                 Literal.Number.Integer,
                 Literal.Number.Float,
                 Literal.String.Single,
-                Literal.String.Symbol,
             ):
                 token.value = MASK_TOKEN
             elif token.is_group:
@@ -113,7 +114,9 @@ def mask_literals_with_sqlfluff(query: str, parser: LineageRunner) -> str:
 
 
 def mask_query(
-    query: str, dialect: str = Dialect.ANSI.value, parser: LineageRunner = None
+    query: str,
+    dialect: str = Dialect.ANSI.value,
+    parser: Optional[LineageRunner] = None,
 ) -> str:
     """
     Mask a query using sqlparse or sqlfluff.

metadata/ingestion/lineage/sql_lineage.py

@@ -333,9 +333,11 @@ def get_source_table_names(
     """
     try:
         if not isinstance(source_table, DataFunction):
-            yield EntityReference(
-                id=procedure.id.root, type="storedProcedure"
-            ) if procedure else None, str(source_table)
+            yield (
+                EntityReference(id=procedure.id.root, type="storedProcedure")
+                if procedure
+                else None
+            ), str(source_table)
         else:
             yield from __process_udf_table_names(
                 metadata,
@@ -547,15 +549,19 @@ def _create_lineage_by_table_name(
             # Add nodes and edges with minimal data
             graph.add_node(
                 from_table,
-                fqns=[table.fullyQualifiedName.root for table in from_table_entities]
-                if from_table_entities
-                else [],
+                fqns=(
+                    [table.fullyQualifiedName.root for table in from_table_entities]
+                    if from_table_entities
+                    else []
+                ),
             )
             graph.add_node(
                 to_table,
-                fqns=[table.fullyQualifiedName.root for table in to_table_entities]
-                if to_table_entities
-                else [],
+                fqns=(
+                    [table.fullyQualifiedName.root for table in to_table_entities]
+                    if to_table_entities
+                    else []
+                ),
             )
             graph.add_edge(from_table, to_table)
             return

metadata/ingestion/source/api/rest/metadata.py

@@ -300,38 +300,79 @@ class RestSource(ApiServiceSource):
             logger.warning(f"Error while parsing response schema: {err}")
         return None
 
-    def process_schema_fields(self, schema_ref: str) -> Optional[List[FieldModel]]:
+    def process_schema_fields(
+        self, schema_ref: str, parent_refs: Optional[List[str]] = None
+    ) -> Optional[List[FieldModel]]:
         try:
+            if parent_refs is None:
+                parent_refs = []
             schema_name = schema_ref.split("/")[-1]
             schema_fields = (
                 self.json_response.get("components").get("schemas").get(schema_name)
             )
-
+            parent_refs.append(schema_ref)
             fetched_fields = []
             for key, val in schema_fields.get("properties", {}).items():
                 dtype = val.get("type")
                 if dtype:
+                    dtype = "INT" if dtype.upper() == "INTEGER" else dtype
                     parsed_dtype = (
                         DataTypeTopic[dtype.upper()]
                         if dtype.upper() in DataTypeTopic.__members__
                         else DataTypeTopic.UNKNOWN
                     )
-                    fetched_fields.append(FieldModel(name=key, dataType=parsed_dtype))
+                    children = None
+                    if parsed_dtype.value == DataTypeTopic.ARRAY.value:
+                        # If field of array type then parse children
+                        children_ref = val.get("items", {}).get("$ref")
+                        if children_ref:
+                            # check infinite recursion by checking pre-processed schemas(parent_refs)
+                            if children_ref not in parent_refs:
+                                logger.debug(
+                                    f"Processing array fields inside schema: {children_ref}"
+                                )
+                                children = self.process_schema_fields(
+                                    children_ref, parent_refs
+                                )
+                                logger.debug(
+                                    f"Completed processing array fields inside schema: {children_ref}"
+                                )
+                            else:
+                                logger.debug(
+                                    f"Skipping array fields inside schema: {children_ref} to avoid infinite recursion"
+                                )
+                    fetched_fields.append(
+                        FieldModel(name=key, dataType=parsed_dtype, children=children)
+                    )
                 else:
                     # If type of field is not defined then check for sub-schema
                     # Check if it's `object` type field
-                    # check infinite recrusrion by comparing with parent(schema_ref)
-                    object_children = None
-                    if val.get("$ref") and val.get("$ref") != schema_ref:
-                        object_children = self.process_schema_fields(val.get("$ref"))
+                    children = None
+                    if val.get("$ref"):
+                        # check infinite recursion by checking pre-processed schemas(parent_refs)
+                        if val.get("$ref") not in parent_refs:
+                            children = self.process_schema_fields(
+                                val.get("$ref"), parent_refs
+                            )
+                        else:
+                            logger.debug(
+                                f"Skipping object fields inside schema: {val.get('$ref')} to avoid infinite recursion"
+                            )
                     fetched_fields.append(
                         FieldModel(
                             name=key,
                             dataType=DataTypeTopic.UNKNOWN,
-                            children=object_children,
+                            children=children,
                         )
                     )
+            if parent_refs and (schema_ref in parent_refs):
+                parent_refs.pop()
             return fetched_fields
         except Exception as err:
             logger.warning(f"Error while processing schema fields: {err}")
+            if parent_refs and (schema_ref in parent_refs):
+                parent_refs.pop()
+                logger.debug(
+                    f"Popping {schema_ref} from parent_refs due to processing error"
+                )
         return None

metadata/ingestion/source/dashboard/dashboard_service.py

@@ -13,7 +13,7 @@ Base class for ingesting dashboard services
 """
 import traceback
 from abc import ABC, abstractmethod
-from typing import Any, Iterable, List, Optional, Set, Union
+from typing import Any, Iterable, List, Optional, Set, Tuple, Union
 
 from pydantic import BaseModel, Field
 from typing_extensions import Annotated
@@ -251,7 +251,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: Any,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
@@ -340,16 +340,27 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                         f"Error to yield dashboard lineage details for data model name [{str(datamodel)}]: {err}"
                     )
 
-    def get_db_service_names(self) -> List[str]:
+    def get_db_service_prefixes(self) -> List[str]:
         """
-        Get the list of db service names
+        Get the list of db service prefixes
         """
         return (
-            self.source_config.lineageInformation.dbServiceNames or []
+            self.source_config.lineageInformation.dbServicePrefixes or []
             if self.source_config.lineageInformation
             else []
         )
 
+    def parse_db_service_prefix(
+        self, db_service_prefix: Optional[str]
+    ) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
+        """
+        Parse the db service prefix
+        Returns:
+            Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: service, database, schema, table
+        """
+        prefix_parts = (db_service_prefix or "").split(".")
+        return prefix_parts + ([None] * (4 - len(prefix_parts)))
+
     def yield_dashboard_lineage(
         self, dashboard_details: Any
     ) -> Iterable[Either[OMetaLineageRequest]]:
@@ -364,15 +375,12 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
             yield from self.yield_lineage_request(lineage)
 
         # yield datamodel lineage with tables from db services
-        db_service_names = self.get_db_service_names()
-        if not db_service_names:
+        db_service_prefixes = self.get_db_service_prefixes()
+        for db_service_prefix in db_service_prefixes or [None]:
             for lineage in (
-                self.yield_dashboard_lineage_details(dashboard_details) or []
-            ):
-                yield from self.yield_lineage_request(lineage)
-        for db_service_name in db_service_names or []:
-            for lineage in (
-                self.yield_dashboard_lineage_details(dashboard_details, db_service_name)
+                self.yield_dashboard_lineage_details(
+                    dashboard_details, db_service_prefix
+                )
                 or []
             ):
                 yield from self.yield_lineage_request(lineage)

metadata/ingestion/source/dashboard/domodashboard/metadata.py

@@ -127,9 +127,11 @@ class DomodashboardSource(DashboardServiceSource):
                 name=EntityName(dashboard_details.id),
                 sourceUrl=SourceUrl(dashboard_url),
                 displayName=dashboard_details.name,
-                description=Markdown(dashboard_details.description)
-                if dashboard_details.description
-                else None,
+                description=(
+                    Markdown(dashboard_details.description)
+                    if dashboard_details.description
+                    else None
+                ),
                 charts=[
                     FullyQualifiedEntityName(
                         fqn.build(
@@ -228,9 +230,11 @@ class DomodashboardSource(DashboardServiceSource):
                     yield Either(
                         right=CreateChartRequest(
                             name=EntityName(str(chart_id)),
-                            description=Markdown(chart.description)
-                            if chart.description
-                            else None,
+                            description=(
+                                Markdown(chart.description)
+                                if chart.description
+                                else None
+                            ),
                             displayName=chart.name,
                             sourceUrl=SourceUrl(chart_url),
                             service=self.context.get().dashboard_service,
@@ -250,6 +254,6 @@ class DomodashboardSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: dict,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """No lineage implemented"""

metadata/ingestion/source/dashboard/lightdash/metadata.py

@@ -124,9 +124,11 @@ class LightdashSource(DashboardServiceSource):
                 sourceUrl=SourceUrl(dashboard_url),
                 project=self.client.get_project_name(dashboard_details.projectUuid),
                 displayName=dashboard_details.name,
-                description=Markdown(dashboard_details.description)
-                if dashboard_details.description
-                else None,
+                description=(
+                    Markdown(dashboard_details.description)
+                    if dashboard_details.description
+                    else None
+                ),
                 charts=[
                     FullyQualifiedEntityName(
                         fqn.build(
@@ -178,9 +180,9 @@ class LightdashSource(DashboardServiceSource):
                     right=CreateChartRequest(
                         name=EntityName(chart.uuid),
                         displayName=chart.name,
-                        description=Markdown(chart.description)
-                        if chart.description
-                        else None,
+                        description=(
+                            Markdown(chart.description) if chart.description else None
+                        ),
                         sourceUrl=SourceUrl(chart_url),
                         service=self.context.get().dashboard_service,
                         chartType=chart_type,
@@ -194,7 +196,8 @@ class LightdashSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: LightdashDashboard,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
+        _: Optional[str] = None,
     ) -> Optional[Iterable[AddLineageRequest]]:
         """Get lineage method
 

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -247,7 +247,7 @@ class LookerSource(DashboardServiceSource):
                 BitBucketCredentials,
                 GitlabCredentials,
             ]
-        ]
+        ],
     ) -> "LookMLRepo":
         repo_name = (
             f"{credentials.repositoryOwner.root}/{credentials.repositoryName.root}"
@@ -471,9 +471,9 @@ class LookerSource(DashboardServiceSource):
                 explore_datamodel = CreateDashboardDataModelRequest(
                     name=EntityName(datamodel_name),
                     displayName=model.name,
-                    description=Markdown(model.description)
-                    if model.description
-                    else None,
+                    description=(
+                        Markdown(model.description) if model.description else None
+                    ),
                     service=self.context.get().dashboard_service,
                     tags=get_tag_labels(
                         metadata=self.metadata,
@@ -586,9 +586,9 @@ class LookerSource(DashboardServiceSource):
                 data_model_request = CreateDashboardDataModelRequest(
                     name=EntityName(datamodel_view_name),
                     displayName=view.name,
-                    description=Markdown(view.description)
-                    if view.description
-                    else None,
+                    description=(
+                        Markdown(view.description) if view.description else None
+                    ),
                     service=self.context.get().dashboard_service,
                     tags=get_tag_labels(
                         metadata=self.metadata,
@@ -816,12 +816,15 @@ class LookerSource(DashboardServiceSource):
                     " while processing view lineage."
                 )
 
-            db_service_names = self.get_db_service_names()
+            db_service_prefixes = self.get_db_service_prefixes()
 
             if view.sql_table_name:
                 sql_table_name = self._render_table_name(view.sql_table_name)
 
-                for db_service_name in db_service_names or []:
+                for db_service_prefix in db_service_prefixes or []:
+                    db_service_name, *_ = self.parse_db_service_prefix(
+                        db_service_prefix
+                    )
                     dialect = self._get_db_dialect(db_service_name)
                     source_table_name = self._clean_table_name(sql_table_name, dialect)
                     self._parsed_views[view.name] = source_table_name
@@ -832,7 +835,7 @@ class LookerSource(DashboardServiceSource):
                     # View to the source is only there if we are informing the dbServiceNames
                     yield self.build_lineage_request(
                         source=source_table_name,
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         to_entity=self._view_data_model,
                         column_lineage=column_lineage,
                     )
@@ -870,7 +873,8 @@ class LookerSource(DashboardServiceSource):
         """
         Parse the SQL query and build lineage for the view.
         """
-        for db_service_name in self.get_db_service_names() or []:
+        for db_service_prefix in self.get_db_service_prefixes() or []:
+            db_service_name, *_ = self.parse_db_service_prefix(db_service_prefix)
             lineage_parser = LineageParser(
                 f"create view {view_name} as {sql_query}",
                 self._get_db_dialect(db_service_name),
@@ -889,17 +893,21 @@ class LookerSource(DashboardServiceSource):
                         ):
                             column_lineage.append(
                                 (
-                                    column_tuple[0].raw_name
-                                    if hasattr(column_tuple[0], "raw_name")
-                                    else column_tuple[0],
-                                    column_tuple[-1].raw_name
-                                    if hasattr(column_tuple[-1], "raw_name")
-                                    else column_tuple[-1],
+                                    (
+                                        column_tuple[0].raw_name
+                                        if hasattr(column_tuple[0], "raw_name")
+                                        else column_tuple[0]
+                                    ),
+                                    (
+                                        column_tuple[-1].raw_name
+                                        if hasattr(column_tuple[-1], "raw_name")
+                                        else column_tuple[-1]
+                                    ),
                                 )
                             )
                     yield self.build_lineage_request(
                         source=str(from_table_name),
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         to_entity=self._view_data_model,
                         column_lineage=column_lineage if column_lineage else None,
                     )
@@ -976,9 +984,11 @@ class LookerSource(DashboardServiceSource):
         dashboard_request = CreateDashboardRequest(
             name=EntityName(clean_dashboard_name(dashboard_details.id)),
             displayName=dashboard_details.title,
-            description=Markdown(dashboard_details.description)
-            if dashboard_details.description
-            else None,
+            description=(
+                Markdown(dashboard_details.description)
+                if dashboard_details.description
+                else None
+            ),
             charts=[
                 FullyQualifiedEntityName(
                     fqn.build(
@@ -1103,7 +1113,7 @@ class LookerSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: LookerDashboard,
-        _: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between charts and data sources.
@@ -1207,7 +1217,7 @@ class LookerSource(DashboardServiceSource):
     def build_lineage_request(
         self,
         source: str,
-        db_service_name: str,
+        db_service_prefix: str,
         to_entity: Union[Dashboard, DashboardDataModel],
         column_lineage: Optional[List[Tuple[Column, Column]]] = None,
     ) -> Optional[Either[AddLineageRequest]]:
@@ -1219,21 +1229,48 @@ class LookerSource(DashboardServiceSource):
 
         Args:
             source: table name from the source list
-            db_service_name: name of the service from the config
+            db_service_prefix: db service prefix from the config
             to_entity: Dashboard Entity being used
         """
         logger.debug(f"Building lineage request for {source} to {to_entity.name}")
 
         source_elements = fqn.split_table_name(table_name=source)
 
+        (
+            db_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         for database_name in [source_elements["database"], None]:
+            if (
+                (
+                    prefix_database_name
+                    and database_name
+                    and prefix_database_name.lower() != database_name.lower()
+                )
+                or (
+                    prefix_schema_name
+                    and source_elements["database_schema"]
+                    and prefix_schema_name.lower()
+                    != source_elements["database_schema"].lower()
+                )
+                or (
+                    prefix_table_name
+                    and source_elements["table"]
+                    and prefix_table_name.lower() != source_elements["table"].lower()
+                )
+            ):
+                continue
+
             from_fqn = fqn.build(
                 self.metadata,
                 entity_type=Table,
                 service_name=db_service_name,
-                database_name=database_name,
-                schema_name=source_elements["database_schema"],
-                table_name=source_elements["table"],
+                database_name=prefix_database_name or database_name,
+                schema_name=prefix_schema_name or source_elements["database_schema"],
+                table_name=prefix_table_name or source_elements["table"],
             )
 
             from_entity: Table = self.metadata.get_by_name(