openmetadata-ingestion 1.6.7.0__py3-none-any.whl → 1.7.0.0rc1__py3-none-any.whl

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -95,6 +95,30 @@ class PowerBiColumns(BaseModel):
     name: str
     dataType: Optional[str] = None
     columnType: Optional[str] = None
+    description: Optional[str] = None
+
+
+class PowerBiMeasureModel(BaseModel):
+    """
+    Represents a Power BI measure, used before converting to a Column instance.
+    """
+
+    dataType: str
+    dataTypeDisplay: str
+    name: str
+    description: str
+
+
+class PowerBiMeasures(BaseModel):
+    """
+    PowerBI Column Model
+    Definition: https://learn.microsoft.com/en-us/rest/api/power-bi/push-datasets/datasets-get-tables-in-group#measure
+    """
+
+    name: str
+    expression: str
+    description: Optional[str] = None
+    isHidden: bool
 
 
 class PowerBITableSource(BaseModel):
@@ -113,6 +137,7 @@ class PowerBiTable(BaseModel):
 
     name: str
     columns: Optional[List[PowerBiColumns]] = None
+    measures: Optional[List[PowerBiMeasures]] = None
     description: Optional[str] = None
     source: Optional[List[PowerBITableSource]] = None
 

metadata/ingestion/source/dashboard/qlikcloud/client.py

@@ -29,6 +29,8 @@ from metadata.ingestion.source.dashboard.qlikcloud.constants import (
 from metadata.ingestion.source.dashboard.qlikcloud.models import (
     QlikApp,
     QlikAppResponse,
+    QlikSpace,
+    QlikSpaceResponse,
 )
 from metadata.ingestion.source.dashboard.qliksense.models import (
     QlikDataModelResult,
@@ -105,7 +107,7 @@ class QlikCloudClient:
 
     def get_dashboard_charts(self, dashboard_id: str) -> List[QlikSheet]:
         """
-        Get dahsboard chart list
+        Get dashboard chart list
         """
         try:
             self.connect_websocket(dashboard_id)
@@ -164,7 +166,7 @@ class QlikCloudClient:
 
     def get_dashboard_models(self) -> List[QlikTable]:
         """
-        Get dahsboard data models
+        Get dashboard data models
         """
         try:
             self._websocket_send_request(APP_LOADMODEL_REQ)
@@ -181,3 +183,24 @@ class QlikCloudClient:
             logger.debug(traceback.format_exc())
             logger.warning("Failed to fetch the dashboard datamodels")
         return []
+
+    def get_projects_list(self) -> Iterable[QlikSpace]:
+        """
+        Get list of all spaces
+        """
+        try:
+            link = f"/v1/spaces?limit={API_LIMIT}"
+            while True:
+                resp_spaces = self.client.get(link)
+                if resp_spaces:
+                    resp = QlikSpaceResponse(**resp_spaces)
+                    yield from resp.spaces
+                    if resp.links and resp.links.next and resp.links.next.href:
+                        link = resp.links.next.href.replace(
+                            f"{self.config.hostPort}{API_VERSION}", ""
+                        )
+                    else:
+                        break
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the space list")

metadata/ingestion/source/dashboard/qlikcloud/metadata.py

@@ -11,7 +11,7 @@
 """QlikCloud source module"""
 
 import traceback
-from typing import Iterable, List, Optional
+from typing import Dict, Iterable, List, Optional
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
@@ -42,11 +42,16 @@ from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.qlikcloud.client import QlikCloudClient
-from metadata.ingestion.source.dashboard.qlikcloud.models import QlikApp
+from metadata.ingestion.source.dashboard.qlikcloud.models import (
+    QlikApp,
+    QlikSpace,
+    QlikSpaceType,
+)
 from metadata.ingestion.source.dashboard.qliksense.metadata import QliksenseSource
 from metadata.ingestion.source.dashboard.qliksense.models import QlikTable
 from metadata.utils import fqn
-from metadata.utils.filters import filter_by_chart
+from metadata.utils.filters import filter_by_chart, filter_by_project
+from metadata.utils.fqn import build_es_fqn_search_string
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 
@@ -80,9 +85,41 @@ class QlikcloudSource(QliksenseSource):
         metadata: OpenMetadata,
     ):
         super().__init__(config, metadata)
+        self.projects_map: Dict[str, QlikSpace] = {}
         self.collections: List[QlikApp] = []
         self.data_models: List[QlikTable] = []
 
+    def prepare(self):
+        """
+        Get all spaces/projects from QlikCloud to filter out dashboards.
+        """
+        spaces = self.client.get_projects_list()
+        for space in spaces:
+            self.projects_map[space.id] = space
+        self.projects_map[""] = QlikSpace(
+            name="Personal",
+            description="Represents personal space of QlikCloud.",
+            id="",  # dashboards under personal space have spaceId=""
+            type=QlikSpaceType.PERSONAL,
+        )
+
+        return super().prepare()
+
+    def filter_projects_by_type(self, project: QlikSpace) -> bool:
+        """
+        Filter space based on space types configured in connection config.
+        """
+        spaceTypes = self.service_connection.spaceTypes
+        if spaceTypes is None:
+            return False
+        return project.type.value not in [space_type.value for space_type in spaceTypes]
+
+    def is_personal_project(self, project: QlikSpace) -> bool:
+        """
+        Check if space is a personal space.
+        """
+        return project.type == QlikSpaceType.PERSONAL
+
     def filter_draft_dashboard(self, dashboard: QlikApp) -> bool:
         # When only published(non-draft) dashboards are allowed, filter dashboard based on "published" flag from QlikApp
         return (not self.source_config.includeDraftDashboard) and (
@@ -97,6 +134,15 @@ class QlikcloudSource(QliksenseSource):
             if self.filter_draft_dashboard(dashboard):
                 # Skip unpublished dashboards
                 continue
+            project = self.projects_map[dashboard.space_id]
+            if self.filter_projects_by_type(project):
+                # Skip dashboard based on space type filter
+                continue
+            if not self.is_personal_project(project) and filter_by_project(
+                self.service_connection.projectFilterPattern, project.name
+            ):
+                # Skip dashboard based on project filter pattern
+                continue
             # clean data models for next iteration
             self.data_models = []
             yield dashboard
@@ -126,9 +172,11 @@ class QlikcloudSource(QliksenseSource):
                 name=EntityName(dashboard_details.id),
                 sourceUrl=SourceUrl(dashboard_url),
                 displayName=dashboard_details.name,
-                description=Markdown(dashboard_details.description)
-                if dashboard_details.description
-                else None,
+                description=(
+                    Markdown(dashboard_details.description)
+                    if dashboard_details.description
+                    else None
+                ),
                 project=self.context.get().project_name,
                 charts=[
                     FullyQualifiedEntityName(
@@ -188,18 +236,22 @@ class QlikcloudSource(QliksenseSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: QlikApp,
-        db_service_name: Optional[str],
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """Get lineage method"""
-        db_service_entity = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=db_service_name
-        )
         for datamodel in self.data_models or []:
             try:
                 data_model_entity = self._get_datamodel(datamodel_id=datamodel.id)
                 if data_model_entity:
-                    om_table = self._get_database_table(
-                        db_service_entity, data_model_entity
+                    fqn_search_string = build_es_fqn_search_string(
+                        database_name=None,
+                        schema_name=None,
+                        service_name=db_service_name or "*",
+                        table_name=data_model_entity.displayName,
+                    )
+                    om_table = self.metadata.search_in_any_service(
+                        entity_type=Table,
+                        fqn_search_string=fqn_search_string,
                     )
                     if om_table:
                         columns_list = [col.name for col in datamodel.fields]
@@ -243,9 +295,11 @@ class QlikcloudSource(QliksenseSource):
                     right=CreateChartRequest(
                         name=EntityName(chart.qInfo.qId),
                         displayName=chart.qMeta.title,
-                        description=Markdown(chart.qMeta.description)
-                        if chart.qMeta.description
-                        else None,
+                        description=(
+                            Markdown(chart.qMeta.description)
+                            if chart.qMeta.description
+                            else None
+                        ),
                         chartType=ChartType.Other,
                         sourceUrl=SourceUrl(chart_url),
                         service=self.context.get().dashboard_service,

metadata/ingestion/source/dashboard/qlikcloud/models.py

@@ -11,9 +11,44 @@
 """
 QlikCloud Models
 """
+from enum import Enum
 from typing import List, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
+
+
+class QlikSpaceType(Enum):
+    MANAGED = "Managed"
+    SHARED = "Shared"
+    PERSONAL = "Personal"
+
+
+# Space Models
+class QlikSpace(BaseModel):
+    """QlikCloud Space Model"""
+
+    name: Optional[str] = None
+    description: Optional[str] = None
+    id: str
+    type: QlikSpaceType
+
+    # Field validator for normalizing and validating space type
+    @field_validator("type", mode="before")
+    @classmethod
+    def normalize_and_validate_type(cls, value):
+        """
+        Normalize the space type by capitalizing the input value and
+        ensure it corresponds to a valid QlikSpaceType enum.
+
+        Args:
+            value (str): The space type to validate.
+
+        Returns:
+            QlikSpaceType: The corresponding enum member of QlikSpaceType.
+        """
+        if isinstance(value, str):
+            value = value.capitalize()
+        return QlikSpaceType(value)
 
 
 # App Models
@@ -24,6 +59,7 @@ class QlikApp(BaseModel):
     name: Optional[str] = None
     id: str
     app_id: Optional[str] = Field(None, alias="resourceId")
+    space_id: Optional[str] = Field("", alias="spaceId")
     published: Optional[bool] = None
 
 
@@ -35,6 +71,13 @@ class QlikLinks(BaseModel):
     next: Optional[QlikLink] = None
 
 
+class QlikSpaceResponse(BaseModel):
+    """QlikCloud Spaces List"""
+
+    spaces: Optional[List[QlikSpace]] = Field(None, alias="data")
+    links: Optional[QlikLinks] = None
+
+
 class QlikAppResponse(BaseModel):
     """QlikCloud Apps List"""
 

metadata/ingestion/source/dashboard/qliksense/metadata.py

@@ -55,6 +55,7 @@ from metadata.ingestion.source.dashboard.qliksense.models import (
 )
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_chart, filter_by_datamodel
+from metadata.utils.fqn import build_es_fqn_search_string
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 
@@ -276,7 +277,11 @@ class QliksenseSource(DashboardServiceSource):
         return None
 
     def _get_database_table(
-        self, db_service_entity: DatabaseService, datamodel: QlikTable
+        self,
+        db_service_entity: DatabaseService,
+        datamodel: QlikTable,
+        schema_name: Optional[str],
+        database_name: Optional[str],
     ) -> Optional[Table]:
         """
         Get the table entity for lineage
@@ -284,17 +289,6 @@ class QliksenseSource(DashboardServiceSource):
         # table.name in tableau can come as db.schema.table_name. Hence the logic to split it
         if datamodel.tableName and db_service_entity:
             try:
-                if len(datamodel.connectorProperties.tableQualifiers) > 1:
-                    (
-                        database_name,
-                        schema_name,
-                    ) = datamodel.connectorProperties.tableQualifiers[-2:]
-                elif len(datamodel.connectorProperties.tableQualifiers) == 1:
-                    schema_name = datamodel.connectorProperties.tableQualifiers[-1]
-                    database_name = None
-                else:
-                    schema_name, database_name = None, None
-
                 table_fqn = fqn.build(
                     self.metadata,
                     entity_type=Table,
@@ -316,18 +310,32 @@ class QliksenseSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: QlikDashboard,
-        db_service_name: Optional[str],
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """Get lineage method"""
-        db_service_entity = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=db_service_name
-        )
         for datamodel in self.data_models or []:
             try:
                 data_model_entity = self._get_datamodel(datamodel_id=datamodel.id)
                 if data_model_entity:
-                    om_table = self._get_database_table(
-                        db_service_entity, datamodel=datamodel
+                    if len(datamodel.connectorProperties.tableQualifiers) > 1:
+                        (
+                            database_name,
+                            schema_name,
+                        ) = datamodel.connectorProperties.tableQualifiers[-2:]
+                    elif len(datamodel.connectorProperties.tableQualifiers) == 1:
+                        schema_name = datamodel.connectorProperties.tableQualifiers[-1]
+                        database_name = None
+                    else:
+                        schema_name, database_name = None, None
+                    fqn_search_string = build_es_fqn_search_string(
+                        database_name=database_name,
+                        schema_name=schema_name,
+                        service_name=db_service_name or "*",
+                        table_name=datamodel.tableName,
+                    )
+                    om_table = self.metadata.search_in_any_service(
+                        entity_type=Table,
+                        fqn_search_string=fqn_search_string,
                     )
                     if om_table:
                         columns_list = [col.name for col in datamodel.fields]

metadata/ingestion/source/dashboard/quicksight/metadata.py

@@ -49,9 +49,8 @@ from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
-from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
+from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
 from metadata.ingestion.lineage.parser import LineageParser
-from metadata.ingestion.lineage.sql_lineage import search_table_entities
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import (
     LINEAGE_MAP,
@@ -69,6 +68,7 @@ from metadata.ingestion.source.dashboard.quicksight.models import (
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_chart
+from metadata.utils.fqn import build_es_fqn_search_string
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -254,12 +254,14 @@ class QuicksightSource(DashboardServiceSource):
         data_model_entity,
         data_source_resp: DataSourceModel,
         dashboard_details: DashboardDetail,
-        db_service_entity,
+        db_service_name: Optional[str],
     ) -> Iterable[Either[AddLineageRequest]]:
         """yield lineage from table(parsed form query source) <-> dashboard"""
-        if not db_service_entity:
-            logger.debug(f"db service is not ingested")
-            return None
+        db_service_entity = None
+        if db_service_name:
+            db_service_entity = self.metadata.get_by_name(
+                entity=DatabaseService, fqn=db_service_name
+            )
         sql_query = data_source_resp.data_source_resp.query
         source_database_names = []
         try:
@@ -278,7 +280,7 @@ class QuicksightSource(DashboardServiceSource):
                     db_service_entity.serviceType.value
                 )
                 if db_service_entity
-                else None,
+                else Dialect.ANSI,
             )
             lineage_details = LineageDetails(
                 source=LineageSource.DashboardLineage, sqlQuery=sql_query
@@ -289,14 +291,18 @@ class QuicksightSource(DashboardServiceSource):
                     database_schema_name = self.check_database_schema_name(
                         database_schema_name
                     )
-                    from_entities = search_table_entities(
-                        metadata=self.metadata,
-                        database=db_name,
-                        service_name=db_service_entity.name.root,
-                        database_schema=database_schema_name,
-                        table=table,
+                    fqn_search_string = build_es_fqn_search_string(
+                        database_name=db_name,
+                        schema_name=database_schema_name,
+                        service_name=db_service_name or "*",
+                        table_name=table,
                     )
-                    for from_entity in from_entities:
+                    from_entities = self.metadata.search_in_any_service(
+                        entity_type=Table,
+                        fqn_search_string=fqn_search_string,
+                        fetch_multiple_entities=True,
+                    )
+                    for from_entity in from_entities or []:
                         if from_entity is not None and data_model_entity is not None:
                             columns = [
                                 col.name.root for col in data_model_entity.columns
@@ -354,7 +360,7 @@ class QuicksightSource(DashboardServiceSource):
                     containers = self.metadata.es_search_container_by_path(
                         full_path=f"s3://{bucket_name}/{key_name}"
                     )
-                    for container in containers:
+                    for container in containers or []:
                         if container is not None and data_model_entity is not None:
                             storage_entity = EntityReference(
                                 id=Uuid(container.id.root),
@@ -385,7 +391,7 @@ class QuicksightSource(DashboardServiceSource):
         data_model_entity,
         data_source_resp: DataSourceModel,
         dashboard_details: DashboardDetail,
-        db_service_entity,
+        db_service_name: Optional[str],
     ) -> Iterable[Either[AddLineageRequest]]:
         """yield lineage from table <-> dashboard"""
         try:
@@ -394,17 +400,15 @@ class QuicksightSource(DashboardServiceSource):
             if data_source_resp and data_source_resp.DataSourceParameters:
                 data_source_dict = data_source_resp.DataSourceParameters
                 for db in data_source_dict.keys() or []:
-                    from_fqn = fqn.build(
-                        self.metadata,
-                        entity_type=Table,
-                        service_name=db_service_entity.name.root,
+                    fqn_search_string = build_es_fqn_search_string(
                         database_name=data_source_dict[db].get("Database"),
                         schema_name=schema_name,
+                        service_name=db_service_name or "*",
                         table_name=table_name,
                     )
-                    from_entity = self.metadata.get_by_name(
-                        entity=Table,
-                        fqn=from_fqn,
+                    from_entity = self.metadata.search_in_any_service(
+                        entity_type=Table,
+                        fqn_search_string=fqn_search_string,
                     )
                     if from_entity is not None and data_model_entity is not None:
                         columns = [col.name.root for col in data_model_entity.columns]
@@ -440,14 +444,13 @@ class QuicksightSource(DashboardServiceSource):
         return None
 
     def yield_dashboard_lineage_details(  # pylint: disable=too-many-locals
-        self, dashboard_details: DashboardDetail, db_service_name: str
+        self,
+        dashboard_details: DashboardDetail,
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
         """
-        db_service_entity = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=db_service_name
-        )
         for datamodel in self.data_models or []:
             try:
                 data_model_entity = self._get_datamodel(
@@ -460,7 +463,7 @@ class QuicksightSource(DashboardServiceSource):
                         data_model_entity,
                         datamodel.DataSource,
                         dashboard_details,
-                        db_service_entity,
+                        db_service_name,
                     )
                 elif isinstance(
                     datamodel.DataSource.data_source_resp, DataSourceRespS3
@@ -473,7 +476,7 @@ class QuicksightSource(DashboardServiceSource):
                         data_model_entity,
                         datamodel.DataSource,
                         dashboard_details,
-                        db_service_entity,
+                        db_service_name,
                     )
             except Exception as exc:  # pylint: disable=broad-except
                 yield Either(

metadata/ingestion/source/dashboard/redash/metadata.py

@@ -48,6 +48,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_chart
+from metadata.utils.fqn import build_es_fqn_search_string
 from metadata.utils.helpers import clean_uri, get_standard_chart_type
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_labels
@@ -197,14 +198,15 @@ class RedashSource(DashboardServiceSource):
             )
 
     def yield_dashboard_lineage_details(  # pylint: disable=too-many-locals
-        self, dashboard_details: dict, db_service_name: str
+        self,
+        dashboard_details: dict,
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
         In redash we do not get table, database_schema or database name but we do get query
         the lineage is being generated based on the query
         """
-
         to_fqn = fqn.build(
             self.metadata,
             entity_type=LineageDashboard,
@@ -229,17 +231,17 @@ class RedashSource(DashboardServiceSource):
                         database_schema_name = self.check_database_schema_name(
                             database_schema
                         )
-                        from_fqn = fqn.build(
-                            self.metadata,
-                            entity_type=Table,
-                            service_name=db_service_name,
+                        if not database_schema_table.get("table"):
+                            continue
+                        fqn_search_string = build_es_fqn_search_string(
+                            database_name=database_schema_table.get("database"),
                             schema_name=database_schema_name,
+                            service_name=db_service_name or "*",
                             table_name=database_schema_table.get("table"),
-                            database_name=database_schema_table.get("database"),
                         )
-                        from_entity = self.metadata.get_by_name(
-                            entity=Table,
-                            fqn=from_fqn,
+                        from_entity = self.metadata.search_in_any_service(
+                            entity_type=Table,
+                            fqn_search_string=fqn_search_string,
                         )
                         if from_entity and to_entity:
                             yield self._get_add_lineage_request(

metadata/ingestion/source/dashboard/sigma/metadata.py

@@ -52,6 +52,7 @@ from metadata.ingestion.source.dashboard.sigma.models import (
 )
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_chart
+from metadata.utils.fqn import build_es_fqn_search_string
 from metadata.utils.helpers import get_standard_chart_type
 from metadata.utils.logger import ingestion_logger
 
@@ -196,39 +197,36 @@ class SigmaSource(DashboardServiceSource):
         return None
 
     def _get_table_entity_from_node(
-        self, node: NodeDetails, db_service_name: str
+        self, node: NodeDetails, db_service_name: Optional[str]
     ) -> Optional[Table]:
         """
         Get the table entity for lineage
         """
         if node.node_schema:
             try:
-                table_fqn = fqn.build(
-                    self.metadata,
-                    entity_type=Table,
-                    service_name=db_service_name,
+                fqn_search_string = build_es_fqn_search_string(
+                    database_name=None,
                     schema_name=node.node_schema,
+                    service_name=db_service_name or "*",
                     table_name=node.name,
-                    database_name="",
                 )
-                if table_fqn:
-                    return self.metadata.get_by_name(
-                        entity=Table,
-                        fqn=table_fqn,
-                    )
+                return self.metadata.search_in_any_service(
+                    entity_type=Table,
+                    fqn_search_string=fqn_search_string,
+                )
             except Exception as exc:
                 logger.debug(traceback.format_exc())
                 logger.warning(f"Error occured while finding table fqn: {exc}")
         return None
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: WorkbookDetails, db_service_name: Optional[str]
+        self,
+        dashboard_details: WorkbookDetails,
+        db_service_name: Optional[str] = None,
     ):
         """
         yield dashboard lineage
         """
-        if not db_service_name:
-            return
         # charts and datamodels are same here as we are using charts as metadata for datamodels
         for data_model in self.data_models or []:
             try: