openmetadata-ingestion 1.7.2.0__py3-none-any.whl → 1.7.3.0__py3-none-any.whl

metadata/ingestion/source/dashboard/microstrategy/helpers.py

@@ -0,0 +1,67 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Microstrategy source helpers.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict
+
+from metadata.generated.schema.entity.data.table import Column, DataType
+
+
+class MicroStrategyColumnParser:
+    """
+    Responsible for containing the logic to parse a column from MicroStrategy to OpenMetadata
+    """
+
+    datatype_mapping = {
+        "big decimal": DataType.DECIMAL,
+        "binary": DataType.BYTES,
+        "char": DataType.CHAR,
+        "date": DataType.DATE,
+        "decimal": DataType.DECIMAL,
+        "double": DataType.DOUBLE,
+        "float": DataType.FLOAT,
+        "integer": DataType.INT,
+        "longvarbin": DataType.BLOB,
+        "longvarchar": DataType.CLOB,
+        "nchar": DataType.CHAR,
+        "numeric": DataType.NUMERIC,
+        "nvarchar": DataType.VARCHAR,
+        "real": DataType.STRING,
+        "time": DataType.TIME,
+        "timestamp": DataType.TIMESTAMP,
+        "unsigned": DataType.INT,
+        "varbin": DataType.BYTES,
+        "varchar": DataType.VARCHAR,
+        "utf8char": DataType.CHAR,
+    }
+
+    @classmethod
+    def parse(cls, field: Dict[str, Any]) -> Column:
+        """
+        Parses a MicroStrategy table column into an OpenMetadata column.
+        """
+
+        array_data_type = None
+        data_type = cls.datatype_mapping.get(
+            field["dataType"].lower(), DataType.UNKNOWN
+        )
+
+        column_def = {
+            "name": field["name"],
+            "dataTypeDisplay": field["dataType"],
+            "dataType": data_type,
+            "arrayDataType": array_data_type,
+        }
+
+        return Column(**column_def)

metadata/ingestion/source/dashboard/microstrategy/metadata.py

@@ -14,11 +14,23 @@ from typing import Iterable, List, Optional
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
+from metadata.generated.schema.api.data.createDashboardDataModel import (
+    CreateDashboardDataModelRequest,
+)
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.chart import Chart
+from metadata.generated.schema.entity.data.dashboardDataModel import (
+    DashboardDataModel,
+    DataModelType,
+)
+from metadata.generated.schema.entity.data.table import Column, DataType
 from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
     MicroStrategyConnection,
 )
+from metadata.generated.schema.entity.services.dashboardService import (
+    DashboardServiceType,
+)
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
@@ -29,18 +41,29 @@ from metadata.generated.schema.type.basic import (
     EntityName,
     FullyQualifiedEntityName,
     SourceUrl,
+    Uuid,
 )
+from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
+from metadata.generated.schema.type.entityLineage import Source as LineageSource
+from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
+from metadata.ingestion.lineage.parser import LineageParser
+from metadata.ingestion.lineage.sql_lineage import get_table_entities_from_query
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
+from metadata.ingestion.source.dashboard.microstrategy.helpers import (
+    MicroStrategyColumnParser,
+)
 from metadata.ingestion.source.dashboard.microstrategy.models import (
     MstrDashboard,
     MstrDashboardDetails,
+    MstrDataset,
     MstrPage,
 )
 from metadata.utils import fqn
-from metadata.utils.filters import filter_by_chart
+from metadata.utils.filters import filter_by_chart, filter_by_datamodel
 from metadata.utils.helpers import clean_uri, get_standard_chart_type
 from metadata.utils.logger import ingestion_logger
 
@@ -163,7 +186,75 @@ class MicrostrategySource(DashboardServiceSource):
         dashboard_details: MstrDashboardDetails,
         db_service_name: Optional[str] = None,
     ) -> Optional[Iterable[AddLineageRequest]]:
-        """Not Implemented"""
+        """
+        Get lineage between datamodel and data sources
+        """
+        if not db_service_name:
+            return
+
+        database_service = self.metadata.get_by_name(
+            entity=DatabaseService, fqn=db_service_name
+        )
+        dialect = ConnectionTypeDialectMapper.dialect_of(
+            database_service.serviceType.value
+        )
+
+        for dataset in dashboard_details.datasets:
+            cube_sql = self.client.get_cube_sql_details(
+                dashboard_details.projectId, dataset.id
+            )
+            if not cube_sql:
+                continue
+
+            datamodel_fqn = fqn.build(
+                self.metadata,
+                entity_type=DashboardDataModel,
+                service_name=self.context.get().dashboard_service,
+                data_model_name=dataset.id,
+            )
+            datamodel_entity = self.metadata.get_by_name(
+                entity=DashboardDataModel, fqn=datamodel_fqn
+            )
+
+            try:
+                lineage_parser = LineageParser(cube_sql, dialect=dialect)
+                for table in lineage_parser.source_tables:
+                    table_entities = get_table_entities_from_query(
+                        metadata=self.metadata,
+                        service_name=db_service_name,
+                        database_name="*",
+                        database_schema="*",
+                        table_name=str(table),
+                    )
+                    if not table_entities:
+                        logger.debug(f"Table not found in metadata: {str(table)}")
+                        continue
+                    for table_entity in table_entities or []:
+                        yield Either(
+                            right=AddLineageRequest(
+                                edge=EntitiesEdge(
+                                    fromEntity=EntityReference(
+                                        id=Uuid(table_entity.id.root),
+                                        type="table",
+                                    ),
+                                    toEntity=EntityReference(
+                                        id=Uuid(datamodel_entity.id.root),
+                                        type="dashboardDataModel",
+                                    ),
+                                    lineageDetails=LineageDetails(
+                                        source=LineageSource.DashboardLineage
+                                    ),
+                                )
+                            )
+                        )
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name="Dashboard Lineage",
+                        error=f"Error to yield dashboard lineage details: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
 
     def yield_dashboard_chart(
         self, dashboard_details: MstrDashboardDetails
@@ -213,6 +304,74 @@ class MicrostrategySource(DashboardServiceSource):
                     )
                 )
 
+    def _get_column_info(self, dataset: MstrDataset) -> Optional[List[Column]]:
+        """Build columns from dataset"""
+        datasource_columns = []
+        for available_object in dataset.availableObjects or []:
+            try:
+                parsed_column = {
+                    "dataTypeDisplay": available_object.type.title(),
+                    "dataType": DataType.UNKNOWN,
+                    "name": available_object.name,
+                    "displayName": available_object.name,
+                }
+                parsed_column_children = []
+                for form in available_object.forms or []:
+                    parsed_column_children.append(MicroStrategyColumnParser.parse(form))
+                if parsed_column_children:
+                    parsed_column["children"] = parsed_column_children
+
+                datasource_columns.append(Column(**parsed_column))
+            except Exception as exc:
+                logger.debug(traceback.format_exc())
+                logger.warning(f"Error to yield datamodel column: {exc}")
+        return datasource_columns
+
+    def yield_datamodel(
+        self, dashboard_details: MstrDashboardDetails
+    ) -> Optional[Iterable[CreateDashboardDataModelRequest]]:
+        """Get datamodel method
+
+        Args:
+            dashboard_details:
+        Returns:
+            Iterable[CreateDashboardDataModelRequest]
+        """
+        try:
+            if self.source_config.includeDataModels:
+                for dataset in dashboard_details.datasets:
+                    if filter_by_datamodel(
+                        self.source_config.dataModelFilterPattern, dataset.name
+                    ):
+                        self.status.filter(dataset.name, "Data model filtered out.")
+                        continue
+                    data_model_type = DataModelType.MicroStrategyDataset.value
+                    datamodel_columns = self._get_column_info(dataset)
+
+                    data_model_request = CreateDashboardDataModelRequest(
+                        name=EntityName(dataset.id),
+                        displayName=dataset.name,
+                        service=FullyQualifiedEntityName(
+                            self.context.get().dashboard_service
+                        ),
+                        dataModelType=data_model_type,
+                        serviceType=DashboardServiceType.MicroStrategy.value,
+                        columns=datamodel_columns,
+                        project=self.get_project_name(
+                            dashboard_details=dashboard_details
+                        ),
+                    )
+                    yield Either(right=data_model_request)
+                    self.register_record_datamodel(datamodel_request=data_model_request)
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=dataset.name,
+                    error=f"Error yielding Data Model [{dataset.name}]: {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+
     def close(self):
         # close the api session
         self.client.close_api_session()

metadata/ingestion/source/dashboard/microstrategy/models.py

@@ -12,7 +12,7 @@
 MicroStrategy Models
 """
 from datetime import datetime
-from typing import Any, List, Optional
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel
 
@@ -134,6 +134,17 @@ class MstrAvailableObject(BaseModel):
     id: str
     name: str
     type: str
+    forms: Optional[List[Dict[str, Any]]] = None
+
+
+class MstrDataset(BaseModel):
+    id: str
+    name: str
+    availableObjects: Optional[List[MstrAvailableObject]] = None
+    rows: Optional[List[Dict[str, Any]]] = None
+    columns: Optional[List[Dict[str, Any]]] = None
+    pageBy: Optional[List[Dict[str, Any]]] = None
+    sqlStatement: Optional[str] = None
 
 
 class MstrDashboardDetails(BaseModel):
@@ -143,6 +154,7 @@ class MstrDashboardDetails(BaseModel):
     projectName: str
     currentChapter: str
     chapters: List[MstrChapter]
+    datasets: List[MstrDataset]
 
 
 class AuthHeaderCookie(BaseModel):

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -80,6 +80,8 @@ from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
+OWNER_ACCESS_RIGHTS_KEYWORDS = ["owner", "write", "admin"]
+
 
 class PowerbiSource(DashboardServiceSource):
     """PowerBi Source Class"""
@@ -1017,42 +1019,49 @@ class PowerbiSource(DashboardServiceSource):
                 elif isinstance(dashboard_details, PowerBIDashboard):
                     access_right = owner.dashboardUserAccessRight
 
-                if owner.userType != "Member" or (
-                    isinstance(
-                        dashboard_details, (Dataflow, PowerBIReport, PowerBIDashboard)
-                    )
-                    and access_right != "Owner"
-                ):
+                if owner.userType != "Member":
                     logger.warning(
-                        f"User is not a member and has no access to the {dashboard_details.id}: ({owner.displayName}, {owner.email})"
+                        f"User is not a member of {dashboard_details.id}:"
+                        f" ({owner.displayName}, {owner.email})"
                     )
                     continue
-                if owner.email:
-                    try:
-                        owner_email = EmailStr._validate(owner.email)
-                    except PydanticCustomError:
-                        logger.warning(f"Invalid email for owner: {owner.email}")
-                        owner_email = None
-                    if owner_email:
+                if access_right and any(
+                    keyword in access_right.lower()
+                    for keyword in OWNER_ACCESS_RIGHTS_KEYWORDS
+                ):
+                    if owner.email:
+                        try:
+                            owner_email = EmailStr._validate(owner.email)
+                        except PydanticCustomError:
+                            logger.warning(f"Invalid email for owner: {owner.email}")
+                            owner_email = None
+                        if owner_email:
+                            try:
+                                owner_ref = self.metadata.get_reference_by_email(
+                                    owner_email.lower()
+                                )
+                            except Exception as err:
+                                logger.warning(
+                                    f"Could not process owner data with email"
+                                    f" {owner.email} in {dashboard_details.id}: {err}"
+                                )
+                    elif owner.displayName:
                         try:
-                            owner_ref = self.metadata.get_reference_by_email(
-                                owner_email.lower()
+                            owner_ref = self.metadata.get_reference_by_name(
+                                name=owner.displayName
                             )
                         except Exception as err:
                             logger.warning(
-                                f"Could not fetch owner data with email {owner.email} in {dashboard_details.id}: {err}"
+                                f"Could not process owner data with name"
+                                f" {owner.displayName} in {dashboard_details.id}: {err}"
                             )
-                elif owner.displayName:
-                    try:
-                        owner_ref = self.metadata.get_reference_by_name(
-                            name=owner.displayName
-                        )
-                    except Exception as err:
-                        logger.warning(
-                            f"Could not process owner data with name {owner.displayName} in {dashboard_details.id}: {err}"
-                        )
-                if owner_ref:
-                    owner_ref_list.append(owner_ref.root[0])
+                    if owner_ref:
+                        owner_ref_list.append(owner_ref.root[0])
+                else:
+                    logger.warning(
+                        f"User does not have owner, admin or write access to"
+                        f" {dashboard_details.id}: ({owner.displayName}, {owner.email})"
+                    )
             # check for last modified, configuredBy user
             current_active_user = None
             if isinstance(dashboard_details, Dataset):

metadata/ingestion/source/dashboard/qlikcloud/metadata.py

@@ -134,6 +134,12 @@ class QlikcloudSource(QliksenseSource):
             if self.filter_draft_dashboard(dashboard):
                 # Skip unpublished dashboards
                 continue
+            if dashboard.space_id not in self.projects_map:
+                logger.warning(
+                    f"Project ID '{dashboard.space_id}' for Dashboard '{dashboard.name}' is not present"
+                    " in projects map"
+                )
+                continue
             project = self.projects_map[dashboard.space_id]
             if self.filter_projects_by_type(project):
                 # Skip dashboard based on space type filter

metadata/ingestion/source/dashboard/qlikcloud/models.py

@@ -21,6 +21,7 @@ class QlikSpaceType(Enum):
     MANAGED = "Managed"
     SHARED = "Shared"
     PERSONAL = "Personal"
+    DATA = "Data"
 
 
 # Space Models

metadata/ingestion/source/dashboard/tableau/models.py

@@ -13,9 +13,10 @@
 Tableau Source Model module
 """
 
+import uuid
 from typing import Dict, List, Optional, Set, Union
 
-from pydantic import BaseModel, ConfigDict, Field, validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator, validator
 
 from metadata.generated.schema.entity.data.chart import ChartType
 from metadata.generated.schema.entity.data.table import Table
@@ -28,9 +29,18 @@ class TableauBaseModel(BaseModel):
 
     model_config = ConfigDict(extra="allow")
 
-    id: str
+    # in case of personal space workbooks, the project id is returned as a UUID
+    id: Union[str, uuid.UUID]
     name: Optional[str] = None
 
+    # pylint: disable=no-self-argument
+    @field_validator("id", mode="before")
+    def coerce_uuid_to_string(cls, value):
+        """Ensure id is always stored as a string internally"""
+        if isinstance(value, uuid.UUID):
+            return str(value)
+        return value
+
     def __hash__(self):
         return hash(self.id)
 

metadata/ingestion/source/database/unitycatalog/metadata.py

@@ -340,7 +340,7 @@ class UnitycatalogSource(
                         schema_name=schema_name,
                     )
                 ),
-                owners=self.get_owner_ref(table_name),
+                owners=self.get_owner_ref(table.owner),
             )
             yield Either(right=table_request)
 
@@ -403,6 +403,7 @@ class UnitycatalogSource(
             )
             if referred_table_fqn:
                 for parent_column in column.parent_columns:
+                    # pylint: disable=protected-access
                     col_fqn = fqn._build(referred_table_fqn, parent_column, quote=False)
                     if col_fqn:
                         referred_column_fqns.append(FullyQualifiedEntityName(col_fqn))
@@ -419,6 +420,7 @@ class UnitycatalogSource(
 
         return table_constraints
 
+    # pylint: disable=arguments-differ
     def update_table_constraints(
         self, table_constraints, foreign_columns, columns
     ) -> List[TableConstraint]:
@@ -524,18 +526,25 @@ class UnitycatalogSource(
     def close(self):
         """Nothing to close"""
 
-    def get_owner_ref(self, table_name: str) -> Optional[EntityReferenceList]:
+    # pylint: disable=arguments-renamed
+    def get_owner_ref(
+        self, table_owner: Optional[str]
+    ) -> Optional[EntityReferenceList]:
         """
         Method to process the table owners
         """
+        if self.source_config.includeOwners is False:
+            return None
         try:
-            full_table_name = f"{self.context.get().database}.{self.context.get().database_schema}.{table_name}"
-            owner = self.api_client.get_owner_info(full_table_name)
-            if not owner:
-                return
-            owner_ref = self.metadata.get_reference_by_email(email=owner)
+            if not table_owner or not isinstance(table_owner, str):
+                return None
+            owner_ref = self.metadata.get_reference_by_email(email=table_owner)
+            if owner_ref:
+                return owner_ref
+            table_name = table_owner.split("@")[0]
+            owner_ref = self.metadata.get_reference_by_name(name=table_name)
             return owner_ref
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.warning(f"Error processing owner for table {table_name}: {exc}")
-        return
+            logger.warning(f"Error processing owner {table_owner}: {exc}")
+        return None

metadata/ingestion/source/search/opensearch/connection.py

@@ -141,11 +141,7 @@ def get_connection(connection: OpenSearchConnection) -> OpenSearch:
 
     # Check for AWS IAM Authentication
     if isinstance(connection.authType, AWSCredentials):
-        aws_access_key = (
-            connection.authType.awsAccessKeyId.get_secret_value()
-            if connection.authType.awsAccessKeyId
-            else None
-        )
+        aws_access_key = connection.authType.awsAccessKeyId
         aws_secret_key = (
             connection.authType.awsSecretAccessKey.get_secret_value()
             if connection.authType.awsSecretAccessKey