openmetadata-ingestion 1.7.2.1__py3-none-any.whl → 1.7.4.0__py3-none-any.whl

metadata/ingestion/source/dashboard/microstrategy/helpers.py

@@ -0,0 +1,67 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Microstrategy source helpers.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict
+
+from metadata.generated.schema.entity.data.table import Column, DataType
+
+
+class MicroStrategyColumnParser:
+    """
+    Responsible for containing the logic to parse a column from MicroStrategy to OpenMetadata
+    """
+
+    datatype_mapping = {
+        "big decimal": DataType.DECIMAL,
+        "binary": DataType.BYTES,
+        "char": DataType.CHAR,
+        "date": DataType.DATE,
+        "decimal": DataType.DECIMAL,
+        "double": DataType.DOUBLE,
+        "float": DataType.FLOAT,
+        "integer": DataType.INT,
+        "longvarbin": DataType.BLOB,
+        "longvarchar": DataType.CLOB,
+        "nchar": DataType.CHAR,
+        "numeric": DataType.NUMERIC,
+        "nvarchar": DataType.VARCHAR,
+        "real": DataType.STRING,
+        "time": DataType.TIME,
+        "timestamp": DataType.TIMESTAMP,
+        "unsigned": DataType.INT,
+        "varbin": DataType.BYTES,
+        "varchar": DataType.VARCHAR,
+        "utf8char": DataType.CHAR,
+    }
+
+    @classmethod
+    def parse(cls, field: Dict[str, Any]) -> Column:
+        """
+        Parses a MicroStrategy table column into an OpenMetadata column.
+        """
+
+        array_data_type = None
+        data_type = cls.datatype_mapping.get(
+            field["dataType"].lower(), DataType.UNKNOWN
+        )
+
+        column_def = {
+            "name": field["name"],
+            "dataTypeDisplay": field["dataType"],
+            "dataType": data_type,
+            "arrayDataType": array_data_type,
+        }
+
+        return Column(**column_def)

metadata/ingestion/source/dashboard/microstrategy/metadata.py

@@ -14,11 +14,23 @@ from typing import Iterable, List, Optional
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
+from metadata.generated.schema.api.data.createDashboardDataModel import (
+    CreateDashboardDataModelRequest,
+)
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.chart import Chart
+from metadata.generated.schema.entity.data.dashboardDataModel import (
+    DashboardDataModel,
+    DataModelType,
+)
+from metadata.generated.schema.entity.data.table import Column, DataType
 from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
     MicroStrategyConnection,
 )
+from metadata.generated.schema.entity.services.dashboardService import (
+    DashboardServiceType,
+)
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
@@ -29,18 +41,29 @@ from metadata.generated.schema.type.basic import (
     EntityName,
     FullyQualifiedEntityName,
     SourceUrl,
+    Uuid,
 )
+from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
+from metadata.generated.schema.type.entityLineage import Source as LineageSource
+from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
+from metadata.ingestion.lineage.parser import LineageParser
+from metadata.ingestion.lineage.sql_lineage import get_table_entities_from_query
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
+from metadata.ingestion.source.dashboard.microstrategy.helpers import (
+    MicroStrategyColumnParser,
+)
 from metadata.ingestion.source.dashboard.microstrategy.models import (
     MstrDashboard,
     MstrDashboardDetails,
+    MstrDataset,
     MstrPage,
 )
 from metadata.utils import fqn
-from metadata.utils.filters import filter_by_chart
+from metadata.utils.filters import filter_by_chart, filter_by_datamodel
 from metadata.utils.helpers import clean_uri, get_standard_chart_type
 from metadata.utils.logger import ingestion_logger
 
@@ -163,7 +186,75 @@ class MicrostrategySource(DashboardServiceSource):
         dashboard_details: MstrDashboardDetails,
         db_service_name: Optional[str] = None,
     ) -> Optional[Iterable[AddLineageRequest]]:
-        """Not Implemented"""
+        """
+        Get lineage between datamodel and data sources
+        """
+        if not db_service_name:
+            return
+
+        database_service = self.metadata.get_by_name(
+            entity=DatabaseService, fqn=db_service_name
+        )
+        dialect = ConnectionTypeDialectMapper.dialect_of(
+            database_service.serviceType.value
+        )
+
+        for dataset in dashboard_details.datasets:
+            cube_sql = self.client.get_cube_sql_details(
+                dashboard_details.projectId, dataset.id
+            )
+            if not cube_sql:
+                continue
+
+            datamodel_fqn = fqn.build(
+                self.metadata,
+                entity_type=DashboardDataModel,
+                service_name=self.context.get().dashboard_service,
+                data_model_name=dataset.id,
+            )
+            datamodel_entity = self.metadata.get_by_name(
+                entity=DashboardDataModel, fqn=datamodel_fqn
+            )
+
+            try:
+                lineage_parser = LineageParser(cube_sql, dialect=dialect)
+                for table in lineage_parser.source_tables:
+                    table_entities = get_table_entities_from_query(
+                        metadata=self.metadata,
+                        service_name=db_service_name,
+                        database_name="*",
+                        database_schema="*",
+                        table_name=str(table),
+                    )
+                    if not table_entities:
+                        logger.debug(f"Table not found in metadata: {str(table)}")
+                        continue
+                    for table_entity in table_entities or []:
+                        yield Either(
+                            right=AddLineageRequest(
+                                edge=EntitiesEdge(
+                                    fromEntity=EntityReference(
+                                        id=Uuid(table_entity.id.root),
+                                        type="table",
+                                    ),
+                                    toEntity=EntityReference(
+                                        id=Uuid(datamodel_entity.id.root),
+                                        type="dashboardDataModel",
+                                    ),
+                                    lineageDetails=LineageDetails(
+                                        source=LineageSource.DashboardLineage
+                                    ),
+                                )
+                            )
+                        )
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name="Dashboard Lineage",
+                        error=f"Error to yield dashboard lineage details: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
 
     def yield_dashboard_chart(
         self, dashboard_details: MstrDashboardDetails
@@ -213,6 +304,74 @@ class MicrostrategySource(DashboardServiceSource):
                     )
                 )
 
+    def _get_column_info(self, dataset: MstrDataset) -> Optional[List[Column]]:
+        """Build columns from dataset"""
+        datasource_columns = []
+        for available_object in dataset.availableObjects or []:
+            try:
+                parsed_column = {
+                    "dataTypeDisplay": available_object.type.title(),
+                    "dataType": DataType.UNKNOWN,
+                    "name": available_object.name,
+                    "displayName": available_object.name,
+                }
+                parsed_column_children = []
+                for form in available_object.forms or []:
+                    parsed_column_children.append(MicroStrategyColumnParser.parse(form))
+                if parsed_column_children:
+                    parsed_column["children"] = parsed_column_children
+
+                datasource_columns.append(Column(**parsed_column))
+            except Exception as exc:
+                logger.debug(traceback.format_exc())
+                logger.warning(f"Error to yield datamodel column: {exc}")
+        return datasource_columns
+
+    def yield_datamodel(
+        self, dashboard_details: MstrDashboardDetails
+    ) -> Optional[Iterable[CreateDashboardDataModelRequest]]:
+        """Get datamodel method
+
+        Args:
+            dashboard_details:
+        Returns:
+            Iterable[CreateDashboardDataModelRequest]
+        """
+        try:
+            if self.source_config.includeDataModels:
+                for dataset in dashboard_details.datasets:
+                    if filter_by_datamodel(
+                        self.source_config.dataModelFilterPattern, dataset.name
+                    ):
+                        self.status.filter(dataset.name, "Data model filtered out.")
+                        continue
+                    data_model_type = DataModelType.MicroStrategyDataset.value
+                    datamodel_columns = self._get_column_info(dataset)
+
+                    data_model_request = CreateDashboardDataModelRequest(
+                        name=EntityName(dataset.id),
+                        displayName=dataset.name,
+                        service=FullyQualifiedEntityName(
+                            self.context.get().dashboard_service
+                        ),
+                        dataModelType=data_model_type,
+                        serviceType=DashboardServiceType.MicroStrategy.value,
+                        columns=datamodel_columns,
+                        project=self.get_project_name(
+                            dashboard_details=dashboard_details
+                        ),
+                    )
+                    yield Either(right=data_model_request)
+                    self.register_record_datamodel(datamodel_request=data_model_request)
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=dataset.name,
+                    error=f"Error yielding Data Model [{dataset.name}]: {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+
     def close(self):
         # close the api session
         self.client.close_api_session()

metadata/ingestion/source/dashboard/microstrategy/models.py

@@ -12,7 +12,7 @@
 MicroStrategy Models
 """
 from datetime import datetime
-from typing import Any, List, Optional
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel
 
@@ -134,6 +134,17 @@ class MstrAvailableObject(BaseModel):
     id: str
     name: str
     type: str
+    forms: Optional[List[Dict[str, Any]]] = None
+
+
+class MstrDataset(BaseModel):
+    id: str
+    name: str
+    availableObjects: Optional[List[MstrAvailableObject]] = None
+    rows: Optional[List[Dict[str, Any]]] = None
+    columns: Optional[List[Dict[str, Any]]] = None
+    pageBy: Optional[List[Dict[str, Any]]] = None
+    sqlStatement: Optional[str] = None
 
 
 class MstrDashboardDetails(BaseModel):
@@ -143,6 +154,7 @@ class MstrDashboardDetails(BaseModel):
     projectName: str
     currentChapter: str
     chapters: List[MstrChapter]
+    datasets: List[MstrDataset]
 
 
 class AuthHeaderCookie(BaseModel):

metadata/ingestion/source/dashboard/qlikcloud/metadata.py

@@ -134,6 +134,12 @@ class QlikcloudSource(QliksenseSource):
             if self.filter_draft_dashboard(dashboard):
                 # Skip unpublished dashboards
                 continue
+            if dashboard.space_id not in self.projects_map:
+                logger.warning(
+                    f"Project ID '{dashboard.space_id}' for Dashboard '{dashboard.name}' is not present"
+                    " in projects map"
+                )
+                continue
             project = self.projects_map[dashboard.space_id]
             if self.filter_projects_by_type(project):
                 # Skip dashboard based on space type filter

metadata/ingestion/source/dashboard/qlikcloud/models.py

@@ -21,6 +21,7 @@ class QlikSpaceType(Enum):
     MANAGED = "Managed"
     SHARED = "Shared"
     PERSONAL = "Personal"
+    DATA = "Data"
 
 
 # Space Models

metadata/ingestion/source/dashboard/tableau/models.py

@@ -13,10 +13,9 @@
 Tableau Source Model module
 """
 
-import uuid
 from typing import Dict, List, Optional, Set, Union
 
-from pydantic import BaseModel, ConfigDict, Field, validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 from metadata.generated.schema.entity.data.chart import ChartType
 from metadata.generated.schema.entity.data.table import Table
@@ -29,18 +28,9 @@ class TableauBaseModel(BaseModel):
 
     model_config = ConfigDict(extra="allow")
 
-    # in case of personal space workbooks, the project id is returned as a UUID
-    id: Union[str, uuid.UUID]
+    id: str
     name: Optional[str] = None
 
-    # pylint: disable=no-self-argument
-    @field_validator("id", mode="before")
-    def coerce_uuid_to_string(cls, value):
-        """Ensure id is always stored as a string internally"""
-        if isinstance(value, uuid.UUID):
-            return str(value)
-        return value
-
     def __hash__(self):
         return hash(self.id)
 
@@ -145,7 +135,7 @@ class UpstreamTable(BaseModel):
     database: Optional[TableauDatabase] = None
     referencedByQueries: Optional[List[CustomSQLTable]] = None
 
-    @validator("referencedByQueries", pre=True)
+    @field_validator("referencedByQueries", mode="before")
     @classmethod
     def filter_none_queries(cls, v):
         """Filter out CustomSQLTable items where query==None."""
@@ -197,7 +187,7 @@ class TableauDashboard(TableauBaseModel):
     tags: Optional[Set] = []
     webpageUrl: Optional[str] = None
     charts: Optional[List[TableauChart]] = None
-    dataModels: Optional[List[DataSource]] = []
+    dataModels: List[DataSource] = []
     custom_sql_queries: Optional[List[str]] = None
     user_views: Optional[int] = None
 

metadata/ingestion/source/database/unitycatalog/lineage.py

@@ -16,6 +16,7 @@ from typing import Iterable, Optional
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.database import Database
+from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.database.unityCatalogConnection import (
     UnityCatalogConnection,
@@ -42,6 +43,7 @@ from metadata.ingestion.source.database.unitycatalog.client import UnityCatalogC
 from metadata.ingestion.source.database.unitycatalog.connection import get_connection
 from metadata.ingestion.source.database.unitycatalog.models import LineageTableStreams
 from metadata.utils import fqn
+from metadata.utils.filters import filter_by_database, filter_by_schema, filter_by_table
 from metadata.utils.helpers import retry_with_docker_host
 from metadata.utils.logger import ingestion_logger
 
@@ -175,16 +177,47 @@ class UnitycatalogLineageSource(Source):
         for database in self.metadata.list_all_entities(
             entity=Database, params={"service": self.config.serviceName}
         ):
-            for table in self.metadata.list_all_entities(
-                entity=Table, params={"database": database.fullyQualifiedName.root}
+            if filter_by_database(
+                self.source_config.databaseFilterPattern, database.name.root
             ):
-                databricks_table_fqn = f"{table.database.name}.{table.databaseSchema.name}.{table.name.root}"
-                table_streams: LineageTableStreams = self.client.get_table_lineage(
-                    databricks_table_fqn
-                )
-                yield from self._handle_upstream_table(
-                    table_streams, table, databricks_table_fqn
+                self.status.filter(
+                    database.fullyQualifiedName.root,
+                    "Catalog Filtered Out",
                 )
+                continue
+            for schema in self.metadata.list_all_entities(
+                entity=DatabaseSchema,
+                params={"database": database.fullyQualifiedName.root},
+            ):
+                if filter_by_schema(
+                    self.source_config.schemaFilterPattern, schema.name.root
+                ):
+                    self.status.filter(
+                        schema.fullyQualifiedName.root,
+                        "Schema Filtered Out",
+                    )
+                    continue
+                for table in self.metadata.list_all_entities(
+                    entity=Table,
+                    params={"databaseSchema": schema.fullyQualifiedName.root},
+                ):
+                    if filter_by_table(
+                        self.source_config.tableFilterPattern, table.name.root
+                    ):
+                        self.status.filter(
+                            table.fullyQualifiedName.root,
+                            "Table Filtered Out",
+                        )
+                        continue
+
+                    databricks_table_fqn = f"{table.database.name}.{table.databaseSchema.name}.{table.name.root}"
+                    logger.debug(f"Processing table: {databricks_table_fqn}")
+                    table_streams: LineageTableStreams = self.client.get_table_lineage(
+                        databricks_table_fqn
+                    )
+                    yield from self._handle_upstream_table(
+                        table_streams, table, databricks_table_fqn
+                    )
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)

metadata/ingestion/source/database/unitycatalog/metadata.py

@@ -340,7 +340,7 @@ class UnitycatalogSource(
                         schema_name=schema_name,
                     )
                 ),
-                owners=self.get_owner_ref(table_name),
+                owners=self.get_owner_ref(table.owner),
             )
             yield Either(right=table_request)
 
@@ -403,6 +403,7 @@ class UnitycatalogSource(
             )
             if referred_table_fqn:
                 for parent_column in column.parent_columns:
+                    # pylint: disable=protected-access
                     col_fqn = fqn._build(referred_table_fqn, parent_column, quote=False)
                     if col_fqn:
                         referred_column_fqns.append(FullyQualifiedEntityName(col_fqn))
@@ -419,6 +420,7 @@ class UnitycatalogSource(
 
         return table_constraints
 
+    # pylint: disable=arguments-differ
     def update_table_constraints(
         self, table_constraints, foreign_columns, columns
     ) -> List[TableConstraint]:
@@ -524,18 +526,25 @@ class UnitycatalogSource(
     def close(self):
         """Nothing to close"""
 
-    def get_owner_ref(self, table_name: str) -> Optional[EntityReferenceList]:
+    # pylint: disable=arguments-renamed
+    def get_owner_ref(
+        self, table_owner: Optional[str]
+    ) -> Optional[EntityReferenceList]:
         """
         Method to process the table owners
         """
+        if self.source_config.includeOwners is False:
+            return None
         try:
-            full_table_name = f"{self.context.get().database}.{self.context.get().database_schema}.{table_name}"
-            owner = self.api_client.get_owner_info(full_table_name)
-            if not owner:
-                return
-            owner_ref = self.metadata.get_reference_by_email(email=owner)
+            if not table_owner or not isinstance(table_owner, str):
+                return None
+            owner_ref = self.metadata.get_reference_by_email(email=table_owner)
+            if owner_ref:
+                return owner_ref
+            table_name = table_owner.split("@")[0]
+            owner_ref = self.metadata.get_reference_by_name(name=table_name)
             return owner_ref
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.warning(f"Error processing owner for table {table_name}: {exc}")
-        return
+            logger.warning(f"Error processing owner {table_owner}: {exc}")
+        return None

metadata/ingestion/source/database/unitycatalog/models.py

@@ -68,4 +68,4 @@ class Type(BaseModel):
     fields: Optional[List[ColumnJson]] = None
 
 
-ColumnJson.update_forward_refs()
+ColumnJson.model_rebuild()

metadata/ingestion/source/search/opensearch/connection.py

@@ -141,11 +141,7 @@ def get_connection(connection: OpenSearchConnection) -> OpenSearch:
 
     # Check for AWS IAM Authentication
     if isinstance(connection.authType, AWSCredentials):
-        aws_access_key = (
-            connection.authType.awsAccessKeyId.get_secret_value()
-            if connection.authType.awsAccessKeyId
-            else None
-        )
+        aws_access_key = connection.authType.awsAccessKeyId
         aws_secret_key = (
             connection.authType.awsSecretAccessKey.get_secret_value()
             if connection.authType.awsSecretAccessKey