openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.9.1.0__py3-none-any.whl

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -436,21 +436,21 @@ class PowerbiSource(DashboardServiceSource):
         measures = []
         for measure in table.measures or []:
             try:
-                measure_type = (
-                    DataType.MEASURE_HIDDEN
-                    if measure.isHidden
-                    else DataType.MEASURE_VISIBLE
-                )
+                measure_type = DataType.MEASURE_VISIBLE
+                if measure.isHidden:
+                    measure_type = DataType.MEASURE_HIDDEN
+                expression_text = f"Expression : {measure.expression}"
                 description_text = (
-                    f"{measure.description}\n\nExpression : {measure.expression}"
+                    f"Description : {measure.description}"
                     if measure.description
-                    else f"Expression : {measure.expression}"
+                    else ""
                 )
+                description_field_text = f"{expression_text}\n\n{description_text}"
                 parsed_measure = PowerBiMeasureModel(
                     dataType=measure_type,
                     dataTypeDisplay=measure_type,
                     name=measure.name,
-                    description=description_text,
+                    description=description_field_text,
                 )
                 measures.append(Column(**parsed_measure.model_dump()))
             except Exception as err:
@@ -866,7 +866,7 @@ class PowerbiSource(DashboardServiceSource):
 
         try:
             table_info = self._parse_table_info_from_source_exp(table, datamodel_entity)
-            table_name = table.name or table_info.get("table")
+            table_name = table_info.get("table") or table.name
             schema_name = table_info.get("schema")
             database_name = table_info.get("database")
             if (

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -135,7 +135,7 @@ class PowerBiMeasures(BaseModel):
     name: str
     expression: str
     description: Optional[str] = None
-    isHidden: bool
+    isHidden: Optional[bool] = False
 
 
 class PowerBITableSource(BaseModel):

metadata/ingestion/source/database/dbt/metadata.py

@@ -924,7 +924,7 @@ class DbtSource(DbtServiceSource):
                 lineages = get_lineage_by_query(
                     self.metadata,
                     query=query,
-                    service_name=source_elements[0],
+                    service_names=source_elements[0],
                     database_name=source_elements[1],
                     schema_name=source_elements[2],
                     dialect=dialect,

metadata/ingestion/source/database/lineage_source.py

@@ -262,10 +262,18 @@ class LineageSource(QueryParserSource, ABC):
 
         for table_query in table_queries or []:
             if not self._query_already_processed(table_query):
+                # Prepare service names for lineage processing
+                service_names = [table_query.serviceName]
+                if (
+                    self.source_config.processCrossDatabaseLineage
+                    and self.source_config.crossDatabaseServiceNames
+                ):
+                    service_names.extend(self.source_config.crossDatabaseServiceNames)
+
                 lineages: Iterable[Either[AddLineageRequest]] = get_lineage_by_query(
                     self.metadata,
                     query=table_query.query,
-                    service_name=table_query.serviceName,
+                    service_names=service_names,
                     database_name=table_query.databaseName,
                     schema_name=table_query.databaseSchema,
                     dialect=self.dialect,
@@ -334,10 +342,18 @@ class LineageSource(QueryParserSource, ABC):
                         "View Filtered Out",
                     )
                     continue
+                # Prepare service names for view lineage processing
+                service_names = [self.config.serviceName]
+                if (
+                    self.source_config.processCrossDatabaseLineage
+                    and self.source_config.crossDatabaseServiceNames
+                ):
+                    service_names.extend(self.source_config.crossDatabaseServiceNames)
+
                 for lineage in get_view_lineage(
                     view=view,
                     metadata=self.metadata,
-                    service_name=self.config.serviceName,
+                    service_names=service_names,
                     connection_type=self.service_connection.type.value,
                     timeout_seconds=self.source_config.parsingTimeoutLimit,
                 ):

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -207,10 +207,18 @@ class StoredProcedureLineageMixin(ABC):
                 ).graph
 
             self.stored_procedure_query_lineage = True
+            # Prepare service names for lineage processing
+            service_names = [self.service_name]
+            if (
+                self.source_config.processCrossDatabaseLineage
+                and self.source_config.crossDatabaseServiceNames
+            ):
+                service_names.extend(self.source_config.crossDatabaseServiceNames)
+
             for either_lineage in get_lineage_by_query(
                 self.metadata,
                 query=query_by_procedure.query_text,
-                service_name=self.service_name,
+                service_names=service_names,
                 database_name=query_by_procedure.query_database_name,
                 schema_name=query_by_procedure.query_schema_name,
                 dialect=ConnectionTypeDialectMapper.dialect_of(

metadata/ingestion/source/pipeline/airflow/connection.py

@@ -27,7 +27,7 @@ from metadata.generated.schema.entity.services.connections.database.mysqlConnect
     MysqlConnection as MysqlConnectionConfig,
 )
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
-    PostgresConnection,
+    PostgresConnection as PostgresConnectionConfig,
 )
 from metadata.generated.schema.entity.services.connections.database.sqliteConnection import (
     SQLiteConnection,
@@ -70,16 +70,16 @@ def _(_: BackendConnection) -> Engine:
 def _(airflow_connection: MysqlConnectionConfig) -> Engine:
     from metadata.ingestion.source.database.mysql.connection import MySQLConnection
 
-    return MySQLConnection(airflow_connection).get_client()
+    return MySQLConnection(airflow_connection)._get_client()
 
 
 @_get_connection.register
-def _(airflow_connection: PostgresConnection) -> Engine:
+def _(airflow_connection: PostgresConnectionConfig) -> Engine:
     from metadata.ingestion.source.database.postgres.connection import (
-        get_connection as get_postgres_connection,
+        PostgresConnection,
     )
 
-    return get_postgres_connection(airflow_connection)
+    return PostgresConnection(airflow_connection)._get_client()
 
 
 @_get_connection.register
@@ -131,7 +131,9 @@ def test_connection(
 
     def test_pipeline_details_access(session):
         try:
-            result = session.query(SerializedDagModel).first()
+            # Query only the dag_id column to avoid version compatibility issues
+            # The data_compressed column doesn't exist in Airflow 2.2.5
+            result = session.query(SerializedDagModel.dag_id).first()
             return result
         except Exception as e:
             raise AirflowPipelineDetailsAccessError(

metadata/utils/db_utils.py

@@ -14,7 +14,7 @@ Helpers module for db sources
 """
 
 import traceback
-from typing import Iterable
+from typing import Iterable, List, Union
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.table import Table
@@ -50,13 +50,16 @@ def get_host_from_host_port(uri: str) -> str:
 def get_view_lineage(
     view: TableView,
     metadata: OpenMetadata,
-    service_name: str,
+    service_names: Union[str, List[str]],
     connection_type: str,
     timeout_seconds: int = LINEAGE_PARSING_TIMEOUT,
 ) -> Iterable[Either[AddLineageRequest]]:
     """
     Method to generate view lineage
+    Now supports cross-database lineage by accepting a list of service names.
     """
+    if isinstance(service_names, str):
+        service_names = [service_names]
     table_name = view.table_name
     schema_name = view.schema_name
     db_name = view.db_name
@@ -65,7 +68,7 @@ def get_view_lineage(
     table_fqn = fqn.build(
         metadata,
         entity_type=Table,
-        service_name=service_name,
+        service_name=service_names[0],  # Use first service for table entity lookup
         database_name=db_name,
         schema_name=schema_name,
         table_name=table_name,
@@ -95,7 +98,7 @@ def get_view_lineage(
             yield from get_lineage_by_query(
                 metadata,
                 query=view_definition,
-                service_name=service_name,
+                service_names=service_names,
                 database_name=db_name,
                 schema_name=schema_name,
                 dialect=dialect,
@@ -108,7 +111,7 @@ def get_view_lineage(
             yield from get_lineage_via_table_entity(
                 metadata,
                 table_entity=table_entity,
-                service_name=service_name,
+                service_names=service_names,
                 database_name=db_name,
                 schema_name=schema_name,
                 query=view_definition,

metadata/utils/helpers.py

@@ -105,6 +105,11 @@ om_chart_type_dict = {
     "pie": ChartType.Pie,
     "text": ChartType.Text,
     "scatter": ChartType.Scatter,
+    "gauge": ChartType.Gauge,
+    "map": ChartType.Map,
+    "graph": ChartType.Graph,
+    "heatmap": ChartType.Heatmap,
+    "timeline": ChartType.Timeline,
 }
 
 
@@ -183,7 +188,7 @@ def replace_special_with(raw: str, replacement: str) -> str:
     return re.sub(r"[^a-zA-Z0-9]", replacement, raw)
 
 
-def get_standard_chart_type(raw_chart_type: str) -> ChartType.Other:
+def get_standard_chart_type(raw_chart_type: str) -> ChartType:
     """
     Get standard chart type supported by OpenMetadata based on raw chart type input
     :param raw_chart_type: raw chart type to be standardize