openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/source/dashboard/metabase/client.py

@@ -48,6 +48,7 @@ PASSWORD_HEADER = "password"
 SESSION_HEADERS = {"Content-Type": "application/json", "Accept": "*/*"}
 DEFAULT_TIMEOUT = 30
 METABASE_SESSION_HEADER = "X-Metabase-Session"
+METABASE_API_HEADER = "X-API-KEY"
 API_VERSION = "api"
 
 
@@ -56,8 +57,13 @@ class MetabaseClient:
     Client Handling API communication with Metabase
     """
 
-    def _get_metabase_session(self) -> str:
+    def _get_metabase_session(self) -> Optional[str]:
         try:
+            # If API token is provided, return None as we don't need a session
+            if self.config.apiKey:
+                return None
+
+            # Otherwise use username/password authentication
             params = {USERNAME_HEADER: self.config.username}
             if self.config.password:
                 params[PASSWORD_HEADER] = self.config.password.get_secret_value()
@@ -82,14 +88,30 @@ class MetabaseClient:
         config: MetabaseConnection,
     ):
         self.config = config
-        session_token = self._get_metabase_session()
-        client_config: ClientConfig = ClientConfig(
-            base_url=clean_uri(str(self.config.hostPort)),
-            api_version=API_VERSION,
-            auth_header=AUTHORIZATION_HEADER,
-            auth_token=lambda: (NO_ACCESS_TOKEN, 0),
-            extra_headers={METABASE_SESSION_HEADER: session_token},
-        )
+
+        # Determine authentication method and set headers accordingly
+        if self.config.apiKey:
+            # Use API token authentication
+            client_config: ClientConfig = ClientConfig(
+                base_url=clean_uri(str(self.config.hostPort)),
+                api_version=API_VERSION,
+                auth_header=AUTHORIZATION_HEADER,
+                auth_token=lambda: (NO_ACCESS_TOKEN, 0),
+                extra_headers={
+                    METABASE_API_HEADER: self.config.apiKey.get_secret_value()
+                },
+            )
+        else:
+            # Use session-based authentication
+            session_token = self._get_metabase_session()
+            client_config: ClientConfig = ClientConfig(
+                base_url=clean_uri(str(self.config.hostPort)),
+                api_version=API_VERSION,
+                auth_header=AUTHORIZATION_HEADER,
+                auth_token=lambda: (NO_ACCESS_TOKEN, 0),
+                extra_headers={METABASE_SESSION_HEADER: session_token},
+            )
+
         self.client = REST(client_config)
 
     def get_dashboards_list(
@@ -160,9 +182,9 @@ class MetabaseClient:
                     chart = MetabaseChart.model_validate(chart_data)
                     charts_dict[chart.id] = chart
             return charts_dict
-        except Exception:
+        except Exception as e:
             logger.debug(traceback.format_exc())
-            logger.warning("Failed to fetch the cards")
+            logger.warning(f"Failed to fetch the cards : {e}")
         return {}
 
     def _create_default_dashboard_details(

metadata/ingestion/source/dashboard/metabase/metadata.py

@@ -108,6 +108,8 @@ class MetabaseSource(DashboardServiceSource):
         """
         Get List of all dashboards
         """
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         self.dashboards_list = self.client.get_dashboards_list(self.collections)
         return self.dashboards_list
 
@@ -180,6 +182,8 @@ class MetabaseSource(DashboardServiceSource):
         Get dashboard owner from email
         """
         try:
+            if not self.source_config.includeOwners:
+                return None
             if dashboard_details.creator_id:
                 owner_details = self.client.get_user_details(
                     dashboard_details.creator_id

metadata/ingestion/source/dashboard/metabase/models.py

@@ -11,9 +11,11 @@
 """
 Metabase Models
 """
+import ast
+import json
 from typing import List, Optional
 
-from pydantic import BaseModel, BeforeValidator, Field
+from pydantic import BaseModel, BeforeValidator, Field, field_validator
 from typing_extensions import Annotated
 
 MetabaseStrId = Annotated[str, BeforeValidator(lambda x: str(x))]
@@ -84,6 +86,52 @@ class MetabaseChart(BaseModel):
     display: Optional[str] = None
     dashboard_ids: List[str] = []
 
+    @field_validator("dataset_query", mode="before")
+    @classmethod
+    def parse_dataset_query(cls, v):
+        if v is None:
+            return None
+
+        # If it's already a dict or DatasetQuery object, return as is
+        if isinstance(v, (dict, DatasetQuery)):
+            return v
+
+        # If it's a string, try multiple parsing strategies
+        if isinstance(v, str):
+            # Strategy 1: Try standard JSON parsing
+            try:
+                return json.loads(v)
+            except json.JSONDecodeError:
+                pass
+
+            # Strategy 2: Try ast.literal_eval for Python dict strings
+            try:
+                parsed = ast.literal_eval(v)
+                if isinstance(parsed, dict):
+                    return parsed
+            except (ValueError, SyntaxError):
+                pass
+
+            # Strategy 3: More sophisticated quote replacement
+            try:
+                # Handle None values and booleans
+                json_str = (
+                    v.replace("'", '"')
+                    .replace("None", "null")
+                    .replace("True", "true")
+                    .replace("False", "false")
+                )
+                return json.loads(json_str)
+            except json.JSONDecodeError:
+                pass
+
+            # If all strategies fail, log and return None
+            print(f"Failed to parse dataset_query string: {v[:100]}...")
+            return None
+
+        # For any other type, return as is and let Pydantic handle validation
+        return v
+
 
 class DashCard(BaseModel):
     card: MetabaseChart

metadata/ingestion/source/dashboard/microstrategy/metadata.py

@@ -228,7 +228,7 @@ class MicrostrategySource(DashboardServiceSource):
                 for table in lineage_parser.source_tables:
                     table_entities = get_table_entities_from_query(
                         metadata=self.metadata,
-                        service_name=prefix_db_service_name,
+                        service_names=prefix_db_service_name,
                         database_name="*",
                         database_schema="*",
                         table_name=str(table),

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -38,6 +38,7 @@ from metadata.ingestion.source.dashboard.powerbi.models import (
     PowerBIReport,
     PowerBiTable,
     PowerBiToken,
+    ReportPagesAPIResponse,
     ReportsResponse,
     TablesResponse,
     Tile,
@@ -46,6 +47,7 @@ from metadata.ingestion.source.dashboard.powerbi.models import (
     WorkSpaceScanResponse,
 )
 from metadata.utils.filters import validate_regex
+from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import utils_logger
 
 logger = utils_logger()
@@ -74,7 +76,7 @@ class PowerBiApiClient:
             authority=self.config.authorityURI + self.config.tenantId,
         )
         client_config = ClientConfig(
-            base_url="https://api.powerbi.com",
+            base_url=clean_uri(self.config.apiURL),
             api_version="v1.0",
             auth_token=self.get_auth_token,
             auth_header="Authorization",
@@ -280,6 +282,21 @@ class PowerBiApiClient:
 
         return None
 
+    def fetch_report_pages(self, group_id: str, report_id: str) -> Optional[List[dict]]:
+        # get report pages for report url formation
+        try:
+            # https://api.powerbi.com/v1.0/myorg/groups/4e57dcbb-***/reports/a2902011-***/pages
+            response_data = self.client.get(
+                f"/myorg/groups/{group_id}/reports/{report_id}/pages"
+            )
+            if response_data:
+                response = ReportPagesAPIResponse(**response_data)
+                return response.value
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error fetching report pages: {exc}")
+        return []
+
     def regex_to_odata_condition(self, regex: str) -> str:
         """
         Convert a regex pattern to an OData filter condition

metadata/ingestion/source/dashboard/powerbi/databricks_parser.py

@@ -0,0 +1,108 @@
+import re
+from typing import List, Optional
+
+from metadata.ingestion.lineage.models import Dialect
+from metadata.ingestion.lineage.parser import LineageParser
+from metadata.utils.logger import ingestion_logger
+
+NATIVE_QUERY_PARSER_EXPRESSION = re.compile(
+    r"Value\.NativeQuery\(\s*"
+    r"(?P<catalog_type>[A-Za-z0-9_\.]+)\("
+    r"(?P<catalog_info>.*?)\)"
+    r"(?P<catalog_parameters>\{\[.*?\]\})\[Data\],\s*"
+    r'"(?P<query>.*?)"',
+    re.DOTALL,
+)
+logger = ingestion_logger()
+
+
+def parse_databricks_native_query_source(
+    source_expression: str,
+) -> Optional[List[dict]]:
+
+    groups = NATIVE_QUERY_PARSER_EXPRESSION.search(source_expression)
+
+    if groups:
+        details = groups.groupdict()
+        catalog_info = details.get("catalog_info", "")
+        catalog_parameters = details.get("catalog_parameters", "")
+        catalog_info_match = re.match(
+            r".*Catalog\s*=\s*(?P<catalog>[^,]+?)\s*,", catalog_info
+        )
+        if not catalog_info_match:
+            logger.error(f"Could not find catalog in info: {catalog_info}")
+            catalog = None
+        else:
+            catalog_groups = catalog_info_match.groupdict()
+            catalog = catalog_groups.get("catalog", None)
+        database_match = re.search(
+            r'Name\s*=\s*(?P<database>[^,]+?)\s*,\s*Kind\s*=\s*"Database"',
+            catalog_parameters,
+        )
+        database = None
+        if database_match:
+            database = database_match.groupdict().get("database", None)
+        else:
+            logger.error(f"Could not find database in parameters: {catalog_parameters}")
+            database = catalog
+        if not (database or catalog):
+            logger.error(f"Could not find database in {source_expression}")
+            return None
+
+        database = database.strip('"').strip()
+        parser_query = details.get("query")
+
+        # Clean the query for parser
+
+        # 2. Remove PowerBI line feed markers #(lf) and clean up the query
+        parser_query = parser_query.replace("#(lf)", "\n")
+
+        # 3. Remove SQL comments that might cause issues (// style comments)
+        parser_query = re.sub(r"//[^\n]*", "", parser_query)
+
+        # 4. Clean up excessive whitespace
+        parser_query = re.sub(r"\s+", " ", parser_query).strip()
+
+        logger.debug(
+            f"Attempting LineageParser with cleaned query: {parser_query[:200]}"
+        )
+        if re.match(
+            "^([A-Za-z0-9_]+)(?:\.([A-Za-z0-9_]+))?(?:\.([A-Za-z0-9_]+))?$",
+            parser_query,
+        ):
+            logger.debug(
+                "Query appears to be a simple table reference, skipping LineageParser."
+            )
+            schema_table = parser_query.split(".")
+            schema, table = (
+                schema_table[-2:] if len(schema_table) > 1 else [None, schema_table[0]]
+            )
+
+            return [{"database": database, "schema": schema, "table": table}]
+        try:
+            parser = LineageParser(
+                parser_query, dialect=Dialect.DATABRICKS, timeout_seconds=30
+            )
+            if parser.query_parsing_success is False:
+                raise Exception(parser.query_parsing_failure_reason)
+        except Exception as parser_exc:
+            logger.error(
+                f"LineageParser failed parsing query with error {parser_query[:200]} ",
+                exc_info=parser_exc,
+            )
+            return None
+
+        lineage_tables_list = []
+        for source_table in parser.source_tables:
+            lineage_tables_list.append(
+                {
+                    "database": database,
+                    "schema": source_table.schema.raw_name,
+                    "table": source_table.raw_name,
+                }
+            )
+        return lineage_tables_list
+
+    else:
+        logger.error(f"Invalid Databricks Native Query Syntax: {source_expression}")
+        return None