openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -135,7 +135,7 @@ class PowerBiMeasures(BaseModel):
     name: str
     expression: str
     description: Optional[str] = None
-    isHidden: bool
+    isHidden: Optional[bool] = False
 
 
 class PowerBITableSource(BaseModel):
@@ -302,3 +302,22 @@ class DataModelSchema(BaseModel):
 
     tables: Optional[List[PowerBiTable]] = None
     connectionFile: Optional[ConnectionFile] = None
+
+
+class ReportPage(BaseModel):
+    """
+    PowerBI report pages API response
+    single report Page object
+    """
+
+    name: str
+    displayName: Optional[str] = None
+
+
+class ReportPagesAPIResponse(BaseModel):
+    """
+    PowerBI report pages API response
+    """
+
+    odata_context: str = Field(alias="@odata.context")
+    value: Optional[List[ReportPage]] = None

metadata/ingestion/source/dashboard/redash/metadata.py

@@ -110,6 +110,8 @@ class RedashSource(DashboardServiceSource):
         )
 
     def get_dashboards_list(self) -> Optional[List[dict]]:
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         return self.dashboard_list
 
     def get_dashboard_name(self, dashboard: dict) -> str:
@@ -123,6 +125,8 @@ class RedashSource(DashboardServiceSource):
         Get owner from email
         """
         try:
+            if not self.source_config.includeOwners:
+                return None
             if dashboard_details.get("user") and dashboard_details["user"].get("email"):
                 return self.metadata.get_reference_by_email(
                     dashboard_details["user"].get("email")

metadata/ingestion/source/dashboard/sigma/metadata.py

@@ -91,6 +91,8 @@ class SigmaSource(DashboardServiceSource):
         """
         get list of dashboard
         """
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         return self.client.get_dashboards()
 
     def get_dashboard_name(self, dashboard: Workbook) -> Optional[str]:
@@ -367,6 +369,8 @@ class SigmaSource(DashboardServiceSource):
         Get owner from email
         """
         try:
+            if not self.source_config.includeOwners:
+                return None
             if dashboard_details.ownerId:
                 owner = self.client.get_owner_detail(dashboard_details.ownerId)
                 return self.metadata.get_reference_by_email(owner.email)

metadata/ingestion/source/dashboard/superset/api_source.py

@@ -77,6 +77,8 @@ class SupersetAPISource(SupersetSourceMixin):
         """
         Get List of all dashboards
         """
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         current_page = 0
         page_size = 25
         total_dashboards = self.client.fetch_total_dashboards()

metadata/ingestion/source/dashboard/superset/db_source.py

@@ -111,6 +111,8 @@ class SupersetDBSource(SupersetSourceMixin):
         """
         Get List of all dashboards
         """
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         query = (
             FETCH_DASHBOARDS
             if self.source_config.includeDraftDashboard

metadata/ingestion/source/dashboard/superset/mixin.py

@@ -117,6 +117,8 @@ class SupersetSourceMixin(DashboardServiceSource):
         self, dashboard_details: Union[DashboardResult, FetchDashboard]
     ) -> Optional[EntityReferenceList]:
         try:
+            if not self.source_config.includeOwners:
+                return None
             if hasattr(dashboard_details, "owners"):
                 for owner in dashboard_details.owners or []:
                     if owner.email:

metadata/ingestion/source/dashboard/tableau/client.py

@@ -106,8 +106,15 @@ class TableauClient:
     def site_id(self) -> str:
         return self.tableau_server.site_id
 
-    def get_tableau_owner(self, owner_id: str) -> Optional[TableauOwner]:
+    def get_tableau_owner(
+        self, owner_id: str, include_owners: bool = True
+    ) -> Optional[TableauOwner]:
+        """
+        Get tableau owner with optional include_owners flag
+        """
         try:
+            if not include_owners:
+                return None
             if owner_id in self.owner_cache:
                 return self.owner_cache[owner_id]
             owner = self.tableau_server.users.get_by_id(owner_id) if owner_id else None
@@ -122,7 +129,7 @@ class TableauClient:
         return None
 
     def get_workbook_charts_and_user_count(
-        self, views: List[ViewItem]
+        self, views: List[ViewItem], include_owners: bool = True
     ) -> Optional[Tuple[Optional[int], Optional[List[TableauChart]]]]:
         """
         Fetches workbook charts and dashboard user view count
@@ -136,7 +143,7 @@ class TableauClient:
                         id=str(view.id),
                         name=view.name,
                         tags=view.tags,
-                        owner=self.get_tableau_owner(view.owner_id),
+                        owner=self.get_tableau_owner(view.owner_id, include_owners),
                         contentUrl=view.content_url,
                         sheetType=view.sheet_type,
                     )
@@ -202,7 +209,7 @@ class TableauClient:
             logger.debug(f"Failed to get project parents by id: {str(e)}")
         return None
 
-    def get_workbooks(self) -> Iterable[TableauDashboard]:
+    def get_workbooks(self, include_owners: bool = True) -> Iterable[TableauDashboard]:
         """
         Fetch all tableau workbooks
         """
@@ -212,7 +219,7 @@ class TableauClient:
             try:
                 self.tableau_server.workbooks.populate_views(workbook, usage=True)
                 charts, user_views = self.get_workbook_charts_and_user_count(
-                    workbook.views
+                    workbook.views, include_owners
                 )
                 workbook = TableauDashboard(
                     id=str(workbook.id),
@@ -220,7 +227,7 @@ class TableauClient:
                     project=TableauBaseModel(
                         id=str(workbook.project_id), name=workbook.project_name
                     ),
-                    owner=self.get_tableau_owner(workbook.owner_id),
+                    owner=self.get_tableau_owner(workbook.owner_id, include_owners),
                     description=workbook.description,
                     tags=workbook.tags,
                     webpageUrl=workbook.webpage_url,
@@ -248,9 +255,11 @@ class TableauClient:
             "Please check if the user has permissions to access the Dashboards information"
         )
 
-    def test_get_workbook_views(self):
+    def test_get_workbook_views(self, include_owners: bool = True):
         workbook = self.test_get_workbooks()
-        charts, _ = self.get_workbook_charts_and_user_count(workbook.views)
+        charts, _ = self.get_workbook_charts_and_user_count(
+            workbook.views, include_owners
+        )
         if charts:
             return True
         raise TableauChartsException(
@@ -258,9 +267,11 @@ class TableauClient:
             "Please check if the user has permissions to access the Charts information"
         )
 
-    def test_get_owners(self) -> Optional[List[TableauOwner]]:
+    def test_get_owners(
+        self, include_owners: bool = True
+    ) -> Optional[List[TableauOwner]]:
         workbook = self.test_get_workbooks()
-        owners = self.get_tableau_owner(workbook.owner_id)
+        owners = self.get_tableau_owner(workbook.owner_id, include_owners)
         if owners is not None:
             return owners
         raise TableauOwnersNotFound(
@@ -320,14 +331,18 @@ class TableauClient:
                     workbook_id=dashboard_id, first=entities_per_page, offset=offset
                 )
             )
-            if datasources_graphql_result:
-                if datasources_graphql_result and datasources_graphql_result.get(
-                    "data"
-                ):
+            if datasources_graphql_result and datasources_graphql_result.get("data"):
+                if datasources_graphql_result["data"].get("workbooks"):
                     tableau_datasource_connection = TableauDatasourcesConnection(
                         **datasources_graphql_result["data"]["workbooks"][0]
                     )
                     return tableau_datasource_connection.embeddedDatasourcesConnection
+                else:
+                    logger.warning(
+                        f"No Datasources found in GraphQL datasources query result for the workbook {dashboard_id}. "
+                        "If this is a recently created or updated workbook, it may take some time "
+                        f"to become available for querying via the GraphQL API. : \n graphql = {datasources_graphql_result}\n"
+                    )
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning(

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -132,7 +132,11 @@ class TableauSource(DashboardServiceSource):
         return cls(config, metadata)
 
     def get_dashboards_list(self) -> Iterable[TableauDashboard]:
-        yield from self.client.get_workbooks()
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
+        yield from self.client.get_workbooks(
+            include_owners=self.source_config.includeOwners
+        )
 
     def get_dashboard_name(self, dashboard: TableauDashboard) -> str:
         return dashboard.name
@@ -151,6 +155,8 @@ class TableauSource(DashboardServiceSource):
         Get dashboard owner from email
         """
         try:
+            if not self.source_config.includeOwners:
+                return None
             if dashboard_details.owner and dashboard_details.owner.email:
                 return self.metadata.get_reference_by_email(
                     dashboard_details.owner.email

metadata/ingestion/source/database/athena/utils.py

@@ -173,7 +173,7 @@ def get_columns(self, connection, table_name, schema=None, **kw):
             for glue_col in glue_columns:
                 col_name = glue_col["Name"]
                 col_type = glue_col["Type"]
-                col_comment = glue_col.get("Comment", "")
+                col_comment = glue_col.get("Comment")
                 col_parameters = glue_col.get("Parameters", {})
 
                 # Check if this is a non-current Iceberg column

metadata/ingestion/source/database/bigquery/connection.py

@@ -31,6 +31,7 @@ from metadata.generated.schema.entity.services.connections.testConnectionResult
     TestConnectionResult,
 )
 from metadata.generated.schema.security.credentials.gcpCredentials import (
+    GcpADC,
     GcpCredentialsPath,
 )
 from metadata.generated.schema.security.credentials.gcpValues import (
@@ -68,25 +69,21 @@ def get_connection_url(connection: BigQueryConnection) -> str:
             connection.credentials.gcpConfig.projectId, SingleProjectId
         ):
             if not connection.credentials.gcpConfig.projectId.root:
-                return f"{connection.scheme.value}://{connection.billingProjectId or connection.credentials.gcpConfig.projectId.root or ''}"
+                return f"{connection.scheme.value}://{connection.credentials.gcpConfig.projectId.root or ''}"
             if (
                 not connection.credentials.gcpConfig.privateKey
                 and connection.credentials.gcpConfig.projectId.root
             ):
                 project_id = connection.credentials.gcpConfig.projectId.root
-                os.environ["GOOGLE_CLOUD_PROJECT"] = (
-                    connection.billingProjectId or project_id
-                )
-            return f"{connection.scheme.value}://{connection.billingProjectId or connection.credentials.gcpConfig.projectId.root}"
+                os.environ["GOOGLE_CLOUD_PROJECT"] = project_id
+            return f"{connection.scheme.value}://{connection.credentials.gcpConfig.projectId.root}"
         elif isinstance(connection.credentials.gcpConfig.projectId, MultipleProjectId):
             for project_id in connection.credentials.gcpConfig.projectId.root:
                 if not connection.credentials.gcpConfig.privateKey and project_id:
                     # Setting environment variable based on project id given by user / set in ADC
-                    os.environ["GOOGLE_CLOUD_PROJECT"] = (
-                        connection.billingProjectId or project_id
-                    )
-                return f"{connection.scheme.value}://{connection.billingProjectId or project_id}"
-            return f"{connection.scheme.value}://{connection.billingProjectId or ''}"
+                    os.environ["GOOGLE_CLOUD_PROJECT"] = project_id
+                return f"{connection.scheme.value}://{project_id}"
+            return f"{connection.scheme.value}://"
 
     # If gcpConfig is the JSON key path and projectId is defined, we use it by default
     elif (
@@ -96,13 +93,27 @@ def get_connection_url(connection: BigQueryConnection) -> str:
         if isinstance(  # pylint: disable=no-else-return
             connection.credentials.gcpConfig.projectId, SingleProjectId
         ):
-            return f"{connection.scheme.value}://{connection.billingProjectId or connection.credentials.gcpConfig.projectId.root}"
+            return f"{connection.scheme.value}://{connection.credentials.gcpConfig.projectId.root}"
+
+        elif isinstance(connection.credentials.gcpConfig.projectId, MultipleProjectId):
+            for project_id in connection.credentials.gcpConfig.projectId.root:
+                return f"{connection.scheme.value}://{project_id}"
+
+    # If gcpConfig is the GCP ADC and projectId is defined, we use it by default
+    elif (
+        isinstance(connection.credentials.gcpConfig, GcpADC)
+        and connection.credentials.gcpConfig.projectId
+    ):
+        if isinstance(  # pylint: disable=no-else-return
+            connection.credentials.gcpConfig.projectId, SingleProjectId
+        ):
+            return f"{connection.scheme.value}://{connection.credentials.gcpConfig.projectId.root}"
 
         elif isinstance(connection.credentials.gcpConfig.projectId, MultipleProjectId):
             for project_id in connection.credentials.gcpConfig.projectId.root:
-                return f"{connection.scheme.value}://{connection.billingProjectId or project_id}"
+                return f"{connection.scheme.value}://{project_id}"
 
-    return f"{connection.scheme.value}://{connection.billingProjectId or ''}"
+    return f"{connection.scheme.value}://"
 
 
 def get_connection(connection: BigQueryConnection) -> Engine:
@@ -110,10 +121,15 @@ def get_connection(connection: BigQueryConnection) -> Engine:
     Prepare the engine and the GCP credentials
     """
     set_google_credentials(gcp_credentials=connection.credentials)
+    kwargs = {}
+    if connection.billingProjectId:
+        kwargs["billing_project_id"] = connection.billingProjectId
+
     return create_generic_db_connection(
         connection=connection,
         get_connection_url_fn=get_connection_url,
         get_connection_args_fn=get_connection_args_common,
+        **kwargs,
     )
 
 
@@ -137,6 +153,10 @@ def test_connection(
             return policy_tags
 
     def test_tags():
+        if not service_connection.includePolicyTags:
+            logger.info("'includePolicyTags' is set to false, so skipping this test.")
+            return None
+
         taxonomy_project_ids = []
         if engine.url.host:
             taxonomy_project_ids.append(engine.url.host)

metadata/ingestion/source/database/bigquery/lineage.py

@@ -11,8 +11,6 @@
 """
 Handle big query lineage extraction
 """
-from typing import Dict, List
-
 from metadata.ingestion.source.database.bigquery.queries import (
     BIGQUERY_GET_STORED_PROCEDURE_QUERIES,
     BIGQUERY_STATEMENT,
@@ -22,7 +20,6 @@ from metadata.ingestion.source.database.bigquery.query_parser import (
 )
 from metadata.ingestion.source.database.lineage_source import LineageSource
 from metadata.ingestion.source.database.stored_procedures_mixin import (
-    QueryByProcedure,
     StoredProcedureLineageMixin,
 )
 from metadata.utils.helpers import get_start_and_end
@@ -42,20 +39,18 @@ class BigqueryLineageSource(
         AND (
             statement_type IN ("MERGE", "CREATE_TABLE_AS_SELECT", "UPDATE") 
             OR (statement_type = "INSERT" and UPPER(query) like '%%INSERT%%INTO%%SELECT%%')
+            OR (statement_type = "CREATE_TABLE" and UPPER(query) like '%%CLONE%%')
         )
     """
 
-    def get_stored_procedure_queries_dict(self) -> Dict[str, List[QueryByProcedure]]:
+    def get_stored_procedure_sql_statement(self) -> str:
         """
-        Pick the stored procedure name from the context
-        and return the list of associated queries
+        Return the SQL statement to get the stored procedure queries
         """
         start, _ = get_start_and_end(self.source_config.queryLogDuration)
         query = BIGQUERY_GET_STORED_PROCEDURE_QUERIES.format(
             start_date=start,
             region=self.service_connection.usageLocation,
         )
-        queries_dict = self.procedure_queries_dict(
-            query=query,
-        )
-        return queries_dict
+
+        return query

metadata/ingestion/source/database/bigquery/metadata.py

@@ -187,34 +187,12 @@ def get_columns(bq_schema):
             "max_length": field.max_length,
             "system_data_type": get_system_data_type(col_type),
             "is_complex": is_complex_type(str(col_type)),
-            "policy_tags": None,
+            "policy_tags": field.policy_tags,
         }
         if getattr(field, "fields", None):
             # Nested Columns available
             col_obj["children"] = get_columns(field.fields)
-        try:
-            if field.policy_tags:
-                policy_tag_name = field.policy_tags.names[0]
-                taxonomy_name = (
-                    policy_tag_name.split("/policyTags/")[0] if policy_tag_name else ""
-                )
-                if not taxonomy_name:
-                    raise NotImplementedError(
-                        f"Taxonomy Name not present for {field.name}"
-                    )
-                col_obj["taxonomy"] = (
-                    PolicyTagManagerClient()
-                    .get_taxonomy(name=taxonomy_name)
-                    .display_name
-                )
-                col_obj["policy_tags"] = (
-                    PolicyTagManagerClient()
-                    .get_policy_tag(name=policy_tag_name)
-                    .display_name
-                )
-        except Exception as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(f"Skipping Policy Tag: {exc}")
+
         col_list.append(col_obj)
     return col_list
 
@@ -440,14 +418,6 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
         logic on how to handle table types, e.g., material views,...
         """
 
-        if self.incremental.enabled:
-            view_names = [
-                view_name
-                for view_name in view_names
-                if view_name
-                in self.incremental_table_processor.get_not_deleted(schema_name)
-            ]
-
         table_name_and_types = []
         for table_type, query in {
             TableType.View: BIGQUERY_GET_VIEW_NAMES,
@@ -513,6 +483,13 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
                         metadata=self.metadata,
                         system_tags=True,
                     )
+
+            if not self.service_connection.includePolicyTags:
+                logger.info(
+                    "'includePolicyTags' is set to false so skipping policy tag ingestion"
+                )
+                return
+
             # Fetching policy tags on the column level
             list_project_ids = [self.context.get().database]
             if not self.service_connection.taxonomyProjectID:
@@ -712,6 +689,32 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
                     table_tag_labels.append(tag_label)
         return table_tag_labels
 
+    def get_policy_tags_for_column(self, column: dict) -> dict:
+        try:
+            if column.get("policy_tags"):
+                policy_tag_name = column["policy_tags"].names[0]
+                taxonomy_name = (
+                    policy_tag_name.split("/policyTags/")[0] if policy_tag_name else ""
+                )
+                if not taxonomy_name:
+                    raise NotImplementedError(
+                        f"Taxonomy Name not present for {column['name']}"
+                    )
+                column["taxonomy"] = (
+                    PolicyTagManagerClient()
+                    .get_taxonomy(name=taxonomy_name)
+                    .display_name
+                )
+                column["policy_tags"] = (
+                    PolicyTagManagerClient()
+                    .get_policy_tag(name=policy_tag_name)
+                    .display_name
+                )
+                return column
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Skipping Policy Tag: {exc}")
+
     def get_column_tag_labels(
         self, table_name: str, column: dict
     ) -> Optional[List[TagLabel]]:
@@ -719,12 +722,14 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
         This will only get executed if the tags context
         is properly informed
         """
-        if column.get("policy_tags"):
+        if self.service_connection.includePolicyTags and column.get("policy_tags"):
+            self.get_policy_tags_for_column(column)
             return fetch_tag_labels_om(
                 metadata=self.metadata,
                 tags=[column["policy_tags"]],
                 classification_name=column["taxonomy"],
-                include_tags=self.source_config.includeTags,
+                include_tags=self.source_config.includeTags
+                and self.service_connection.includePolicyTags,
             )
         return None
 

metadata/ingestion/source/database/bigquery/queries.py

@@ -211,13 +211,18 @@ class BigQueryQueryResult(BaseModel):
         usage_location: str,
         dataset_id: str,
         project_id: str,
+        billing_project_id: Optional[str] = None,
     ):
+        # Use billing project for the INFORMATION_SCHEMA query if provided
+        query_project_id = billing_project_id or project_id
+
         rows = session.execute(
             text(
                 JOBS.format(
                     usage_location=usage_location,
                     dataset_id=dataset_id,
                     project_id=project_id,
+                    query_project_id=query_project_id,
                     insert=DatabaseDMLOperations.INSERT.value,
                     update=DatabaseDMLOperations.UPDATE.value,
                     delete=DatabaseDMLOperations.DELETE.value,
@@ -240,7 +245,7 @@ JOBS = """
         dml_statistics.deleted_row_count as deleted_row_count,
         dml_statistics.updated_row_count as updated_row_count
     FROM
-        `region-{usage_location}`.INFORMATION_SCHEMA.JOBS
+        `{query_project_id}`.`region-{usage_location}`.INFORMATION_SCHEMA.JOBS
     WHERE
         DATE(creation_time) >= CURRENT_DATE() - 1 AND
         destination_table.dataset_id = '{dataset_id}' AND

metadata/ingestion/source/database/column_type_parser.py

@@ -293,7 +293,7 @@ class ColumnTypeParser:
         "STRG": "STRING",
         "RSTR": "STRING",
         # azuresql
-        "HIERARCHYID": "UNKNOWN",
+        "HIERARCHYID": "HIERARCHYID",
     }
 
     _COMPLEX_TYPE = re.compile("^(struct|map|array|uniontype)")

metadata/ingestion/source/database/common/data_diff/databricks_base.py

@@ -0,0 +1,42 @@
+"""Base class for Databricks-based data diff implementations"""
+
+from typing import Optional, Union
+
+from metadata.data_quality.validations.runtime_param_setter.base_diff_params_setter import (
+    BaseTableParameter,
+)
+
+
+class DatabricksBaseTableParameter(BaseTableParameter):
+    """Base class for Databricks-based table parameter setters"""
+
+    @classmethod
+    def _get_service_connection_config(
+        cls,
+        service_connection_config,
+    ) -> Optional[Union[str, dict]]:
+        """Build connection URL for Databricks-based connections"""
+        if not service_connection_config:
+            return None
+
+        scheme = getattr(service_connection_config, "scheme", "databricks+connector")
+        # Handle enum values properly
+        if hasattr(scheme, "value"):
+            scheme = scheme.value
+
+        host_port = getattr(service_connection_config, "hostPort", "localhost:443")
+        token = getattr(service_connection_config, "token", "")
+        token_value = (
+            token.get_secret_value()
+            if hasattr(token, "get_secret_value")
+            else str(token)
+        )
+
+        # Include httpPath if available (required for data_diff library)
+        http_path = getattr(service_connection_config, "httpPath", "")
+        if http_path:
+            # Ensure http_path starts with /
+            if not http_path.startswith("/"):
+                http_path = "/" + http_path
+            return f"{scheme}://:{token_value}@{host_port}{http_path}"
+        return f"{scheme}://:{token_value}@{host_port}"

metadata/ingestion/source/database/common_pg_mappings.py

@@ -45,5 +45,12 @@ ischema_names.update(
         "txid_snapshot": create_sqlalchemy_type("TXID_SNAPSHOT"),
         "xid": SqlAlchemyString,
         "xml": create_sqlalchemy_type("XML"),
+        # PostgreSQL range types (used by TimescaleDB for chunk boundaries)
+        "int4range": create_sqlalchemy_type("INT4RANGE"),
+        "int8range": create_sqlalchemy_type("INT8RANGE"),
+        "numrange": create_sqlalchemy_type("NUMRANGE"),
+        "tsrange": create_sqlalchemy_type("TSRANGE"),
+        "tstzrange": create_sqlalchemy_type("TSTZRANGE"),
+        "daterange": create_sqlalchemy_type("DATERANGE"),
     }
 )