openmetadata-ingestion 1.9.12.0__py3-none-any.whl → 1.10.0.0__py3-none-any.whl

metadata/ingestion/ometa/mixins/patch_mixin.py

@@ -20,6 +20,9 @@ from typing import Dict, List, Optional, Type, TypeVar, Union
 
 from pydantic import BaseModel
 
+from metadata.generated.schema.entity.automations.response.queryRunnerResponse import (
+    QueryRunnerResponse,
+)
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
@@ -128,6 +131,11 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
         Given an Entity type and Source entity and Destination entity,
         generate a JSON Patch and apply it.
 
+        This method provides fine-grained control over entity updates and can
+        override fields that create_or_update() cannot due to server-side restrictions
+        across various entity types. Use override_metadata=True to force updates of
+        protected metadata fields.
+
         Args
             entity (T): Entity Type
             source: Source payload which is current state of the source in OpenMetadata
@@ -135,7 +143,8 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
             allowed_fields: List of field names to filter from source and destination models
             restrict_update_fields: List of field names which will only support add operation
             array_entity_fields: List of array fields to sort for consistent patching
-            override_metadata: Whether to override existing metadata fields
+            override_metadata: Whether to override existing metadata fields. Set to True
+                to force updates of protected fields across various entity types.
             skip_on_failure: Whether to skip the patch operation on failure (default: True)
 
         Returns
@@ -188,12 +197,16 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
         """
         Given an Entity type and ID, JSON PATCH the description.
 
+        This method is useful when you need to update descriptions that cannot be
+        overridden through create_or_update() due to server-side business rules
+        across various entity types. Use force=True to override existing descriptions.
+
         Args
             entity (T): Entity Type
             source: source entity object
             description: new description to add
             force: if True, we will patch any existing description. Otherwise, we will maintain
-                the existing data.
+                the existing data. Set to True to override existing descriptions across entity types.
             skip_on_failure: if True, return None on failure instead of raising exception
         Returns
             Updated Entity
@@ -547,7 +560,9 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
     def patch_automation_workflow_response(
         self,
         automation_workflow: AutomationWorkflow,
-        result: Union[TestConnectionResult, ReverseIngestionResponse],
+        result: Union[
+            TestConnectionResult, ReverseIngestionResponse, QueryRunnerResponse
+        ],
         workflow_status: WorkflowStatus,
     ) -> None:
         """
@@ -560,17 +575,16 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
         }
 
         # for deserializing into json convert enum object to string
-        if isinstance(result, TestConnectionResult):
-            result_data[PatchField.VALUE]["status"] = result_data[PatchField.VALUE][
-                "status"
-            ].value
-        else:
+        if isinstance(result, ReverseIngestionResponse):
             # Convert UUID in string
             data = result_data[PatchField.VALUE]
             data["serviceId"] = str(data["serviceId"])
             for operation_result in data["results"]:
                 operation_result["id"] = str(operation_result["id"])
-
+        else:
+            result_data[PatchField.VALUE]["status"] = result_data[PatchField.VALUE][
+                "status"
+            ].value
         status_data: Dict = {
             PatchField.PATH: PatchPath.STATUS,
             PatchField.OPERATION: PatchOperation.ADD,

metadata/ingestion/ometa/ometa_api.py

@@ -33,6 +33,7 @@ from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.models.custom_pydantic import BaseModel
 from metadata.ingestion.ometa.auth_provider import OpenMetadataAuthenticationProvider
 from metadata.ingestion.ometa.client import REST, APIError, ClientConfig
+from metadata.ingestion.ometa.mixins.csv_mixin import CSVMixin
 from metadata.ingestion.ometa.mixins.custom_property_mixin import (
     OMetaCustomPropertyMixin,
 )
@@ -44,6 +45,7 @@ from metadata.ingestion.ometa.mixins.es_mixin import ESMixin
 from metadata.ingestion.ometa.mixins.ingestion_pipeline_mixin import (
     OMetaIngestionPipelineMixin,
 )
+from metadata.ingestion.ometa.mixins.logs_mixin import OMetaLogsMixin
 from metadata.ingestion.ometa.mixins.mlmodel_mixin import OMetaMlModelMixin
 from metadata.ingestion.ometa.mixins.patch_mixin import OMetaPatchMixin
 from metadata.ingestion.ometa.mixins.pipeline_mixin import OMetaPipelineMixin
@@ -109,6 +111,7 @@ class OpenMetadataSettings(BaseSettings):
 
 
 class OpenMetadata(
+    CSVMixin,
     OMetaPipelineMixin,
     OMetaMlModelMixin,
     OMetaTableMixin,
@@ -123,6 +126,7 @@ class OpenMetadata(
     OMetaTestsMixin,
     DataInsightMixin,
     OMetaIngestionPipelineMixin,
+    OMetaLogsMixin,
     OMetaUserMixin,
     OMetaQueryMixin,
     OMetaRolePolicyMixin,
@@ -339,7 +343,22 @@ class OpenMetadata(
         return entity_class(**resp)
 
     def create_or_update(self, data: C) -> T:
-        """Run a PUT requesting via create request C"""
+        """
+        Run a PUT request via create request C.
+
+        Note: This method uses PUT operations with server-side business rules that may prevent
+        certain field overwrites across various entity types for data integrity reasons. If you
+        need to override existing metadata fields, consider using patch methods instead:
+
+        - For descriptions: Use patch_description(force=True)
+        - For general metadata: Use patch(override_metadata=True)
+
+        Args:
+            data: Create request object
+
+        Returns:
+            Updated or created entity
+        """
         return self._create(data=data, method="put")
 
     def create(self, data: C) -> T:

metadata/ingestion/ometa/routes.py

@@ -328,5 +328,8 @@ ROUTES = {
     # Data Contracts
     DataContract.__name__: "/dataContracts",
     CreateDataContractRequest.__name__: "/dataContracts",
+    # Drives
+    DriveService.__name__: "/services/driveServices",
+    CreateDriveServiceRequest.__name__: "/services/driveServices",
     EntityProfile.__name__: "/entity/profiles",
 }

metadata/ingestion/source/dashboard/domodashboard/metadata.py

@@ -79,6 +79,8 @@ class DomodashboardSource(DashboardServiceSource):
         return cls(config, metadata)
 
     def get_dashboards_list(self) -> Optional[List[DomoDashboardDetails]]:
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         dashboards = self.client.domo.page_list()
         dashboard_list = []
         for dashboard in dashboards:
@@ -104,15 +106,23 @@ class DomodashboardSource(DashboardServiceSource):
     def get_owner_ref(
         self, dashboard_details: DomoDashboardDetails
     ) -> Optional[EntityReferenceList]:
-        for owner in dashboard_details.owners or []:
-            try:
-                owner_details = self.client.domo.users_get(owner.id)
-                if owner_details.get("email"):
-                    return self.metadata.get_reference_by_email(owner_details["email"])
-            except Exception as exc:
-                logger.warning(
-                    f"Error while getting details of user {owner.displayName} - {exc}"
-                )
+        try:
+            if not self.source_config.includeOwners:
+                return None
+            for owner in dashboard_details.owners or []:
+                try:
+                    owner_details = self.client.domo.users_get(owner.id)
+                    if owner_details.get("email"):
+                        return self.metadata.get_reference_by_email(
+                            owner_details["email"]
+                        )
+                except Exception as exc:
+                    logger.warning(
+                        f"Error while getting details of user {owner.displayName} - {exc}"
+                    )
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not fetch owner data due to {err}")
         return None
 
     def yield_dashboard(

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -922,6 +922,8 @@ class LookerSource(DashboardServiceSource):
         """
         Get List of all dashboards
         """
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         try:
             return list(
                 self.client.all_dashboards(fields=",".join(LIST_DASHBOARD_FIELDS))
@@ -963,6 +965,8 @@ class LookerSource(DashboardServiceSource):
             Optional[EntityReference]
         """
         try:
+            if not self.source_config.includeOwners:
+                return None
             if dashboard_details.user_id is not None:
                 dashboard_owner = self.client.user(dashboard_details.user_id)
                 if dashboard_owner.email:

metadata/ingestion/source/dashboard/metabase/client.py

@@ -48,6 +48,7 @@ PASSWORD_HEADER = "password"
 SESSION_HEADERS = {"Content-Type": "application/json", "Accept": "*/*"}
 DEFAULT_TIMEOUT = 30
 METABASE_SESSION_HEADER = "X-Metabase-Session"
+METABASE_API_HEADER = "X-API-KEY"
 API_VERSION = "api"
 
 
@@ -56,8 +57,13 @@ class MetabaseClient:
     Client Handling API communication with Metabase
     """
 
-    def _get_metabase_session(self) -> str:
+    def _get_metabase_session(self) -> Optional[str]:
         try:
+            # If API token is provided, return None as we don't need a session
+            if self.config.apiKey:
+                return None
+
+            # Otherwise use username/password authentication
             params = {USERNAME_HEADER: self.config.username}
             if self.config.password:
                 params[PASSWORD_HEADER] = self.config.password.get_secret_value()
@@ -82,14 +88,30 @@ class MetabaseClient:
         config: MetabaseConnection,
     ):
         self.config = config
-        session_token = self._get_metabase_session()
-        client_config: ClientConfig = ClientConfig(
-            base_url=clean_uri(str(self.config.hostPort)),
-            api_version=API_VERSION,
-            auth_header=AUTHORIZATION_HEADER,
-            auth_token=lambda: (NO_ACCESS_TOKEN, 0),
-            extra_headers={METABASE_SESSION_HEADER: session_token},
-        )
+
+        # Determine authentication method and set headers accordingly
+        if self.config.apiKey:
+            # Use API token authentication
+            client_config: ClientConfig = ClientConfig(
+                base_url=clean_uri(str(self.config.hostPort)),
+                api_version=API_VERSION,
+                auth_header=AUTHORIZATION_HEADER,
+                auth_token=lambda: (NO_ACCESS_TOKEN, 0),
+                extra_headers={
+                    METABASE_API_HEADER: self.config.apiKey.get_secret_value()
+                },
+            )
+        else:
+            # Use session-based authentication
+            session_token = self._get_metabase_session()
+            client_config: ClientConfig = ClientConfig(
+                base_url=clean_uri(str(self.config.hostPort)),
+                api_version=API_VERSION,
+                auth_header=AUTHORIZATION_HEADER,
+                auth_token=lambda: (NO_ACCESS_TOKEN, 0),
+                extra_headers={METABASE_SESSION_HEADER: session_token},
+            )
+
         self.client = REST(client_config)
 
     def get_dashboards_list(

metadata/ingestion/source/dashboard/metabase/metadata.py

@@ -108,6 +108,8 @@ class MetabaseSource(DashboardServiceSource):
         """
         Get List of all dashboards
         """
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         self.dashboards_list = self.client.get_dashboards_list(self.collections)
         return self.dashboards_list
 
@@ -180,6 +182,8 @@ class MetabaseSource(DashboardServiceSource):
         Get dashboard owner from email
         """
         try:
+            if not self.source_config.includeOwners:
+                return None
             if dashboard_details.creator_id:
                 owner_details = self.client.get_user_details(
                     dashboard_details.creator_id

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -302,6 +302,24 @@ class PowerbiSource(DashboardServiceSource):
             f"{workspace_id}/reports/{dashboard_id}/ReportSection?experience=power-bi"
         )
 
+    def _get_dataset_url(self, workspace_id: str, dataset_id: str) -> str:
+        """
+        Method to build the dataset url
+        """
+        return (
+            f"{clean_uri(self.service_connection.hostPort)}/groups/"
+            f"{workspace_id}/datasets/{dataset_id}?experience=power-bi"
+        )
+
+    def _get_dataflow_url(self, workspace_id: str, dataflow_id: str) -> str:
+        """
+        Method to build the dataset url
+        """
+        return (
+            f"{clean_uri(self.service_connection.hostPort)}/groups/"
+            f"{workspace_id}/dataflows/{dataflow_id}?experience=power-bi"
+        )
+
     def _get_chart_url(
         self, report_id: Optional[str], workspace_id: str, dashboard_id: str
     ) -> str:
@@ -545,9 +563,17 @@ class PowerbiSource(DashboardServiceSource):
                     if isinstance(dataset, Dataset):
                         data_model_type = DataModelType.PowerBIDataModel.value
                         datamodel_columns = self._get_column_info(dataset)
+                        source_url = self._get_dataset_url(
+                            workspace_id=self.context.get().workspace.id,
+                            dataset_id=dataset.id,
+                        )
                     elif isinstance(dataset, Dataflow):
                         data_model_type = DataModelType.PowerBIDataFlow.value
                         datamodel_columns = []
+                        source_url = self._get_dataflow_url(
+                            workspace_id=self.context.get().workspace.id,
+                            dataflow_id=dataset.id,
+                        )
                     else:
                         logger.warning(
                             f"Unknown dataset type: {type(dataset)}, name: {dataset.name}"
@@ -569,6 +595,7 @@ class PowerbiSource(DashboardServiceSource):
                         columns=datamodel_columns,
                         project=self.get_project_name(dashboard_details=dataset),
                         owners=self.get_owner_ref(dashboard_details=dataset),
+                        sourceUrl=SourceUrl(source_url),
                     )
                     yield Either(right=data_model_request)
                     self.register_record_datamodel(datamodel_request=data_model_request)

metadata/ingestion/source/dashboard/redash/metadata.py

@@ -110,6 +110,8 @@ class RedashSource(DashboardServiceSource):
         )
 
     def get_dashboards_list(self) -> Optional[List[dict]]:
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         return self.dashboard_list
 
     def get_dashboard_name(self, dashboard: dict) -> str:
@@ -123,6 +125,8 @@ class RedashSource(DashboardServiceSource):
         Get owner from email
         """
         try:
+            if not self.source_config.includeOwners:
+                return None
             if dashboard_details.get("user") and dashboard_details["user"].get("email"):
                 return self.metadata.get_reference_by_email(
                     dashboard_details["user"].get("email")

metadata/ingestion/source/dashboard/sigma/metadata.py

@@ -91,6 +91,8 @@ class SigmaSource(DashboardServiceSource):
         """
         get list of dashboard
         """
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         return self.client.get_dashboards()
 
     def get_dashboard_name(self, dashboard: Workbook) -> Optional[str]:
@@ -367,6 +369,8 @@ class SigmaSource(DashboardServiceSource):
         Get owner from email
         """
         try:
+            if not self.source_config.includeOwners:
+                return None
             if dashboard_details.ownerId:
                 owner = self.client.get_owner_detail(dashboard_details.ownerId)
                 return self.metadata.get_reference_by_email(owner.email)

metadata/ingestion/source/dashboard/superset/api_source.py

@@ -77,6 +77,8 @@ class SupersetAPISource(SupersetSourceMixin):
         """
         Get List of all dashboards
         """
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         current_page = 0
         page_size = 25
         total_dashboards = self.client.fetch_total_dashboards()

metadata/ingestion/source/dashboard/superset/db_source.py

@@ -111,6 +111,8 @@ class SupersetDBSource(SupersetSourceMixin):
         """
         Get List of all dashboards
         """
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         query = (
             FETCH_DASHBOARDS
             if self.source_config.includeDraftDashboard

metadata/ingestion/source/dashboard/superset/mixin.py

@@ -117,6 +117,8 @@ class SupersetSourceMixin(DashboardServiceSource):
         self, dashboard_details: Union[DashboardResult, FetchDashboard]
     ) -> Optional[EntityReferenceList]:
         try:
+            if not self.source_config.includeOwners:
+                return None
             if hasattr(dashboard_details, "owners"):
                 for owner in dashboard_details.owners or []:
                     if owner.email:

metadata/ingestion/source/dashboard/tableau/client.py

@@ -106,8 +106,15 @@ class TableauClient:
     def site_id(self) -> str:
         return self.tableau_server.site_id
 
-    def get_tableau_owner(self, owner_id: str) -> Optional[TableauOwner]:
+    def get_tableau_owner(
+        self, owner_id: str, include_owners: bool = True
+    ) -> Optional[TableauOwner]:
+        """
+        Get tableau owner with optional include_owners flag
+        """
         try:
+            if not include_owners:
+                return None
             if owner_id in self.owner_cache:
                 return self.owner_cache[owner_id]
             owner = self.tableau_server.users.get_by_id(owner_id) if owner_id else None
@@ -122,7 +129,7 @@ class TableauClient:
         return None
 
     def get_workbook_charts_and_user_count(
-        self, views: List[ViewItem]
+        self, views: List[ViewItem], include_owners: bool = True
     ) -> Optional[Tuple[Optional[int], Optional[List[TableauChart]]]]:
         """
         Fetches workbook charts and dashboard user view count
@@ -136,7 +143,7 @@ class TableauClient:
                         id=str(view.id),
                         name=view.name,
                         tags=view.tags,
-                        owner=self.get_tableau_owner(view.owner_id),
+                        owner=self.get_tableau_owner(view.owner_id, include_owners),
                         contentUrl=view.content_url,
                         sheetType=view.sheet_type,
                     )
@@ -202,7 +209,7 @@ class TableauClient:
             logger.debug(f"Failed to get project parents by id: {str(e)}")
         return None
 
-    def get_workbooks(self) -> Iterable[TableauDashboard]:
+    def get_workbooks(self, include_owners: bool = True) -> Iterable[TableauDashboard]:
         """
         Fetch all tableau workbooks
         """
@@ -212,7 +219,7 @@ class TableauClient:
             try:
                 self.tableau_server.workbooks.populate_views(workbook, usage=True)
                 charts, user_views = self.get_workbook_charts_and_user_count(
-                    workbook.views
+                    workbook.views, include_owners
                 )
                 workbook = TableauDashboard(
                     id=str(workbook.id),
@@ -220,7 +227,7 @@ class TableauClient:
                     project=TableauBaseModel(
                         id=str(workbook.project_id), name=workbook.project_name
                     ),
-                    owner=self.get_tableau_owner(workbook.owner_id),
+                    owner=self.get_tableau_owner(workbook.owner_id, include_owners),
                     description=workbook.description,
                     tags=workbook.tags,
                     webpageUrl=workbook.webpage_url,
@@ -248,9 +255,11 @@ class TableauClient:
             "Please check if the user has permissions to access the Dashboards information"
         )
 
-    def test_get_workbook_views(self):
+    def test_get_workbook_views(self, include_owners: bool = True):
         workbook = self.test_get_workbooks()
-        charts, _ = self.get_workbook_charts_and_user_count(workbook.views)
+        charts, _ = self.get_workbook_charts_and_user_count(
+            workbook.views, include_owners
+        )
         if charts:
             return True
         raise TableauChartsException(
@@ -258,9 +267,11 @@ class TableauClient:
             "Please check if the user has permissions to access the Charts information"
         )
 
-    def test_get_owners(self) -> Optional[List[TableauOwner]]:
+    def test_get_owners(
+        self, include_owners: bool = True
+    ) -> Optional[List[TableauOwner]]:
         workbook = self.test_get_workbooks()
-        owners = self.get_tableau_owner(workbook.owner_id)
+        owners = self.get_tableau_owner(workbook.owner_id, include_owners)
         if owners is not None:
             return owners
         raise TableauOwnersNotFound(

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -132,7 +132,11 @@ class TableauSource(DashboardServiceSource):
         return cls(config, metadata)
 
     def get_dashboards_list(self) -> Iterable[TableauDashboard]:
-        yield from self.client.get_workbooks()
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
+        yield from self.client.get_workbooks(
+            include_owners=self.source_config.includeOwners
+        )
 
     def get_dashboard_name(self, dashboard: TableauDashboard) -> str:
         return dashboard.name
@@ -151,6 +155,8 @@ class TableauSource(DashboardServiceSource):
         Get dashboard owner from email
         """
         try:
+            if not self.source_config.includeOwners:
+                return None
             if dashboard_details.owner and dashboard_details.owner.email:
                 return self.metadata.get_reference_by_email(
                     dashboard_details.owner.email

metadata/ingestion/source/database/bigquery/connection.py

@@ -153,6 +153,10 @@ def test_connection(
             return policy_tags
 
     def test_tags():
+        if not service_connection.includePolicyTags:
+            logger.info("'includePolicyTags' is set to false, so skipping this test.")
+            return None
+
         taxonomy_project_ids = []
         if engine.url.host:
             taxonomy_project_ids.append(engine.url.host)

metadata/ingestion/source/database/bigquery/metadata.py

@@ -187,34 +187,12 @@ def get_columns(bq_schema):
             "max_length": field.max_length,
             "system_data_type": get_system_data_type(col_type),
             "is_complex": is_complex_type(str(col_type)),
-            "policy_tags": None,
+            "policy_tags": field.policy_tags,
         }
         if getattr(field, "fields", None):
             # Nested Columns available
             col_obj["children"] = get_columns(field.fields)
-        try:
-            if field.policy_tags:
-                policy_tag_name = field.policy_tags.names[0]
-                taxonomy_name = (
-                    policy_tag_name.split("/policyTags/")[0] if policy_tag_name else ""
-                )
-                if not taxonomy_name:
-                    raise NotImplementedError(
-                        f"Taxonomy Name not present for {field.name}"
-                    )
-                col_obj["taxonomy"] = (
-                    PolicyTagManagerClient()
-                    .get_taxonomy(name=taxonomy_name)
-                    .display_name
-                )
-                col_obj["policy_tags"] = (
-                    PolicyTagManagerClient()
-                    .get_policy_tag(name=policy_tag_name)
-                    .display_name
-                )
-        except Exception as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(f"Skipping Policy Tag: {exc}")
+
         col_list.append(col_obj)
     return col_list
 
@@ -513,6 +491,13 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
                         metadata=self.metadata,
                         system_tags=True,
                     )
+
+            if not self.service_connection.includePolicyTags:
+                logger.info(
+                    "'includePolicyTags' is set to false so skipping policy tag ingestion"
+                )
+                return
+
             # Fetching policy tags on the column level
             list_project_ids = [self.context.get().database]
             if not self.service_connection.taxonomyProjectID:
@@ -712,6 +697,32 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
                     table_tag_labels.append(tag_label)
         return table_tag_labels
 
+    def get_policy_tags_for_column(self, column: dict) -> dict:
+        try:
+            if column.get("policy_tags"):
+                policy_tag_name = column["policy_tags"].names[0]
+                taxonomy_name = (
+                    policy_tag_name.split("/policyTags/")[0] if policy_tag_name else ""
+                )
+                if not taxonomy_name:
+                    raise NotImplementedError(
+                        f"Taxonomy Name not present for {column['name']}"
+                    )
+                column["taxonomy"] = (
+                    PolicyTagManagerClient()
+                    .get_taxonomy(name=taxonomy_name)
+                    .display_name
+                )
+                column["policy_tags"] = (
+                    PolicyTagManagerClient()
+                    .get_policy_tag(name=policy_tag_name)
+                    .display_name
+                )
+                return column
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Skipping Policy Tag: {exc}")
+
     def get_column_tag_labels(
         self, table_name: str, column: dict
     ) -> Optional[List[TagLabel]]:
@@ -719,12 +730,14 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
         This will only get executed if the tags context
         is properly informed
         """
-        if column.get("policy_tags"):
+        if self.service_connection.includePolicyTags and column.get("policy_tags"):
+            self.get_policy_tags_for_column(column)
             return fetch_tag_labels_om(
                 metadata=self.metadata,
                 tags=[column["policy_tags"]],
                 classification_name=column["taxonomy"],
-                include_tags=self.source_config.includeTags,
+                include_tags=self.source_config.includeTags
+                and self.service_connection.includePolicyTags,
             )
         return None
 

metadata/ingestion/source/database/column_type_parser.py

@@ -293,7 +293,7 @@ class ColumnTypeParser:
         "STRG": "STRING",
         "RSTR": "STRING",
         # azuresql
-        "HIERARCHYID": "UNKNOWN",
+        "HIERARCHYID": "HIERARCHYID",
     }
 
     _COMPLEX_TYPE = re.compile("^(struct|map|array|uniontype)")

metadata/ingestion/source/database/common_pg_mappings.py

@@ -45,5 +45,12 @@ ischema_names.update(
         "txid_snapshot": create_sqlalchemy_type("TXID_SNAPSHOT"),
         "xid": SqlAlchemyString,
         "xml": create_sqlalchemy_type("XML"),
+        # PostgreSQL range types (used by TimescaleDB for chunk boundaries)
+        "int4range": create_sqlalchemy_type("INT4RANGE"),
+        "int8range": create_sqlalchemy_type("INT8RANGE"),
+        "numrange": create_sqlalchemy_type("NUMRANGE"),
+        "tsrange": create_sqlalchemy_type("TSRANGE"),
+        "tstzrange": create_sqlalchemy_type("TSTZRANGE"),
+        "daterange": create_sqlalchemy_type("DATERANGE"),
     }
 )