acryl-datahub 1.2.0.10rc1__py3-none-any.whl → 1.2.0.10rc3__py3-none-any.whl

datahub/ingestion/source/looker/lookml_source.py

@@ -4,7 +4,7 @@ import tempfile
 from collections import OrderedDict
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from typing import Dict, Iterable, List, Optional, Set, Tuple
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
 
 import lkml
 import lkml.simple
@@ -12,8 +12,7 @@ from looker_sdk.error import SDKError
 
 from datahub.configuration.git import GitInfo
 from datahub.emitter.mce_builder import make_schema_field_urn
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.emitter.mcp_builder import gen_containers
+from datahub.emitter.mcp_builder import mcps_from_mce
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
@@ -77,7 +76,7 @@ from datahub.ingestion.source.state.stale_entity_removal_handler import (
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionSourceBase,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.common import BrowsePaths, Status
+from datahub.metadata.com.linkedin.pegasus2avro.common import Status
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     DatasetLineageTypeClass,
     FineGrainedLineageDownstreamType,
@@ -85,18 +84,15 @@ from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     UpstreamLineage,
     ViewProperties,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
-from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
 from datahub.metadata.schema_classes import (
     AuditStampClass,
-    BrowsePathEntryClass,
-    BrowsePathsV2Class,
-    ContainerClass,
     DatasetPropertiesClass,
     FineGrainedLineageClass,
     FineGrainedLineageUpstreamTypeClass,
-    SubTypesClass,
 )
+from datahub.sdk.container import Container
+from datahub.sdk.dataset import Dataset
+from datahub.sdk.entity import Entity
 from datahub.sql_parsing.sqlglot_lineage import ColumnRef
 
 VIEW_LANGUAGE_LOOKML: str = "lookml"
@@ -428,69 +424,40 @@ class LookMLSource(StatefulIngestionSourceBase):
 
         return dataset_props
 
-    def _build_dataset_mcps(
-        self, looker_view: LookerView
-    ) -> List[MetadataChangeProposalWrapper]:
-        view_urn = looker_view.id.get_urn(self.source_config)
-
-        subTypeEvent = MetadataChangeProposalWrapper(
-            entityUrn=view_urn,
-            aspect=SubTypesClass(typeNames=[DatasetSubTypes.VIEW]),
-        )
-        events = [subTypeEvent]
+    def _build_dataset_entities(self, looker_view: LookerView) -> Iterable[Dataset]:
+        dataset_extra_aspects: List[Union[ViewProperties, Status]] = [
+            Status(removed=False)
+        ]
         if looker_view.view_details is not None:
-            viewEvent = MetadataChangeProposalWrapper(
-                entityUrn=view_urn,
-                aspect=looker_view.view_details,
-            )
-            events.append(viewEvent)
-
-        project_key = gen_project_key(self.source_config, looker_view.id.project_name)
-
-        container = ContainerClass(container=project_key.as_urn())
-        events.append(
-            MetadataChangeProposalWrapper(entityUrn=view_urn, aspect=container)
-        )
-
-        events.append(
-            MetadataChangeProposalWrapper(
-                entityUrn=view_urn,
-                aspect=looker_view.id.get_browse_path_v2(self.source_config),
-            )
-        )
-
-        return events
-
-    def _build_dataset_mce(self, looker_view: LookerView) -> MetadataChangeEvent:
-        """
-        Creates MetadataChangeEvent for the dataset, creating upstream lineage links
-        """
-        logger.debug(f"looker_view = {looker_view.id}")
+            dataset_extra_aspects.append(looker_view.view_details)
 
-        dataset_snapshot = DatasetSnapshot(
-            urn=looker_view.id.get_urn(self.source_config),
-            aspects=[],  # we append to this list later on
-        )
-        browse_paths = BrowsePaths(
-            paths=[looker_view.id.get_browse_path(self.source_config)]
-        )
-
-        dataset_snapshot.aspects.append(browse_paths)
-        dataset_snapshot.aspects.append(Status(removed=False))
-        upstream_lineage = self._get_upstream_lineage(looker_view)
-        if upstream_lineage is not None:
-            dataset_snapshot.aspects.append(upstream_lineage)
         schema_metadata = LookerUtil._get_schema(
             self.source_config.platform_name,
             looker_view.id.view_name,
             looker_view.fields,
             self.reporter,
         )
-        if schema_metadata is not None:
-            dataset_snapshot.aspects.append(schema_metadata)
-        dataset_snapshot.aspects.append(self._get_custom_properties(looker_view))
 
-        return MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
+        custom_properties: DatasetPropertiesClass = self._get_custom_properties(
+            looker_view
+        )
+
+        yield Dataset(
+            platform=self.source_config.platform_name,
+            name=looker_view.id.get_view_dataset_name(self.source_config),
+            display_name=looker_view.id.view_name,
+            platform_instance=self.source_config.platform_instance,
+            env=self.source_config.env,
+            subtype=DatasetSubTypes.VIEW,
+            parent_container=looker_view.id.get_view_dataset_parent_container(
+                self.source_config
+            ),
+            schema=schema_metadata,
+            custom_properties=custom_properties.customProperties,
+            external_url=custom_properties.externalUrl,
+            upstreams=self._get_upstream_lineage(looker_view),
+            extra_aspects=dataset_extra_aspects,
+        )
 
     def get_project_name(self, model_name: str) -> str:
         if self.source_config.project_name is not None:
@@ -554,7 +521,7 @@ class LookMLSource(StatefulIngestionSourceBase):
             ).workunit_processor,
         ]
 
-    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+    def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:
         with tempfile.TemporaryDirectory("lookml_tmp") as tmp_dir:
             # Clone the base_folder if necessary.
             if not self.source_config.base_folder:
@@ -715,7 +682,7 @@ class LookMLSource(StatefulIngestionSourceBase):
                 tmp_dir, project, project_visited, manifest_constants
             )
 
-    def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
+    def get_internal_workunits(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:  # noqa: C901
         assert self.source_config.base_folder
         viewfile_loader = LookerViewFileLoader(
             self.source_config.project_name,
@@ -949,7 +916,7 @@ class LookMLSource(StatefulIngestionSourceBase):
                                         maybe_looker_view.id.project_name
                                         not in self.processed_projects
                                     ):
-                                        yield from self.gen_project_workunits(
+                                        yield from self.gen_project_containers(
                                             maybe_looker_view.id.project_name
                                         )
 
@@ -957,15 +924,10 @@ class LookMLSource(StatefulIngestionSourceBase):
                                             maybe_looker_view.id.project_name
                                         )
 
-                                    for mcp in self._build_dataset_mcps(
+                                    yield from self._build_dataset_entities(
                                         maybe_looker_view
-                                    ):
-                                        yield mcp.as_workunit()
-                                    mce = self._build_dataset_mce(maybe_looker_view)
-                                    yield MetadataWorkUnit(
-                                        id=f"lookml-view-{maybe_looker_view.id}",
-                                        mce=mce,
                                     )
+
                                     processed_view_files.add(include.include)
                                 else:
                                     (
@@ -994,28 +956,24 @@ class LookMLSource(StatefulIngestionSourceBase):
             self.source_config.tag_measures_and_dimensions
             and self.reporter.events_produced != 0
         ):
-            # Emit tag MCEs for measures and dimensions:
+            # Emit tag MCEs for measures and dimensions if we produced any explores:
             for tag_mce in LookerUtil.get_tag_mces():
-                yield MetadataWorkUnit(
-                    id=f"tag-{tag_mce.proposedSnapshot.urn}", mce=tag_mce
-                )
+                # Convert MCE to MCPs
+                for mcp in mcps_from_mce(tag_mce):
+                    yield mcp.as_workunit()
 
-    def gen_project_workunits(self, project_name: str) -> Iterable[MetadataWorkUnit]:
+    def gen_project_containers(self, project_name: str) -> Iterable[Container]:
         project_key = gen_project_key(
             self.source_config,
             project_name,
         )
-        yield from gen_containers(
+
+        yield Container(
             container_key=project_key,
-            name=project_name,
-            sub_types=[BIContainerSubTypes.LOOKML_PROJECT],
+            display_name=project_name,
+            subtype=BIContainerSubTypes.LOOKML_PROJECT,
+            parent_container=["Folders"],
         )
-        yield MetadataChangeProposalWrapper(
-            entityUrn=project_key.as_urn(),
-            aspect=BrowsePathsV2Class(
-                path=[BrowsePathEntryClass("Folders")],
-            ),
-        ).as_workunit()
 
     def report_skipped_unreachable_views(
         self,

datahub/ingestion/source/redash.py

@@ -447,7 +447,7 @@ class RedashSource(StatefulIngestionSourceBase):
                 dataset_urns = sql_parser_in_tables.in_tables
                 if sql_parser_in_tables.debug_info.table_error:
                     self.report.queries_problem_parsing.add(str(query_id))
-                    self.error(
+                    self.warn(
                         logger,
                         "sql-parsing",
                         f"exception {sql_parser_in_tables.debug_info.table_error} in parsing query-{query_id}-datasource-{data_source_id}",

datahub/ingestion/source/superset.py

@@ -12,6 +12,8 @@ import sqlglot
 from pydantic import BaseModel
 from pydantic.class_validators import root_validator, validator
 from pydantic.fields import Field
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 import datahub.emitter.mce_builder as builder
 from datahub.configuration.common import AllowDenyPattern
@@ -109,6 +111,12 @@ logger = logging.getLogger(__name__)
 
 PAGE_SIZE = 25
 
+# Retry configuration constants
+RETRY_MAX_TIMES = 3
+RETRY_STATUS_CODES = [429, 500, 502, 503, 504]
+RETRY_BACKOFF_FACTOR = 1
+RETRY_ALLOWED_METHODS = ["GET"]
+
 
 chart_type_from_viz_type = {
     "line": ChartTypeClass.LINE,
@@ -282,6 +290,7 @@ def get_filter_name(filter_obj):
 )
 @capability(SourceCapability.DOMAINS, "Enabled by `domain` config to assign domain_key")
 @capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
+@capability(SourceCapability.TAGS, "Supported by default")
 class SupersetSource(StatefulIngestionSourceBase):
     """
     This plugin extracts the following:
@@ -327,6 +336,19 @@ class SupersetSource(StatefulIngestionSourceBase):
         logger.debug("Got access token from superset")
 
         requests_session = requests.Session()
+
+        # Configure retry strategy for transient failures
+        retry_strategy = Retry(
+            total=RETRY_MAX_TIMES,
+            status_forcelist=RETRY_STATUS_CODES,
+            backoff_factor=RETRY_BACKOFF_FACTOR,
+            allowed_methods=RETRY_ALLOWED_METHODS,
+            raise_on_status=False,
+        )
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        requests_session.mount("http://", adapter)
+        requests_session.mount("https://", adapter)
+
         requests_session.headers.update(
             {
                 "Authorization": f"Bearer {self.access_token}",
@@ -359,8 +381,13 @@ class SupersetSource(StatefulIngestionSourceBase):
             )
 
             if response.status_code != 200:
-                logger.warning(f"Failed to get {entity_type} data: {response.text}")
-                continue
+                self.report.warning(
+                    title="Failed to fetch data from Superset API",
+                    message="Incomplete metadata extraction due to Superset API failure",
+                    context=f"Entity Type: {entity_type}, HTTP Status Code: {response.status_code}, Page: {current_page}. Response: {response.text}",
+                )
+                # we stop pagination for this entity type and we continue the overall ingestion
+                break
 
             payload = response.json()
             # Update total_items with the actual count from the response
@@ -521,6 +548,11 @@ class SupersetSource(StatefulIngestionSourceBase):
         )
         dashboard_snapshot.aspects.append(owners_info)
 
+        superset_tags = self._extract_and_map_tags(dashboard_data.get("tags", []))
+        tags = self._merge_tags_with_existing(dashboard_urn, superset_tags)
+        if tags:
+            dashboard_snapshot.aspects.append(tags)
+
         return dashboard_snapshot
 
     def _process_dashboard(self, dashboard_data: Any) -> Iterable[MetadataWorkUnit]:
@@ -919,6 +951,12 @@ class SupersetSource(StatefulIngestionSourceBase):
             lastModified=last_modified,
         )
         chart_snapshot.aspects.append(owners_info)
+
+        superset_tags = self._extract_and_map_tags(chart_data.get("tags", []))
+        tags = self._merge_tags_with_existing(chart_urn, superset_tags)
+        if tags:
+            chart_snapshot.aspects.append(tags)
+
         yield MetadataWorkUnit(
             id=chart_urn, mce=MetadataChangeEvent(proposedSnapshot=chart_snapshot)
         )
@@ -1288,17 +1326,18 @@ class SupersetSource(StatefulIngestionSourceBase):
             externalUrl=dataset_url,
             lastModified=TimeStamp(time=modified_ts),
         )
-        global_tags = GlobalTagsClass(tags=[TagAssociationClass(tag=tag_urn)])
-
-        aspects_items: List[Any] = []
-        aspects_items.extend(
-            [
-                self.gen_schema_metadata(dataset_response),
-                dataset_info,
-                upstream_lineage,
-                global_tags,
-            ]
-        )
+
+        dataset_tags = GlobalTagsClass(tags=[TagAssociationClass(tag=tag_urn)])
+        tags = self._merge_tags_with_existing(datasource_urn, dataset_tags)
+
+        aspects_items: List[Any] = [
+            self.gen_schema_metadata(dataset_response),
+            dataset_info,
+            upstream_lineage,
+        ]
+
+        if tags:
+            aspects_items.append(tags)
 
         dataset_snapshot = DatasetSnapshot(
             urn=datasource_urn,
@@ -1320,6 +1359,75 @@ class SupersetSource(StatefulIngestionSourceBase):
 
         return dataset_snapshot
 
+    def _extract_and_map_tags(
+        self, raw_tags: List[Dict[str, Any]]
+    ) -> Optional[GlobalTagsClass]:
+        """Extract and map Superset tags to DataHub GlobalTagsClass.
+
+        Filters out system-generated tags (type != 1) and only processes user-defined tags
+        from the Superset API response.
+
+        Args:
+            raw_tags: List of tag dictionaries from Superset API
+
+        Returns:
+            GlobalTagsClass with user-defined tags, or None if no tags found
+        """
+        user_tags = [
+            tag.get("name", "")
+            for tag in raw_tags
+            if tag.get("type") == 1 and tag.get("name")
+        ]
+
+        if not user_tags:
+            return None
+
+        tag_urns = [builder.make_tag_urn(tag) for tag in user_tags]
+        return GlobalTagsClass(
+            tags=[TagAssociationClass(tag=tag_urn) for tag_urn in tag_urns]
+        )
+
+    def _merge_tags_with_existing(
+        self, entity_urn: str, new_tags: Optional[GlobalTagsClass]
+    ) -> Optional[GlobalTagsClass]:
+        """Merge new tags with existing ones from DataHub to preserve manually added tags.
+
+        This method ensures that tags manually added via DataHub UI are not overwritten
+        during ingestion. It fetches existing tags from the graph and merges them with
+        new tags from the source system, avoiding duplicates.
+
+        Args:
+            entity_urn: URN of the entity to check for existing tags
+            new_tags: New tags to add as GlobalTagsClass object
+
+        Returns:
+            GlobalTagsClass with merged tags preserving existing ones, or None if no tags
+        """
+        if not new_tags or not new_tags.tags:
+            return None
+
+        # Fetch existing tags from DataHub
+        existing_global_tags = None
+        if self.ctx.graph:
+            existing_global_tags = self.ctx.graph.get_aspect(
+                entity_urn=entity_urn, aspect_type=GlobalTagsClass
+            )
+
+        # Merge existing tags with new ones, avoiding duplicates
+        all_tags = []
+        existing_tag_urns = set()
+
+        if existing_global_tags and existing_global_tags.tags:
+            all_tags.extend(existing_global_tags.tags)
+            existing_tag_urns = {tag.tag for tag in existing_global_tags.tags}
+
+        # Add new tags that don't already exist
+        for new_tag in new_tags.tags:
+            if new_tag.tag not in existing_tag_urns:
+                all_tags.append(new_tag)
+
+        return GlobalTagsClass(tags=all_tags) if all_tags else None
+
     def _process_dataset(self, dataset_data: Any) -> Iterable[MetadataWorkUnit]:
         dataset_name = ""
         try:

datahub/ingestion/source/tableau/tableau.py

@@ -524,6 +524,10 @@ class TableauConfig(
         default=False,
         description="Ingest Owner from source. This will override Owner info entered from UI",
     )
+    use_email_as_username: bool = Field(
+        default=False,
+        description="Use email address instead of username for entity owners. Requires ingest_owner to be True.",
+    )
     ingest_tables_external: bool = Field(
         default=False,
         description="Ingest details for tables external to (not embedded in) tableau as entities.",
@@ -678,6 +682,14 @@ class TableauConfig(
             raise ValueError(
                 "tags_for_hidden_assets is only allowed with ingest_tags enabled. Be aware that this will overwrite tags entered from the UI."
             )
+
+        use_email_as_username = values.get("use_email_as_username")
+        ingest_owner = values.get("ingest_owner")
+        if use_email_as_username and not ingest_owner:
+            raise ValueError(
+                "use_email_as_username requires ingest_owner to be enabled."
+            )
+
         return values
 
 
@@ -839,6 +851,9 @@ class TableauSourceReport(
         default_factory=(lambda: defaultdict(int))
     )
 
+    # Owner extraction statistics
+    num_email_fallback_to_username: int = 0
+
 
 def report_user_role(report: TableauSourceReport, server: Server) -> None:
     title: str = "Insufficient Permissions"
@@ -2716,13 +2731,12 @@ class TableauSiteSource:
             dataset_snapshot.aspects.append(browse_paths)
 
         # Ownership
-        owner = (
-            self._get_ownership(datasource_info[c.OWNER][c.USERNAME])
-            if datasource_info
-            and datasource_info.get(c.OWNER)
-            and datasource_info[c.OWNER].get(c.USERNAME)
+        owner_identifier = (
+            self._get_owner_identifier(datasource_info[c.OWNER])
+            if datasource_info and datasource_info.get(c.OWNER)
             else None
         )
+        owner = self._get_ownership(owner_identifier) if owner_identifier else None
         if owner is not None:
             dataset_snapshot.aspects.append(owner)
 
@@ -3127,7 +3141,7 @@ class TableauSiteSource:
 
         creator: Optional[str] = None
         if workbook is not None and workbook.get(c.OWNER) is not None:
-            creator = workbook[c.OWNER].get(c.USERNAME)
+            creator = self._get_owner_identifier(workbook[c.OWNER])
         created_at = sheet.get(c.CREATED_AT, datetime.now())
         updated_at = sheet.get(c.UPDATED_AT, datetime.now())
         last_modified = self.get_last_modified(creator, created_at, updated_at)
@@ -3276,7 +3290,7 @@ class TableauSiteSource:
 
     def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUnit]:
         workbook_container_key = self.gen_workbook_key(workbook[c.ID])
-        creator = workbook.get(c.OWNER, {}).get(c.USERNAME)
+        creator = self._get_owner_identifier(workbook.get(c.OWNER, {}))
 
         owner_urn = (
             builder.make_user_urn(creator)
@@ -3458,7 +3472,7 @@ class TableauSiteSource:
 
         creator: Optional[str] = None
         if workbook is not None and workbook.get(c.OWNER) is not None:
-            creator = workbook[c.OWNER].get(c.USERNAME)
+            creator = self._get_owner_identifier(workbook[c.OWNER])
         created_at = dashboard.get(c.CREATED_AT, datetime.now())
         updated_at = dashboard.get(c.UPDATED_AT, datetime.now())
         last_modified = self.get_last_modified(creator, created_at, updated_at)
@@ -3605,6 +3619,20 @@ class TableauSiteSource:
             )
         return last_modified
 
+    def _get_owner_identifier(self, owner_dict: dict) -> Optional[str]:
+        """Extract owner identifier (email or username) based on configuration."""
+        if not owner_dict:
+            return None
+
+        if self.config.use_email_as_username:
+            email = owner_dict.get(c.EMAIL)
+            if email:
+                return email
+            # Fall back to username if email is not available
+            self.report.num_email_fallback_to_username += 1
+
+        return owner_dict.get(c.USERNAME)
+
     @lru_cache(maxsize=None)
     def _get_ownership(self, user: str) -> Optional[OwnershipClass]:
         if self.config.ingest_owner and user:
@@ -3828,3 +3856,15 @@ class TableauSiteSource:
                     self.report.emit_upstream_tables_timer[self.site_content_url] = (
                         timer.elapsed_seconds(digits=2)
                     )
+
+            # Log owner extraction statistics if there were fallbacks
+            if (
+                self.config.use_email_as_username
+                and self.config.ingest_owner
+                and self.report.num_email_fallback_to_username > 0
+            ):
+                logger.info(
+                    f"Owner extraction summary for site '{self.site_content_url}': "
+                    f"{self.report.num_email_fallback_to_username} entities fell back from email to username "
+                    f"(email was not available)"
+                )

datahub/ingestion/source/tableau/tableau_common.py

@@ -65,6 +65,7 @@ workbook_graphql_query = """
       projectName
       owner {
         username
+        email
       }
       description
       uri
@@ -107,6 +108,7 @@ sheet_graphql_query = """
         luid
         owner {
           username
+          email
         }
     }
     datasourceFields {
@@ -185,6 +187,7 @@ dashboard_graphql_query = """
         luid
         owner {
           username
+          email
         }
     }
 }
@@ -268,6 +271,7 @@ embedded_datasource_graphql_query = """
         luid
         owner {
           username
+          email
         }
     }
 }
@@ -424,6 +428,7 @@ published_datasource_graphql_query = """
     }
     owner {
       username
+      email
     }
     description
     uri

datahub/ingestion/source/tableau/tableau_constant.py

@@ -59,6 +59,7 @@ LUID = "luid"
 EMBEDDED_DATA_SOURCE = "EmbeddedDatasource"
 OWNER = "owner"
 USERNAME = "username"
+EMAIL = "email"
 HAS_EXTRACTS = "hasExtracts"
 EXTRACT_LAST_REFRESH_TIME = "extractLastRefreshTime"
 EXTRACT_LAST_INCREMENTAL_UPDATE_TIME = "extractLastIncrementalUpdateTime"

datahub/ingestion/source/tableau/tableau_server_wrapper.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from typing import Optional
 
 from tableauserverclient import Server, UserItem
 
@@ -10,6 +11,7 @@ class UserInfo:
     user_name: str
     site_role: str
     site_id: str
+    email: Optional[str] = None
 
     def has_site_administrator_explorer_privileges(self):
         return self.site_role in [
@@ -34,4 +36,5 @@ class UserInfo:
             user_name=user.name,
             site_role=user.site_role,
             site_id=server.site_id,
+            email=user.email,
         )