acryl-datahub 1.2.0.10rc2__py3-none-any.whl → 1.2.0.10rc4__py3-none-any.whl

datahub/ingestion/source/tableau/tableau.py

@@ -3,6 +3,7 @@ import logging
 import re
 import time
 from collections import OrderedDict, defaultdict
+from copy import deepcopy
 from dataclasses import dataclass, field as dataclass_field
 from datetime import datetime, timedelta, timezone
 from functools import lru_cache
@@ -474,6 +475,13 @@ class TableauPageSizeConfig(ConfigModel):
         return self.database_table_page_size or self.page_size
 
 
+_IngestHiddenAssetsOptionsType = Literal["worksheet", "dashboard"]
+_IngestHiddenAssetsOptions: List[_IngestHiddenAssetsOptionsType] = [
+    "worksheet",
+    "dashboard",
+]
+
+
 class TableauConfig(
     DatasetLineageProviderConfigBase,
     StatefulIngestionConfigBase,
@@ -524,6 +532,10 @@ class TableauConfig(
         default=False,
         description="Ingest Owner from source. This will override Owner info entered from UI",
     )
+    use_email_as_username: bool = Field(
+        default=False,
+        description="Use email address instead of username for entity owners. Requires ingest_owner to be True.",
+    )
     ingest_tables_external: bool = Field(
         default=False,
         description="Ingest details for tables external to (not embedded in) tableau as entities.",
@@ -621,8 +633,8 @@ class TableauConfig(
         description="Configuration settings for ingesting Tableau groups and their capabilities as custom properties.",
     )
 
-    ingest_hidden_assets: Union[List[Literal["worksheet", "dashboard"]], bool] = Field(
-        default=["worksheet", "dashboard"],
+    ingest_hidden_assets: Union[List[_IngestHiddenAssetsOptionsType], bool] = Field(
+        _IngestHiddenAssetsOptions,
         description=(
             "When enabled, hidden worksheets and dashboards are ingested into Datahub."
             " If a dashboard or worksheet is hidden in Tableau the luid is blank."
@@ -644,6 +656,11 @@ class TableauConfig(
     # pre = True because we want to take some decision before pydantic initialize the configuration to default values
     @root_validator(pre=True)
     def projects_backward_compatibility(cls, values: Dict) -> Dict:
+        # In-place update of the input dict would cause state contamination. This was discovered through test failures
+        # in test_hex.py where the same dict is reused.
+        # So a copy is performed first.
+        values = deepcopy(values)
+
         projects = values.get("projects")
         project_pattern = values.get("project_pattern")
         project_path_pattern = values.get("project_path_pattern")
@@ -655,6 +672,7 @@ class TableauConfig(
             values["project_pattern"] = AllowDenyPattern(
                 allow=[f"^{prj}$" for prj in projects]
             )
+            values.pop("projects")
         elif (project_pattern or project_path_pattern) and projects:
             raise ValueError(
                 "projects is deprecated. Please use project_path_pattern only."
@@ -666,7 +684,7 @@ class TableauConfig(
 
         return values
 
-    @root_validator()
+    @root_validator(skip_on_failure=True)
     def validate_config_values(cls, values: Dict) -> Dict:
         tags_for_hidden_assets = values.get("tags_for_hidden_assets")
         ingest_tags = values.get("ingest_tags")
@@ -678,6 +696,14 @@ class TableauConfig(
             raise ValueError(
                 "tags_for_hidden_assets is only allowed with ingest_tags enabled. Be aware that this will overwrite tags entered from the UI."
             )
+
+        use_email_as_username = values.get("use_email_as_username")
+        ingest_owner = values.get("ingest_owner")
+        if use_email_as_username and not ingest_owner:
+            raise ValueError(
+                "use_email_as_username requires ingest_owner to be enabled."
+            )
+
         return values
 
 
@@ -839,6 +865,9 @@ class TableauSourceReport(
         default_factory=(lambda: defaultdict(int))
     )
 
+    # Owner extraction statistics
+    num_email_fallback_to_username: int = 0
+
 
 def report_user_role(report: TableauSourceReport, server: Server) -> None:
     title: str = "Insufficient Permissions"
@@ -2716,13 +2745,12 @@ class TableauSiteSource:
             dataset_snapshot.aspects.append(browse_paths)
 
         # Ownership
-        owner = (
-            self._get_ownership(datasource_info[c.OWNER][c.USERNAME])
-            if datasource_info
-            and datasource_info.get(c.OWNER)
-            and datasource_info[c.OWNER].get(c.USERNAME)
+        owner_identifier = (
+            self._get_owner_identifier(datasource_info[c.OWNER])
+            if datasource_info and datasource_info.get(c.OWNER)
             else None
         )
+        owner = self._get_ownership(owner_identifier) if owner_identifier else None
         if owner is not None:
             dataset_snapshot.aspects.append(owner)
 
@@ -3127,7 +3155,7 @@ class TableauSiteSource:
 
         creator: Optional[str] = None
         if workbook is not None and workbook.get(c.OWNER) is not None:
-            creator = workbook[c.OWNER].get(c.USERNAME)
+            creator = self._get_owner_identifier(workbook[c.OWNER])
         created_at = sheet.get(c.CREATED_AT, datetime.now())
         updated_at = sheet.get(c.UPDATED_AT, datetime.now())
         last_modified = self.get_last_modified(creator, created_at, updated_at)
@@ -3276,7 +3304,7 @@ class TableauSiteSource:
 
     def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUnit]:
         workbook_container_key = self.gen_workbook_key(workbook[c.ID])
-        creator = workbook.get(c.OWNER, {}).get(c.USERNAME)
+        creator = self._get_owner_identifier(workbook.get(c.OWNER, {}))
 
         owner_urn = (
             builder.make_user_urn(creator)
@@ -3458,7 +3486,7 @@ class TableauSiteSource:
 
         creator: Optional[str] = None
         if workbook is not None and workbook.get(c.OWNER) is not None:
-            creator = workbook[c.OWNER].get(c.USERNAME)
+            creator = self._get_owner_identifier(workbook[c.OWNER])
         created_at = dashboard.get(c.CREATED_AT, datetime.now())
         updated_at = dashboard.get(c.UPDATED_AT, datetime.now())
         last_modified = self.get_last_modified(creator, created_at, updated_at)
@@ -3605,6 +3633,20 @@ class TableauSiteSource:
             )
         return last_modified
 
+    def _get_owner_identifier(self, owner_dict: dict) -> Optional[str]:
+        """Extract owner identifier (email or username) based on configuration."""
+        if not owner_dict:
+            return None
+
+        if self.config.use_email_as_username:
+            email = owner_dict.get(c.EMAIL)
+            if email:
+                return email
+            # Fall back to username if email is not available
+            self.report.num_email_fallback_to_username += 1
+
+        return owner_dict.get(c.USERNAME)
+
     @lru_cache(maxsize=None)
     def _get_ownership(self, user: str) -> Optional[OwnershipClass]:
         if self.config.ingest_owner and user:
@@ -3828,3 +3870,15 @@ class TableauSiteSource:
                     self.report.emit_upstream_tables_timer[self.site_content_url] = (
                         timer.elapsed_seconds(digits=2)
                     )
+
+            # Log owner extraction statistics if there were fallbacks
+            if (
+                self.config.use_email_as_username
+                and self.config.ingest_owner
+                and self.report.num_email_fallback_to_username > 0
+            ):
+                logger.info(
+                    f"Owner extraction summary for site '{self.site_content_url}': "
+                    f"{self.report.num_email_fallback_to_username} entities fell back from email to username "
+                    f"(email was not available)"
+                )

datahub/ingestion/source/tableau/tableau_common.py

@@ -65,6 +65,7 @@ workbook_graphql_query = """
       projectName
       owner {
         username
+        email
       }
       description
       uri
@@ -107,6 +108,7 @@ sheet_graphql_query = """
         luid
         owner {
           username
+          email
         }
     }
     datasourceFields {
@@ -185,6 +187,7 @@ dashboard_graphql_query = """
         luid
         owner {
           username
+          email
         }
     }
 }
@@ -268,6 +271,7 @@ embedded_datasource_graphql_query = """
         luid
         owner {
           username
+          email
         }
     }
 }
@@ -424,6 +428,7 @@ published_datasource_graphql_query = """
     }
     owner {
       username
+      email
     }
     description
     uri

datahub/ingestion/source/tableau/tableau_constant.py

@@ -59,6 +59,7 @@ LUID = "luid"
 EMBEDDED_DATA_SOURCE = "EmbeddedDatasource"
 OWNER = "owner"
 USERNAME = "username"
+EMAIL = "email"
 HAS_EXTRACTS = "hasExtracts"
 EXTRACT_LAST_REFRESH_TIME = "extractLastRefreshTime"
 EXTRACT_LAST_INCREMENTAL_UPDATE_TIME = "extractLastIncrementalUpdateTime"

datahub/ingestion/source/tableau/tableau_server_wrapper.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from typing import Optional
 
 from tableauserverclient import Server, UserItem
 
@@ -10,6 +11,7 @@ class UserInfo:
     user_name: str
     site_role: str
     site_id: str
+    email: Optional[str] = None
 
     def has_site_administrator_explorer_privileges(self):
         return self.site_role in [
@@ -34,4 +36,5 @@ class UserInfo:
             user_name=user.name,
             site_role=user.site_role,
             site_id=server.site_id,
+            email=user.email,
         )

datahub/ingestion/source/unity/config.py

@@ -8,7 +8,12 @@ import pydantic
 from pydantic import Field
 from typing_extensions import Literal
 
-from datahub.configuration.common import AllowDenyPattern, ConfigEnum, ConfigModel
+from datahub.configuration.common import (
+    AllowDenyPattern,
+    ConfigEnum,
+    ConfigModel,
+    HiddenFromDocs,
+)
 from datahub.configuration.source_common import (
     DatasetSourceConfigMixin,
     LowerCaseDatasetUrnConfigMixin,
@@ -285,10 +290,9 @@ class UnityCatalogSourceConfig(
         description="Limit the number of columns to get column level lineage. ",
     )
 
-    lineage_max_workers: int = pydantic.Field(
+    lineage_max_workers: HiddenFromDocs[int] = pydantic.Field(
         default=5 * (os.cpu_count() or 4),
         description="Number of worker threads to use for column lineage thread pool executor. Set to 1 to disable.",
-        hidden_from_docs=True,
     )
 
     databricks_api_page_size: int = pydantic.Field(

datahub/ingestion/source/usage/usage_common.py

@@ -18,7 +18,7 @@ import pydantic
 from pydantic.fields import Field
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.common import AllowDenyPattern
+from datahub.configuration.common import AllowDenyPattern, HiddenFromDocs
 from datahub.configuration.time_window_config import (
     BaseTimeWindowConfig,
     BucketDuration,
@@ -194,13 +194,13 @@ class GenericAggregatedDataset(Generic[ResourceType]):
 
 
 class BaseUsageConfig(BaseTimeWindowConfig):
-    queries_character_limit: int = Field(
+    queries_character_limit: HiddenFromDocs[int] = Field(
+        # Hidden since we don't want to encourage people to break elasticsearch.
         default=DEFAULT_QUERIES_CHARACTER_LIMIT,
         description=(
             "Total character limit for all queries in a single usage aspect."
             " Queries will be truncated to length `queries_character_limit / top_n_queries`."
         ),
-        hidden_from_docs=True,  # Don't want to encourage people to break elasticsearch
     )
 
     top_n_queries: pydantic.PositiveInt = Field(

datahub/ingestion/source_config/pulsar.py

@@ -2,6 +2,7 @@ import re
 from typing import Dict, List, Optional, Union
 from urllib.parse import urlparse
 
+import pydantic
 from pydantic import Field, validator
 
 from datahub.configuration.common import AllowDenyPattern
@@ -121,7 +122,8 @@ class PulsarSourceConfig(
             )
         return client_secret
 
-    @validator("web_service_url")
+    @pydantic.field_validator("web_service_url", mode="after")
+    @classmethod
     def web_service_url_scheme_host_port(cls, val: str) -> str:
         # Tokenize the web url
         url = urlparse(val)

datahub/metadata/_internal_schema_classes.py

@@ -5356,6 +5356,7 @@ class InstitutionalMemoryMetadataClass(DictWrapper):
         description: str,
         createStamp: "AuditStampClass",
         updateStamp: Union[None, "AuditStampClass"]=None,
+        settings: Union[None, "InstitutionalMemoryMetadataSettingsClass"]=None,
     ):
         super().__init__()
         
@@ -5363,12 +5364,14 @@ class InstitutionalMemoryMetadataClass(DictWrapper):
         self.description = description
         self.createStamp = createStamp
         self.updateStamp = updateStamp
+        self.settings = settings
     
     def _restore_defaults(self) -> None:
         self.url = str()
         self.description = str()
         self.createStamp = AuditStampClass._construct_with_defaults()
         self.updateStamp = self.RECORD_SCHEMA.fields_dict["updateStamp"].default
+        self.settings = self.RECORD_SCHEMA.fields_dict["settings"].default
     
     
     @property
@@ -5411,6 +5414,45 @@ class InstitutionalMemoryMetadataClass(DictWrapper):
         self._inner_dict['updateStamp'] = value
     
     
+    @property
+    def settings(self) -> Union[None, "InstitutionalMemoryMetadataSettingsClass"]:
+        """Settings for this record"""
+        return self._inner_dict.get('settings')  # type: ignore
+    
+    @settings.setter
+    def settings(self, value: Union[None, "InstitutionalMemoryMetadataSettingsClass"]) -> None:
+        self._inner_dict['settings'] = value
+    
+    
+class InstitutionalMemoryMetadataSettingsClass(DictWrapper):
+    """Settings related to a record of InstitutionalMemoryMetadata"""
+    
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.common.InstitutionalMemoryMetadataSettings")
+    def __init__(self,
+        showInAssetPreview: Optional[bool]=None,
+    ):
+        super().__init__()
+        
+        if showInAssetPreview is None:
+            # default: False
+            self.showInAssetPreview = self.RECORD_SCHEMA.fields_dict["showInAssetPreview"].default
+        else:
+            self.showInAssetPreview = showInAssetPreview
+    
+    def _restore_defaults(self) -> None:
+        self.showInAssetPreview = self.RECORD_SCHEMA.fields_dict["showInAssetPreview"].default
+    
+    
+    @property
+    def showInAssetPreview(self) -> bool:
+        """Show record in asset preview like on entity header and search previews"""
+        return self._inner_dict.get('showInAssetPreview')  # type: ignore
+    
+    @showInAssetPreview.setter
+    def showInAssetPreview(self, value: bool) -> None:
+        self._inner_dict['showInAssetPreview'] = value
+    
+    
 class MLFeatureDataTypeClass(object):
     """MLFeature Data Type"""
     
@@ -16689,7 +16731,7 @@ class MLModelGroupKeyClass(_Aspect):
 
 
     ASPECT_NAME = 'mlModelGroupKey'
-    ASPECT_INFO = {'keyForEntity': 'mlModelGroup', 'entityCategory': 'core', 'entityAspects': ['glossaryTerms', 'editableMlModelGroupProperties', 'domains', 'applications', 'mlModelGroupProperties', 'ownership', 'status', 'deprecation', 'browsePaths', 'globalTags', 'dataPlatformInstance', 'browsePathsV2', 'structuredProperties', 'forms', 'testResults', 'subTypes', 'container']}
+    ASPECT_INFO = {'keyForEntity': 'mlModelGroup', 'entityCategory': 'core', 'entityAspects': ['glossaryTerms', 'editableMlModelGroupProperties', 'domains', 'applications', 'mlModelGroupProperties', 'ownership', 'status', 'deprecation', 'browsePaths', 'globalTags', 'dataPlatformInstance', 'browsePathsV2', 'structuredProperties', 'forms', 'testResults', 'subTypes', 'container', 'institutionalMemory']}
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.MLModelGroupKey")
 
     def __init__(self,
@@ -27420,6 +27462,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.common.InputFields': InputFieldsClass,
     'com.linkedin.pegasus2avro.common.InstitutionalMemory': InstitutionalMemoryClass,
     'com.linkedin.pegasus2avro.common.InstitutionalMemoryMetadata': InstitutionalMemoryMetadataClass,
+    'com.linkedin.pegasus2avro.common.InstitutionalMemoryMetadataSettings': InstitutionalMemoryMetadataSettingsClass,
     'com.linkedin.pegasus2avro.common.MLFeatureDataType': MLFeatureDataTypeClass,
     'com.linkedin.pegasus2avro.common.Media': MediaClass,
     'com.linkedin.pegasus2avro.common.MediaType': MediaTypeClass,
@@ -27939,6 +27982,7 @@ __SCHEMA_TYPES = {
     'InputFields': InputFieldsClass,
     'InstitutionalMemory': InstitutionalMemoryClass,
     'InstitutionalMemoryMetadata': InstitutionalMemoryMetadataClass,
+    'InstitutionalMemoryMetadataSettings': InstitutionalMemoryMetadataSettingsClass,
     'MLFeatureDataType': MLFeatureDataTypeClass,
     'Media': MediaClass,
     'MediaType': MediaTypeClass,

datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py

@@ -45,6 +45,7 @@ from .....schema_classes import InputFieldClass
 from .....schema_classes import InputFieldsClass
 from .....schema_classes import InstitutionalMemoryClass
 from .....schema_classes import InstitutionalMemoryMetadataClass
+from .....schema_classes import InstitutionalMemoryMetadataSettingsClass
 from .....schema_classes import MLFeatureDataTypeClass
 from .....schema_classes import MediaClass
 from .....schema_classes import MediaTypeClass
@@ -111,6 +112,7 @@ InputField = InputFieldClass
 InputFields = InputFieldsClass
 InstitutionalMemory = InstitutionalMemoryClass
 InstitutionalMemoryMetadata = InstitutionalMemoryMetadataClass
+InstitutionalMemoryMetadataSettings = InstitutionalMemoryMetadataSettingsClass
 MLFeatureDataType = MLFeatureDataTypeClass
 Media = MediaClass
 MediaType = MediaTypeClass

datahub/metadata/schema.avsc

@@ -1074,6 +1074,28 @@
                 "name": "updateStamp",
                 "default": null,
                 "doc": "Audit stamp associated with updation of this record"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "record",
+                    "name": "InstitutionalMemoryMetadataSettings",
+                    "namespace": "com.linkedin.pegasus2avro.common",
+                    "fields": [
+                      {
+                        "type": "boolean",
+                        "name": "showInAssetPreview",
+                        "default": false,
+                        "doc": "Show record in asset preview like on entity header and search previews"
+                      }
+                    ],
+                    "doc": "Settings related to a record of InstitutionalMemoryMetadata"
+                  }
+                ],
+                "name": "settings",
+                "default": null,
+                "doc": "Settings for this record"
               }
             ],
             "doc": "Metadata corresponding to a record of institutional memory."
@@ -4280,7 +4302,8 @@
         "forms",
         "testResults",
         "subTypes",
-        "container"
+        "container",
+        "institutionalMemory"
       ]
     },
     "name": "MLModelGroupKey",

datahub/metadata/schemas/InstitutionalMemory.avsc

@@ -84,6 +84,28 @@
               "name": "updateStamp",
               "default": null,
               "doc": "Audit stamp associated with updation of this record"
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "InstitutionalMemoryMetadataSettings",
+                  "namespace": "com.linkedin.pegasus2avro.common",
+                  "fields": [
+                    {
+                      "type": "boolean",
+                      "name": "showInAssetPreview",
+                      "default": false,
+                      "doc": "Show record in asset preview like on entity header and search previews"
+                    }
+                  ],
+                  "doc": "Settings related to a record of InstitutionalMemoryMetadata"
+                }
+              ],
+              "name": "settings",
+              "default": null,
+              "doc": "Settings for this record"
             }
           ],
           "doc": "Metadata corresponding to a record of institutional memory."

datahub/metadata/schemas/MLModelGroupKey.avsc

@@ -21,7 +21,8 @@
       "forms",
       "testResults",
       "subTypes",
-      "container"
+      "container",
+      "institutionalMemory"
     ]
   },
   "name": "MLModelGroupKey",

datahub/metadata/schemas/MetadataChangeEvent.avsc

@@ -1143,6 +1143,28 @@
                                 "name": "updateStamp",
                                 "default": null,
                                 "doc": "Audit stamp associated with updation of this record"
+                              },
+                              {
+                                "type": [
+                                  "null",
+                                  {
+                                    "type": "record",
+                                    "name": "InstitutionalMemoryMetadataSettings",
+                                    "namespace": "com.linkedin.pegasus2avro.common",
+                                    "fields": [
+                                      {
+                                        "type": "boolean",
+                                        "name": "showInAssetPreview",
+                                        "default": false,
+                                        "doc": "Show record in asset preview like on entity header and search previews"
+                                      }
+                                    ],
+                                    "doc": "Settings related to a record of InstitutionalMemoryMetadata"
+                                  }
+                                ],
+                                "name": "settings",
+                                "default": null,
+                                "doc": "Settings for this record"
                               }
                             ],
                             "doc": "Metadata corresponding to a record of institutional memory."

datahub/sdk/dashboard.py

@@ -171,8 +171,6 @@ class Dashboard(
                     )
                 ),
                 customProperties={},
-                chartEdges=[],
-                datasetEdges=[],
                 dashboards=[],
             )
         )

datahub/sdk/search_filters.py

@@ -39,13 +39,7 @@ _OrFilters = List[_AndSearchFilterRule]
 
 
 class _BaseFilter(ConfigModel):
-    class Config:
-        # We can't wrap this in a TYPE_CHECKING block because the pydantic plugin
-        # doesn't recognize it properly. So unfortunately we'll need to live
-        # with the deprecation warning w/ pydantic v2.
-        allow_population_by_field_name = True
-        if PYDANTIC_VERSION_2:
-            populate_by_name = True
+    model_config = pydantic.ConfigDict(populate_by_name=True)
 
     @abc.abstractmethod
     def compile(self) -> _OrFilters: ...