acryl-datahub 1.2.0.10rc3__py3-none-any.whl → 1.2.0.10rc5__py3-none-any.whl

datahub/ingestion/source/tableau/tableau.py

@@ -3,6 +3,7 @@ import logging
 import re
 import time
 from collections import OrderedDict, defaultdict
+from copy import deepcopy
 from dataclasses import dataclass, field as dataclass_field
 from datetime import datetime, timedelta, timezone
 from functools import lru_cache
@@ -474,6 +475,13 @@ class TableauPageSizeConfig(ConfigModel):
         return self.database_table_page_size or self.page_size
 
 
+_IngestHiddenAssetsOptionsType = Literal["worksheet", "dashboard"]
+_IngestHiddenAssetsOptions: List[_IngestHiddenAssetsOptionsType] = [
+    "worksheet",
+    "dashboard",
+]
+
+
 class TableauConfig(
     DatasetLineageProviderConfigBase,
     StatefulIngestionConfigBase,
@@ -586,13 +594,13 @@ class TableauConfig(
     )
 
     extract_lineage_from_unsupported_custom_sql_queries: bool = Field(
-        default=False,
-        description="[Experimental] Whether to extract lineage from unsupported custom sql queries using SQL parsing",
+        default=True,
+        description="[Experimental] Extract lineage from Custom SQL queries using DataHub's SQL parser in cases where the Tableau Catalog API fails to return lineage for the query.",
     )
 
     force_extraction_of_lineage_from_custom_sql_queries: bool = Field(
         default=False,
-        description="[Experimental] Force extraction of lineage from custom sql queries using SQL parsing, ignoring Tableau metadata",
+        description="[Experimental] Force extraction of lineage from Custom SQL queries using DataHub's SQL parser, even when the Tableau Catalog API returns lineage already.",
     )
 
     sql_parsing_disable_schema_awareness: bool = Field(
@@ -625,8 +633,8 @@ class TableauConfig(
         description="Configuration settings for ingesting Tableau groups and their capabilities as custom properties.",
     )
 
-    ingest_hidden_assets: Union[List[Literal["worksheet", "dashboard"]], bool] = Field(
-        default=["worksheet", "dashboard"],
+    ingest_hidden_assets: Union[List[_IngestHiddenAssetsOptionsType], bool] = Field(
+        _IngestHiddenAssetsOptions,
         description=(
             "When enabled, hidden worksheets and dashboards are ingested into Datahub."
             " If a dashboard or worksheet is hidden in Tableau the luid is blank."
@@ -648,6 +656,11 @@ class TableauConfig(
     # pre = True because we want to take some decision before pydantic initialize the configuration to default values
     @root_validator(pre=True)
     def projects_backward_compatibility(cls, values: Dict) -> Dict:
+        # In-place update of the input dict would cause state contamination. This was discovered through test failures
+        # in test_hex.py where the same dict is reused.
+        # So a copy is performed first.
+        values = deepcopy(values)
+
         projects = values.get("projects")
         project_pattern = values.get("project_pattern")
         project_path_pattern = values.get("project_path_pattern")
@@ -659,6 +672,7 @@ class TableauConfig(
             values["project_pattern"] = AllowDenyPattern(
                 allow=[f"^{prj}$" for prj in projects]
             )
+            values.pop("projects")
         elif (project_pattern or project_path_pattern) and projects:
             raise ValueError(
                 "projects is deprecated. Please use project_path_pattern only."
@@ -670,7 +684,7 @@ class TableauConfig(
 
         return values
 
-    @root_validator()
+    @root_validator(skip_on_failure=True)
     def validate_config_values(cls, values: Dict) -> Dict:
         tags_for_hidden_assets = values.get("tags_for_hidden_assets")
         ingest_tags = values.get("ingest_tags")

datahub/ingestion/source/unity/config.py

@@ -8,7 +8,12 @@ import pydantic
 from pydantic import Field
 from typing_extensions import Literal
 
-from datahub.configuration.common import AllowDenyPattern, ConfigEnum, ConfigModel
+from datahub.configuration.common import (
+    AllowDenyPattern,
+    ConfigEnum,
+    ConfigModel,
+    HiddenFromDocs,
+)
 from datahub.configuration.source_common import (
     DatasetSourceConfigMixin,
     LowerCaseDatasetUrnConfigMixin,
@@ -285,10 +290,9 @@ class UnityCatalogSourceConfig(
         description="Limit the number of columns to get column level lineage. ",
     )
 
-    lineage_max_workers: int = pydantic.Field(
+    lineage_max_workers: HiddenFromDocs[int] = pydantic.Field(
         default=5 * (os.cpu_count() or 4),
         description="Number of worker threads to use for column lineage thread pool executor. Set to 1 to disable.",
-        hidden_from_docs=True,
     )
 
     databricks_api_page_size: int = pydantic.Field(

datahub/ingestion/source/usage/usage_common.py

@@ -18,7 +18,7 @@ import pydantic
 from pydantic.fields import Field
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.common import AllowDenyPattern
+from datahub.configuration.common import AllowDenyPattern, HiddenFromDocs
 from datahub.configuration.time_window_config import (
     BaseTimeWindowConfig,
     BucketDuration,
@@ -194,13 +194,13 @@ class GenericAggregatedDataset(Generic[ResourceType]):
 
 
 class BaseUsageConfig(BaseTimeWindowConfig):
-    queries_character_limit: int = Field(
+    queries_character_limit: HiddenFromDocs[int] = Field(
+        # Hidden since we don't want to encourage people to break elasticsearch.
         default=DEFAULT_QUERIES_CHARACTER_LIMIT,
         description=(
             "Total character limit for all queries in a single usage aspect."
             " Queries will be truncated to length `queries_character_limit / top_n_queries`."
         ),
-        hidden_from_docs=True,  # Don't want to encourage people to break elasticsearch
     )
 
     top_n_queries: pydantic.PositiveInt = Field(

datahub/ingestion/source_config/pulsar.py

@@ -2,6 +2,7 @@ import re
 from typing import Dict, List, Optional, Union
 from urllib.parse import urlparse
 
+import pydantic
 from pydantic import Field, validator
 
 from datahub.configuration.common import AllowDenyPattern
@@ -121,7 +122,8 @@ class PulsarSourceConfig(
             )
         return client_secret
 
-    @validator("web_service_url")
+    @pydantic.field_validator("web_service_url", mode="after")
+    @classmethod
     def web_service_url_scheme_host_port(cls, val: str) -> str:
         # Tokenize the web url
         url = urlparse(val)

datahub/ingestion/transformer/set_browse_path.py

@@ -0,0 +1,112 @@
+import re
+from collections import defaultdict
+from typing import Dict, List, Optional, cast
+
+from datahub.configuration.common import (
+    TransformerSemanticsConfigModel,
+)
+from datahub.emitter.mce_builder import Aspect
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.transformer.base_transformer import (
+    BaseTransformer,
+    SingleAspectTransformer,
+)
+from datahub.metadata.schema_classes import (
+    BrowsePathEntryClass,
+    BrowsePathsV2Class,
+)
+from datahub.utilities.urns.urn import guess_entity_type
+
+
+class SetBrowsePathTransformerConfig(TransformerSemanticsConfigModel):
+    path: List[str]
+
+
+class SetBrowsePathTransformer(BaseTransformer, SingleAspectTransformer):
+    ctx: PipelineContext
+    config: SetBrowsePathTransformerConfig
+
+    def __init__(self, config: SetBrowsePathTransformerConfig, ctx: PipelineContext):
+        super().__init__()
+        self.ctx = ctx
+        self.config = config
+
+    def aspect_name(self) -> str:
+        return "browsePathsV2"
+
+    def entity_types(self) -> List[str]:
+        # This is an arbitrary list, might be adjusted if it makes sense. It might be reasonable to make it configurable
+        return ["dataset", "dataJob", "dataFlow", "chart", "dashboard", "container"]
+
+    @classmethod
+    def create(
+        cls, config_dict: dict, ctx: PipelineContext
+    ) -> "SetBrowsePathTransformer":
+        config = SetBrowsePathTransformerConfig.parse_obj(config_dict)
+        return cls(config, ctx)
+
+    @staticmethod
+    def _build_model(existing_browse_paths: BrowsePathsV2Class) -> Dict[str, List[str]]:
+        template_vars: Dict[str, List[str]] = {}
+        model: Dict[str, List[str]] = defaultdict(list)
+        for entry in existing_browse_paths.path or []:
+            if entry.urn:
+                entity_type = guess_entity_type(entry.urn)
+                model[entity_type].append(entry.urn)
+
+        for entity_type, urns in model.items():
+            template_vars[f"{entity_type}[*]"] = urns
+            for i, urn in enumerate(urns):
+                template_vars[f"{entity_type}[{i}]"] = [urn]
+
+        return template_vars
+
+    @classmethod
+    def _expand_nodes(
+        cls, templates: List[str], template_vars: Dict[str, List[str]]
+    ) -> BrowsePathsV2Class:
+        expanded_nodes: List[str] = []
+        for node in templates:
+            resolved_nodes = cls._resolve_template_to_nodes(node, template_vars)
+            expanded_nodes.extend(resolved_nodes)
+
+        processed_entries: List[BrowsePathEntryClass] = []
+        for node in expanded_nodes:
+            if not node or node.isspace():
+                continue
+            processed_entries.append(
+                BrowsePathEntryClass(
+                    id=node, urn=node if node.startswith("urn:") else None
+                )
+            )
+        return BrowsePathsV2Class(path=processed_entries)
+
+    def transform_aspect(
+        self, entity_urn: str, aspect_name: str, aspect: Optional[Aspect]
+    ) -> Optional[Aspect]:
+        template_vars: Dict[str, List[str]] = {}
+        if aspect is not None:
+            assert isinstance(aspect, BrowsePathsV2Class)
+            template_vars = self._build_model(aspect)
+        new_browse_paths: BrowsePathsV2Class = self._expand_nodes(
+            self.config.path, template_vars
+        )
+        if aspect is not None and not self.config.replace_existing:
+            for node in aspect.path:
+                new_browse_paths.path.append(node)
+
+        return cast(Aspect, new_browse_paths)
+
+    @staticmethod
+    def _resolve_template_to_nodes(
+        template_str: str, template_vars: Dict[str, List[str]]
+    ) -> List[str]:
+        # This mechanism can be made simpler (match against known variables only) or more complex (e.g. by using a
+        # proper templating engine, like jinja).
+        template_str = template_str.strip()
+        var_pattern = re.findall(r"^\$([a-zA-Z]+\[[0-9*]+]$)", template_str)
+
+        if not var_pattern:
+            return [template_str]
+
+        return template_vars.get(var_pattern[0], [])

datahub/sdk/_shared.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import warnings
+from abc import ABC, abstractmethod
 from datetime import datetime
 from typing import (
     TYPE_CHECKING,
@@ -61,6 +62,7 @@ DataPlatformInstanceUrnOrStr: TypeAlias = Union[str, DataPlatformInstanceUrn]
 DataPlatformUrnOrStr: TypeAlias = Union[str, DataPlatformUrn]
 
 ActorUrn: TypeAlias = Union[CorpUserUrn, CorpGroupUrn]
+ActorUrnOrStr: TypeAlias = Union[str, ActorUrn]
 StructuredPropertyUrnOrStr: TypeAlias = Union[str, StructuredPropertyUrn]
 StructuredPropertyValueType: TypeAlias = Union[str, float, int]
 StructuredPropertyInputType: TypeAlias = Dict[
@@ -110,6 +112,130 @@ def parse_time_stamp(ts: Optional[models.TimeStampClass]) -> Optional[datetime]:
     return parse_ts_millis(ts.time)
 
 
+class ChangeAuditStampsMixin(ABC):
+    """Mixin class for managing audit stamps on entities."""
+
+    __slots__ = ()
+
+    @abstractmethod
+    def _get_audit_stamps(self) -> models.ChangeAuditStampsClass:
+        """Get the audit stamps from the entity properties."""
+        pass
+
+    @abstractmethod
+    def _set_audit_stamps(self, audit_stamps: models.ChangeAuditStampsClass) -> None:
+        """Set the audit stamps on the entity properties."""
+        pass
+
+    @property
+    def last_modified(self) -> Optional[datetime]:
+        """Get the last modification timestamp from audit stamps."""
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        if audit_stamps.lastModified.time == 0:
+            return None
+        return datetime.fromtimestamp(
+            audit_stamps.lastModified.time / 1000
+        )  # supports only seconds precision
+
+    def set_last_modified(self, last_modified: datetime) -> None:
+        """Set the last modification timestamp in audit stamps."""
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        audit_stamps.lastModified.time = make_ts_millis(last_modified)
+        self._set_audit_stamps(audit_stamps)
+
+    @property
+    def last_modified_by(self) -> Optional[str]:
+        """Get the last modification actor from audit stamps."""
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        if audit_stamps.lastModified.actor == builder.UNKNOWN_USER:
+            return None
+        return audit_stamps.lastModified.actor
+
+    def set_last_modified_by(self, last_modified_by: ActorUrnOrStr) -> None:
+        """Set the last modification actor in audit stamps."""
+        if isinstance(last_modified_by, str):
+            last_modified_by = make_user_urn(last_modified_by)
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        audit_stamps.lastModified.actor = str(last_modified_by)
+        self._set_audit_stamps(audit_stamps)
+
+    @property
+    def created_at(self) -> Optional[datetime]:
+        """Get the creation timestamp from audit stamps."""
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        if audit_stamps.created.time == 0:
+            return None
+        return datetime.fromtimestamp(
+            audit_stamps.created.time / 1000
+        )  # supports only seconds precision
+
+    def set_created_at(self, created_at: datetime) -> None:
+        """Set the creation timestamp in audit stamps."""
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        audit_stamps.created.time = make_ts_millis(created_at)
+        self._set_audit_stamps(audit_stamps)
+
+    @property
+    def created_by(self) -> Optional[ActorUrnOrStr]:
+        """Get the creation actor from audit stamps."""
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        if audit_stamps.created.actor == builder.UNKNOWN_USER:
+            return None
+        return audit_stamps.created.actor
+
+    def set_created_by(self, created_by: ActorUrnOrStr) -> None:
+        """Set the creation actor in audit stamps."""
+        if isinstance(created_by, str):
+            created_by = make_user_urn(created_by)
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        audit_stamps.created.actor = str(created_by)
+        self._set_audit_stamps(audit_stamps)
+
+    @property
+    def deleted_on(self) -> Optional[datetime]:
+        """Get the deletion timestamp from audit stamps."""
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        if audit_stamps.deleted is None or audit_stamps.deleted.time == 0:
+            return None
+        return datetime.fromtimestamp(
+            audit_stamps.deleted.time / 1000
+        )  # supports only seconds precision
+
+    def set_deleted_on(self, deleted_on: datetime) -> None:
+        """Set the deletion timestamp in audit stamps."""
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        # Default constructor sets deleted to None
+        if audit_stamps.deleted is None:
+            audit_stamps.deleted = models.AuditStampClass(
+                time=0, actor=builder.UNKNOWN_USER
+            )
+        audit_stamps.deleted.time = make_ts_millis(deleted_on)
+        self._set_audit_stamps(audit_stamps)
+
+    @property
+    def deleted_by(self) -> Optional[ActorUrnOrStr]:
+        """Get the deletion actor from audit stamps."""
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        if (
+            audit_stamps.deleted is None
+            or audit_stamps.deleted.actor == builder.UNKNOWN_USER
+        ):
+            return None
+        return audit_stamps.deleted.actor
+
+    def set_deleted_by(self, deleted_by: ActorUrnOrStr) -> None:
+        """Set the deletion actor in audit stamps."""
+        if isinstance(deleted_by, str):
+            deleted_by = make_user_urn(deleted_by)
+        audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
+        if audit_stamps.deleted is None:
+            audit_stamps.deleted = models.AuditStampClass(
+                time=0, actor=builder.UNKNOWN_USER
+            )
+        audit_stamps.deleted.actor = str(deleted_by)
+        self._set_audit_stamps(audit_stamps)
+
+
 class HasPlatformInstance(Entity):
     __slots__ = ()
 

datahub/sdk/chart.py

@@ -10,6 +10,8 @@ import datahub.metadata.schema_classes as models
 from datahub.emitter.enum_helpers import get_enum_options
 from datahub.metadata.urns import ChartUrn, DatasetUrn, Urn
 from datahub.sdk._shared import (
+    ActorUrnOrStr,
+    ChangeAuditStampsMixin,
     DataPlatformInstanceUrnOrStr,
     DataPlatformUrnOrStr,
     DatasetUrnOrStr,
@@ -34,6 +36,7 @@ from datahub.utilities.sentinels import Unset, unset
 
 
 class Chart(
+    ChangeAuditStampsMixin,
     HasPlatformInstance,
     HasSubtype,
     HasOwnership,
@@ -70,6 +73,11 @@ class Chart(
         chart_url: Optional[str] = None,
         custom_properties: Optional[Dict[str, str]] = None,
         last_modified: Optional[datetime] = None,
+        last_modified_by: Optional[ActorUrnOrStr] = None,
+        created_at: Optional[datetime] = None,
+        created_by: Optional[ActorUrnOrStr] = None,
+        deleted_on: Optional[datetime] = None,
+        deleted_by: Optional[ActorUrnOrStr] = None,
         last_refreshed: Optional[datetime] = None,
         chart_type: Optional[Union[str, models.ChartTypeClass]] = None,
         access: Optional[str] = None,
@@ -94,13 +102,60 @@ class Chart(
         self._set_extra_aspects(extra_aspects)
 
         self._set_platform_instance(platform, platform_instance)
-
         self._ensure_chart_props(display_name=display_name)
+        self._init_chart_properties(
+            description,
+            display_name,
+            external_url,
+            chart_url,
+            custom_properties,
+            last_modified,
+            last_modified_by,
+            created_at,
+            created_by,
+            last_refreshed,
+            deleted_on,
+            deleted_by,
+            chart_type,
+            access,
+            input_datasets,
+        )
+        self._init_standard_aspects(
+            parent_container, subtype, owners, links, tags, terms, domain
+        )
 
-        if display_name is not None:
-            self.set_display_name(display_name)
+    @classmethod
+    def _new_from_graph(cls, urn: Urn, current_aspects: models.AspectBag) -> Self:
+        assert isinstance(urn, ChartUrn)
+        entity = cls(
+            platform=urn.dashboard_tool,
+            name=urn.chart_id,
+        )
+        return entity._init_from_graph(current_aspects)
+
+    def _init_chart_properties(
+        self,
+        description: Optional[str],
+        display_name: Optional[str],
+        external_url: Optional[str],
+        chart_url: Optional[str],
+        custom_properties: Optional[Dict[str, str]],
+        last_modified: Optional[datetime],
+        last_modified_by: Optional[ActorUrnOrStr],
+        created_at: Optional[datetime],
+        created_by: Optional[ActorUrnOrStr],
+        last_refreshed: Optional[datetime],
+        deleted_on: Optional[datetime],
+        deleted_by: Optional[ActorUrnOrStr],
+        chart_type: Optional[Union[str, models.ChartTypeClass]],
+        access: Optional[str],
+        input_datasets: Optional[Sequence[Union[DatasetUrnOrStr, Dataset]]],
+    ) -> None:
+        """Initialize chart-specific properties."""
         if description is not None:
             self.set_description(description)
+        if display_name is not None:
+            self.set_display_name(display_name)
         if external_url is not None:
             self.set_external_url(external_url)
         if chart_url is not None:
@@ -109,6 +164,16 @@ class Chart(
             self.set_custom_properties(custom_properties)
         if last_modified is not None:
             self.set_last_modified(last_modified)
+        if last_modified_by is not None:
+            self.set_last_modified_by(last_modified_by)
+        if created_at is not None:
+            self.set_created_at(created_at)
+        if created_by is not None:
+            self.set_created_by(created_by)
+        if deleted_on is not None:
+            self.set_deleted_on(deleted_on)
+        if deleted_by is not None:
+            self.set_deleted_by(deleted_by)
         if last_refreshed is not None:
             self.set_last_refreshed(last_refreshed)
         if chart_type is not None:
@@ -118,6 +183,17 @@ class Chart(
         if input_datasets is not None:
             self.set_input_datasets(input_datasets)
 
+    def _init_standard_aspects(
+        self,
+        parent_container: ParentContainerInputType | Unset,
+        subtype: Optional[str],
+        owners: Optional[OwnersInputType],
+        links: Optional[LinksInputType],
+        tags: Optional[TagsInputType],
+        terms: Optional[TermsInputType],
+        domain: Optional[DomainInputType],
+    ) -> None:
+        """Initialize standard aspects."""
         if parent_container is not unset:
             self._set_container(parent_container)
         if subtype is not None:
@@ -133,15 +209,6 @@ class Chart(
         if domain is not None:
             self.set_domain(domain)
 
-    @classmethod
-    def _new_from_graph(cls, urn: Urn, current_aspects: models.AspectBag) -> Self:
-        assert isinstance(urn, ChartUrn)
-        entity = cls(
-            platform=urn.dashboard_tool,
-            name=urn.chart_id,
-        )
-        return entity._init_from_graph(current_aspects)
-
     @property
     def urn(self) -> ChartUrn:
         assert isinstance(self._urn, ChartUrn)
@@ -159,6 +226,14 @@ class Chart(
             )
         )
 
+    def _get_audit_stamps(self) -> models.ChangeAuditStampsClass:
+        """Get the audit stamps from the chart properties."""
+        return self._ensure_chart_props().lastModified
+
+    def _set_audit_stamps(self, audit_stamps: models.ChangeAuditStampsClass) -> None:
+        """Set the audit stamps on the chart properties."""
+        self._ensure_chart_props().lastModified = audit_stamps
+
     @property
     def name(self) -> str:
         """Get the name of the chart."""
@@ -220,24 +295,6 @@ class Chart(
         """Set the custom properties of the chart."""
         self._ensure_chart_props().customProperties = custom_properties
 
-    @property
-    def last_modified(self) -> Optional[datetime]:
-        """Get the last modification timestamp of the chart."""
-        last_modified_time = self._ensure_chart_props().lastModified.lastModified.time
-        if not last_modified_time:
-            return None
-        return datetime.fromtimestamp(last_modified_time)
-
-    def set_last_modified(self, last_modified: datetime) -> None:
-        """Set the last modification timestamp of the chart."""
-        chart_props = self._ensure_chart_props()
-        chart_props.lastModified = models.ChangeAuditStampsClass(
-            lastModified=models.AuditStampClass(
-                time=int(last_modified.timestamp()),
-                actor="urn:li:corpuser:datahub",
-            )
-        )
-
     @property
     def last_refreshed(self) -> Optional[datetime]:
         """Get the last refresh timestamp of the chart."""