acryl-datahub 1.1.1rc3__py3-none-any.whl → 1.2.0__py3-none-any.whl

{acryl_datahub-1.1.1rc3.dist-info → acryl_datahub-1.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{acryl_datahub-1.1.1rc3.dist-info → acryl_datahub-1.2.0.dist-info}/entry_points.txt

@@ -36,8 +36,10 @@ csv-enricher = datahub.ingestion.source.csv_enricher:CSVEnricherSource
 datahub = datahub.ingestion.source.datahub.datahub_source:DataHubSource
 datahub-apply = datahub.ingestion.source.apply.datahub_apply:DataHubApplySource
 datahub-business-glossary = datahub.ingestion.source.metadata.business_glossary:BusinessGlossaryFileSource
+datahub-debug = datahub.ingestion.source.debug.datahub_debug:DataHubDebugSource
 datahub-gc = datahub.ingestion.source.gc.datahub_gc:DataHubGcSource
 datahub-lineage-file = datahub.ingestion.source.metadata.lineage:LineageFileSource
+datahub-mock-data = datahub.ingestion.source.mock_data.datahub_mock_data:DataHubMockDataSource
 dbt = datahub.ingestion.source.dbt.dbt_core:DBTCoreSource
 dbt-cloud = datahub.ingestion.source.dbt.dbt_cloud:DBTCloudSource
 delta-lake = datahub.ingestion.source.delta_lake:DeltaLakeSource

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.1.1rc3"
+__version__ = "1.2.0"
 
 
 def is_dev_mode() -> bool:

datahub/api/entities/dataset/dataset.py

@@ -383,7 +383,7 @@ class Dataset(StrictModel):
     urn: Optional[str] = None
     description: Optional[str] = None
     name: Optional[str] = None
-    schema_metadata: Optional[SchemaSpecification] = Field(alias="schema")
+    schema_metadata: Optional[SchemaSpecification] = Field(default=None, alias="schema")
     downstreams: Optional[List[str]] = None
     properties: Optional[Dict[str, str]] = None
     subtype: Optional[str] = None
@@ -786,6 +786,7 @@ class Dataset(StrictModel):
         if schema_metadata:
             # If the schema is built off of an avro schema, we only extract the fields if they have structured properties
             # Otherwise, we extract all fields
+            schema_fields = []
             if (
                 schema_metadata.platformSchema
                 and isinstance(schema_metadata.platformSchema, models.OtherSchemaClass)

datahub/api/entities/external/external_entities.py

@@ -0,0 +1,239 @@
+import logging
+from abc import abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from typing import Iterable, List, Optional, Union
+
+import cachetools
+from pydantic import BaseModel
+
+from datahub.api.entities.platformresource.platform_resource import (
+    PlatformResource,
+    PlatformResourceKey,
+)
+from datahub.ingestion.graph.client import DataHubGraph
+from datahub.metadata.urns import PlatformResourceUrn, Urn
+from datahub.utilities.search_utils import ElasticDocumentQuery
+
+logger = logging.getLogger(__name__)
+
+
+class PlatformResourceRepository:
+    def __init__(self, graph: DataHubGraph):
+        self.graph = graph
+        self.cache: cachetools.TTLCache = cachetools.TTLCache(maxsize=1000, ttl=60 * 5)
+
+    def search_by_filter(
+        self, filter: ElasticDocumentQuery, add_to_cache: bool = True
+    ) -> Iterable[PlatformResource]:
+        results = PlatformResource.search_by_filters(self.graph, filter)
+        for platform_resource in results:
+            if add_to_cache:
+                self.cache[platform_resource.id] = platform_resource
+            yield platform_resource
+
+    def create(self, platform_resource: PlatformResource) -> None:
+        platform_resource.to_datahub(self.graph)
+        self.cache[platform_resource.id] = platform_resource
+
+    def get(self, key: PlatformResourceKey) -> Optional[PlatformResource]:
+        return self.cache.get(key.id)
+
+    def delete(self, key: PlatformResourceKey) -> None:
+        self.graph.delete_entity(urn=PlatformResourceUrn(key.id).urn(), hard=True)
+        del self.cache[key.id]
+
+
+class ExternalEntityId:
+    """
+    ExternalEntityId is a unique
+    identifier for an ExternalEntity.
+    """
+
+    @abstractmethod
+    def to_platform_resource_key(self) -> PlatformResourceKey:
+        """
+        Converts the ExternalEntityId to a PlatformResourceKey.
+        """
+        pass
+
+
+class CaseSensitivity(Enum):
+    UPPER = "upper"
+    LOWER = "lower"
+    MIXED = "mixed"
+
+    @staticmethod
+    def detect_case_sensitivity(value: str) -> "CaseSensitivity":
+        if value.isupper():
+            return CaseSensitivity.UPPER
+        elif value.islower():
+            return CaseSensitivity.LOWER
+        return CaseSensitivity.MIXED
+
+    @staticmethod
+    def detect_for_many(values: List[str]) -> "CaseSensitivity":
+        """
+        Detects the case sensitivity for a list of strings.
+        Returns CaseSensitivity.MIXED if the case sensitivity is mixed.
+        """
+        if len(values) == 0:
+            return CaseSensitivity.MIXED
+
+        if all(
+            CaseSensitivity.detect_case_sensitivity(value) == CaseSensitivity.UPPER
+            for value in values
+        ):
+            return CaseSensitivity.UPPER
+        elif all(
+            CaseSensitivity.detect_case_sensitivity(value) == CaseSensitivity.LOWER
+            for value in values
+        ):
+            return CaseSensitivity.LOWER
+        return CaseSensitivity.MIXED
+
+
+class LinkedResourceSet(BaseModel):
+    """
+    A LinkedResourceSet is a set of DataHub URNs that are linked to an ExternalEntity.
+    """
+
+    urns: List[str]
+
+    def _has_conflict(self, urn: Urn) -> bool:
+        """
+        Detects if the urn is safe to add into the set
+        This is used to detect conflicts between DataHub URNs that are linked to
+        the same ExternalEntity.
+        e.g. Case sensitivity of URNs
+        Mixing tags and terms in the same set etc.
+        Return True if the urn is not safe to add into the set, else False.
+        If the urn is already in the set, we don't need to add it again, but
+        that is not a conflict.
+        """
+        if urn.urn() in self.urns:
+            return False
+
+        # Detect the entity_type of the urns in the existing set
+        detected_entity_type = None
+        for existing_urn in self.urns:
+            try:
+                parsed_urn = Urn.from_string(existing_urn)
+                entity_type = parsed_urn.entity_type
+                if detected_entity_type is None:
+                    detected_entity_type = entity_type
+                elif detected_entity_type != entity_type:
+                    logger.warning(
+                        f"Detected entity_type {detected_entity_type} is not equals to {entity_type}"
+                    )
+                    return True
+            except ValueError:
+                # Not a valid URN
+                logger.warning(f"Invalid URN {existing_urn} in LinkedResourceSet")
+                return True
+        try:
+            parsed_urn = urn
+            if (
+                detected_entity_type is not None
+                and parsed_urn.entity_type != detected_entity_type
+            ):
+                logger.warning(
+                    f"Detected entity_type {detected_entity_type} is not equals to parsed_urn's entity_type: {parsed_urn.entity_type}"
+                )
+                return True
+        except ValueError:
+            # Not a valid URN
+            logger.warning(f"Invalid URN: {urn} in LinkedResourceSet")
+            return True
+        return False
+
+    def add(self, urn: Union[str, Urn]) -> bool:
+        """
+        Adds a URN to the set.
+        Returns True if the URN was added, False if it was already in the set.
+        Raises a ValueError if the URN is in conflict with the existing set.
+        """
+        # Deduplicate the URNs if we have somehow duplicate items from concurrent runs
+        self.urns = list(set(self.urns))
+        if isinstance(urn, str):
+            urn = Urn.from_string(urn)
+        if self._has_conflict(urn):
+            raise ValueError(f"Conflict detected when adding URN {urn} to the set")
+        if urn.urn() not in self.urns:
+            self.urns.append(urn.urn())
+            return True
+        return False
+
+
+class ExternalEntity:
+    """
+    An ExternalEntity is a representation of an entity that external to DataHub
+    but could be linked to one or more DataHub entities.
+    """
+
+    @abstractmethod
+    def is_managed_by_datahub(self) -> bool:
+        """
+        Returns whether the entity is managed by DataHub.
+        """
+        pass
+
+    @abstractmethod
+    def datahub_linked_resources(self) -> LinkedResourceSet:
+        """
+        Returns the URNs of the DataHub entities linked to the external entity.
+        Empty list if no linked entities.
+        """
+        pass
+
+    @abstractmethod
+    def as_platform_resource(self) -> PlatformResource:
+        """
+        Converts the ExternalEntity to a PlatformResource.
+        """
+        pass
+
+    @abstractmethod
+    def get_id(self) -> ExternalEntityId:
+        """
+        Returns the ExternalEntityId for the ExternalEntity.
+        """
+        pass
+
+
+@dataclass
+class MissingExternalEntity(ExternalEntity):
+    id: ExternalEntityId
+
+    def is_managed_by_datahub(self) -> bool:
+        return False
+
+    def datahub_linked_resources(self) -> LinkedResourceSet:
+        return LinkedResourceSet(urns=[])
+
+    def as_platform_resource(self) -> Optional[PlatformResource]:  # type: ignore[override]
+        return None
+
+    def get_id(self) -> ExternalEntityId:
+        return self.id
+
+
+class ExternalSystem:
+    @abstractmethod
+    def exists(self, external_entity_id: ExternalEntityId) -> bool:
+        """
+        Returns whether the ExternalEntityId exists in the external system.
+        """
+        pass
+
+    @abstractmethod
+    def get(
+        self,
+        external_entity_id: ExternalEntityId,
+        platform_resource_repository: PlatformResourceRepository,
+    ) -> Optional[ExternalEntity]:
+        """
+        Returns the ExternalEntity for the ExternalEntityId.
+        Uses the platform resource repository to enrich the ExternalEntity with DataHub URNs.
+        """
+        pass

datahub/api/entities/external/external_tag.py

@@ -0,0 +1,145 @@
+"""
+External Tags Module
+
+This module provides tag types that integrate with external systems like DataHub and Unity Catalog.
+It builds on top of RestrictedText to provide sanitized, truncated tag handling with original value preservation.
+
+Classes:
+    - ExternalTag: DataHub-compatible tag with key/value parsing from URNs
+
+Example Usage:
+    # DataHub Tags
+    tag = ExternalTag.from_urn("urn:li:tag:environment:production")
+    datahub_urn = tag.get_datahub_tag  # Returns TagUrn object or string
+
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional, Tuple, Union
+
+from pydantic import BaseModel
+
+from datahub.api.entities.external.restricted_text import RestrictedText
+from datahub.metadata.urns import TagUrn
+
+
+class ExternalTag(BaseModel):
+    """A tag type that parses DataHub Tag URNs into key-value pairs with RestrictedText properties."""
+
+    key: RestrictedText
+    value: Optional[RestrictedText] = None
+
+    def __init__(
+        self,
+        key: Optional[Union[str, RestrictedText]] = None,
+        value: Optional[Union[str, RestrictedText]] = None,
+        **data: Any,
+    ) -> None:
+        """
+        Initialize ExternalTag from either a DataHub Tag URN or explicit key/value.
+
+        Args:
+            key: Explicit key value (optional for Pydantic initialization)
+            value: Explicit value (optional)
+            **data: Additional Pydantic data
+        """
+        if key is not None:
+            # Direct initialization with key/value
+            processed_key = (
+                RestrictedText(key) if not isinstance(key, RestrictedText) else key
+            )
+            processed_value = None
+            if value is not None:
+                processed_value = (
+                    RestrictedText(value)
+                    if not isinstance(value, RestrictedText)
+                    else value
+                )
+
+            super().__init__(
+                key=processed_key,
+                value=processed_value,
+                **data,
+            )
+        else:
+            # Standard pydantic initialization
+            super().__init__(**data)
+
+    @staticmethod
+    def _parse_tag_name(tag_name: str) -> Tuple[str, Optional[str]]:
+        """
+        Parse tag name into key and optional value.
+
+        If tag_name contains ':', split on first ':' into key:value
+        Otherwise, use entire tag_name as key with no value.
+
+        Args:
+            tag_name: The tag name portion from the URN
+
+        Returns:
+            Tuple of (key, value) where value may be None
+        """
+        if ":" in tag_name:
+            parts = tag_name.split(":", 1)  # Split on first ':' only
+            return parts[0], parts[1]
+        else:
+            return tag_name, None
+
+    def to_datahub_tag_urn(self) -> TagUrn:
+        """
+        Generate a DataHub Tag URN from the key and value.
+        This method creates the URN using the original (unprocessed) values.
+
+        Returns:
+            'urn:li:tag:key:value' if value exists, otherwise 'urn:li:tag:key'
+        """
+        if self.value is not None:
+            tag_name = f"{self.key.original}:{self.value.original}"
+        else:
+            tag_name = self.key.original
+
+        return TagUrn(name=tag_name)
+
+    @classmethod
+    def from_urn(cls, tag_urn: Union[str, "TagUrn"]) -> "ExternalTag":
+        """
+        Create an ExternalTag from a DataHub Tag URN.
+
+        Args:
+            tag_urn: DataHub Tag URN string or TagUrn object
+
+        Returns:
+            ExternalTag instance
+        """
+        if isinstance(tag_urn, str):
+            tag_urn = TagUrn.from_string(tag_urn)
+        key, value = cls._parse_tag_name(tag_urn.name)
+        return cls(key=key, value=value)
+
+    @classmethod
+    def from_key_value(cls, key: str, value: Optional[str] = None) -> "ExternalTag":
+        """
+        Create an ExternalTag from explicit key and value.
+
+        Args:
+            key: Tag key
+            value: Optional tag value
+
+        Returns:
+            ExternalTag instance
+        """
+        return cls(key=key, value=value)
+
+    def __str__(self) -> str:
+        """String representation of the tag."""
+        if self.value is not None:
+            return f"{self.key}:{self.value}"
+        else:
+            return str(self.key)
+
+    def __repr__(self) -> str:
+        if self.value is not None:
+            return f"ExternalTag(key={self.key!r}, value={self.value!r})"
+        else:
+            return f"ExternalTag(key={self.key!r})"

datahub/api/entities/external/lake_formation_external_entites.py

@@ -0,0 +1,161 @@
+# Import RestrictedText from your existing module
+# Uncomment and adjust the import path as needed:
+# from your_restricted_text_module import RestrictedText
+# The following is a list of tag constraints:
+# You can assign a maximum of 50 tags to a single securable object.
+# The maximum length of a tag key is 255 characters.
+# The maximum length of a tag value is 1000 characters.
+# The following characters are not allowed in tag keys:
+# . , - = / :
+# Tag search using the workspace search UI is supported only for tables, views, and table columns.
+# Tag search requires exact term matching.
+# https://learn.microsoft.com/en-us/azure/databricks/database-objects/tags#constraint
+from typing import Any, Dict, Optional, Union
+
+from typing_extensions import ClassVar
+
+from datahub.api.entities.external.external_tag import ExternalTag
+from datahub.api.entities.external.restricted_text import RestrictedText
+
+
+class LakeFormationTagKeyText(RestrictedText):
+    """RestrictedText configured for Unity Catalog tag keys."""
+
+    _default_max_length: ClassVar[int] = 50
+    # Unity Catalog tag keys: alphanumeric, hyphens, underscores, periods only
+    _default_replacement_char: ClassVar[str] = "_"
+    _default_truncation_suffix: ClassVar[str] = ""  # No suffix for clean identifiers
+
+
+class LakeFormationTagValueText(RestrictedText):
+    """RestrictedText configured for Unity Catalog tag values."""
+
+    _default_max_length: ClassVar[int] = 50
+    # Unity Catalog tag values are more permissive but still have some restrictions
+    _default_replacement_char: ClassVar[str] = " "
+    _default_truncation_suffix: ClassVar[str] = "..."
+
+
+class LakeFormationTag(ExternalTag):
+    """
+    A tag type specifically designed for LakeFormation tag restrictions.
+
+    LakeFormation Tag Restrictions:
+    - Key: Max 127 characters, alphanumeric + hyphens, underscores, periods only
+    - Value: Max 256 characters, more permissive but no control characters
+    """
+
+    key: LakeFormationTagKeyText
+    value: Optional[LakeFormationTagValueText] = None
+    catalog: Optional[str] = None
+
+    def __init__(
+        self,
+        key: Optional[Union[str, LakeFormationTagKeyText]] = None,
+        value: Optional[Union[str, LakeFormationTagValueText]] = None,
+        **data: Any,
+    ) -> None:
+        """
+        Initialize LakeFormation Tag from either a DataHub Tag URN or explicit key/value.
+
+        Args:
+            key: Explicit key value (optional for Pydantic initialization)
+            value: Explicit value (optional)
+            **data: Additional Pydantic data
+        """
+        if key is not None:
+            # Direct initialization with key/value
+            processed_key = (
+                LakeFormationTagKeyText(key)
+                if not isinstance(key, LakeFormationTagKeyText)
+                else key
+            )
+            processed_value = None
+            if value is not None:
+                processed_value = (
+                    LakeFormationTagValueText(value)
+                    if not isinstance(value, LakeFormationTagValueText)
+                    else value
+                )
+
+            super().__init__(
+                key=processed_key,
+                value=processed_value,
+                **data,
+            )
+        else:
+            # Standard pydantic initialization
+            super().__init__(**data)
+
+    def __eq__(self, other: object) -> bool:
+        """Check equality based on key and value."""
+        if not isinstance(other, LakeFormationTag):
+            return False
+        return str(self.key) == str(other.key) and (
+            str(self.value) if self.value else None
+        ) == (str(other.value) if other.value else None)
+
+    def __hash__(self) -> int:
+        """Make LakeFormationTag hashable based on key and value."""
+        return hash((str(self.key), str(self.value) if self.value else None))
+
+    @classmethod
+    def from_dict(cls, tag_dict: Dict[str, Any]) -> "LakeFormationTag":
+        """
+        Create a LakeFormationTag from a dictionary with 'key' and optional 'value'.
+
+        Args:
+            tag_dict: Dictionary with 'key' and optional 'value' keys
+
+        Returns:
+            LakeFormationTag instance
+        """
+        return cls(key=tag_dict["key"], value=tag_dict.get("value"))
+
+    @classmethod
+    def from_key_value(
+        cls, key: str, value: Optional[str] = None
+    ) -> "LakeFormationTag":
+        """
+        Create a LakeFormationTagPlatformResource from explicit key and value.
+
+        Overrides the parent method to return the correct type.
+
+        Args:
+            key: Tag key
+            value: Optional tag value
+
+        Returns:
+            LakeFormationTag instance
+        """
+        return cls(key=key, value=value)
+
+    def to_dict(self) -> Dict[str, str]:
+        """
+        Convert to dictionary format suitable for LakeFormation tag.
+
+        Returns:
+            Dictionary with 'key' and optionally 'value'
+        """
+        result: Dict[str, str] = {"key": self.key.original}
+        if self.value is not None:
+            result["value"] = self.value.original
+        return result
+
+    def to_display_dict(self) -> Dict[str, str]:
+        """
+        Convert to dictionary format showing processed values.
+
+        Returns:
+            Dictionary with processed 'key' and optional 'value'
+        """
+        result: Dict[str, str] = {"key": str(self.key)}
+        if self.value is not None:
+            result["value"] = str(self.value)
+        return result
+
+    def __repr__(self) -> str:
+        if self.value:
+            return f"LakeFormationTag(key={self.key!r}, value={self.value!r})"
+        else:
+            return f"LakeFormationTag(key={self.key!r})"