acryl-datahub 1.2.0.4rc4__py3-none-any.whl → 1.2.0.5rc1__py3-none-any.whl

datahub/ingestion/source/aws/tag_entities.py

@@ -1,5 +1,10 @@
 import logging
-from typing import List, Optional
+from typing import TYPE_CHECKING, List, Optional
+
+if TYPE_CHECKING:
+    from datahub.ingestion.source.aws.platform_resource_repository import (
+        GluePlatformResourceRepository,
+    )
 
 from pydantic import BaseModel
 
@@ -7,7 +12,6 @@ from datahub.api.entities.external.external_entities import (
     ExternalEntity,
     ExternalEntityId,
     LinkedResourceSet,
-    PlatformResourceRepository,
 )
 from datahub.api.entities.external.lake_formation_external_entites import (
     LakeFormationTag,
@@ -15,10 +19,8 @@ from datahub.api.entities.external.lake_formation_external_entites import (
 from datahub.api.entities.platformresource.platform_resource import (
     PlatformResource,
     PlatformResourceKey,
-    PlatformResourceSearchFields,
 )
 from datahub.metadata.urns import TagUrn
-from datahub.utilities.search_utils import ElasticDocumentQuery
 from datahub.utilities.urns.urn import Urn
 
 logger = logging.getLogger(__name__)
@@ -29,8 +31,12 @@ class LakeFormationTagSyncContext(BaseModel):
     platform_instance: Optional[str] = None
     catalog: Optional[str] = None
 
+    # Making it compatible with SyncContext interface
+    def get_platform_instance(self) -> Optional[str]:
+        return self.platform_instance
+
 
-class LakeFormationTagPlatformResourceId(BaseModel, ExternalEntityId):
+class LakeFormationTagPlatformResourceId(ExternalEntityId):
     """
     A LakeFormationTag is a unique identifier for a Lakeformation tag.
     """
@@ -42,9 +48,6 @@ class LakeFormationTagPlatformResourceId(BaseModel, ExternalEntityId):
     exists_in_lake_formation: bool = False
     persisted: bool = False
 
-    def __hash__(self) -> int:
-        return hash(self.to_platform_resource_key().id)
-
     # this is a hack to make sure the property is a string and not private pydantic field
     @staticmethod
     def _RESOURCE_TYPE() -> str:
@@ -61,24 +64,26 @@ class LakeFormationTagPlatformResourceId(BaseModel, ExternalEntityId):
         )
 
     @classmethod
-    def from_tag(
+    def get_or_create_from_tag(
         cls,
         tag: LakeFormationTag,
-        platform_instance: Optional[str],
-        platform_resource_repository: PlatformResourceRepository,
-        catalog: Optional[str] = None,
+        platform_resource_repository: "GluePlatformResourceRepository",
         exists_in_lake_formation: bool = False,
+        catalog_id: Optional[str] = None,
     ) -> "LakeFormationTagPlatformResourceId":
         """
         Creates a LakeFormationTagPlatformResourceId from a LakeFormationTag.
         """
 
+        # Use catalog_id if provided, otherwise fall back to repository catalog
+        effective_catalog = catalog_id or platform_resource_repository.catalog
+
         existing_platform_resource = cls.search_by_urn(
             tag.to_datahub_tag_urn().urn(),
             platform_resource_repository=platform_resource_repository,
             tag_sync_context=LakeFormationTagSyncContext(
-                platform_instance=platform_instance,
-                catalog=catalog,
+                platform_instance=platform_resource_repository.platform_instance,
+                catalog=effective_catalog,
             ),
         )
         if existing_platform_resource:
@@ -90,9 +95,9 @@ class LakeFormationTagPlatformResourceId(BaseModel, ExternalEntityId):
         return LakeFormationTagPlatformResourceId(
             tag_key=str(tag.key),
             tag_value=str(tag.value) if tag.value is not None else None,
-            platform_instance=platform_instance,
+            platform_instance=platform_resource_repository.platform_instance,
+            catalog=effective_catalog,
             exists_in_lake_formation=exists_in_lake_formation,
-            catalog=catalog,
             persisted=False,
         )
 
@@ -100,64 +105,48 @@ class LakeFormationTagPlatformResourceId(BaseModel, ExternalEntityId):
     def search_by_urn(
         cls,
         urn: str,
-        platform_resource_repository: PlatformResourceRepository,
+        platform_resource_repository: "GluePlatformResourceRepository",
         tag_sync_context: LakeFormationTagSyncContext,
     ) -> Optional["LakeFormationTagPlatformResourceId"]:
-        mapped_tags = [
-            t
-            for t in platform_resource_repository.search_by_filter(
-                ElasticDocumentQuery.create_from(
-                    (
-                        PlatformResourceSearchFields.RESOURCE_TYPE,
-                        str(LakeFormationTagPlatformResourceId._RESOURCE_TYPE()),
-                    ),
-                    (PlatformResourceSearchFields.SECONDARY_KEYS, urn),
+        """
+        Search for existing Lake Formation tag entity by URN using repository caching.
+
+        This method now delegates to the repository's search_entity_by_urn method to ensure
+        consistent caching behavior across all platform implementations.
+        """
+        # Use repository's cached search method instead of duplicating search logic
+        existing_entity_id = platform_resource_repository.search_entity_by_urn(urn)
+
+        if existing_entity_id:
+            # Verify platform instance and catalog match
+            if (
+                existing_entity_id.platform_instance
+                == tag_sync_context.platform_instance
+                and existing_entity_id.catalog == tag_sync_context.catalog
+            ):
+                logger.info(
+                    f"Found existing LakeFormationTagPlatformResourceId for URN {urn}: {existing_entity_id}"
                 )
-            )
-        ]
+                # Create a new ID with the correct state instead of mutating
+                return LakeFormationTagPlatformResourceId(
+                    tag_key=existing_entity_id.tag_key,
+                    tag_value=existing_entity_id.tag_value,
+                    platform_instance=existing_entity_id.platform_instance,
+                    catalog=existing_entity_id.catalog,
+                    exists_in_lake_formation=True,  # This tag exists in Lake Formation
+                    persisted=True,  # And it's persisted in DataHub
+                )
+
         logger.info(
-            f"Found {len(mapped_tags)} mapped tags for URN {urn}. {mapped_tags}"
+            f"No mapped tag found for URN {urn} with platform instance {tag_sync_context.platform_instance}. Creating a new LakeFormationTagPlatformResourceId."
         )
-        if len(mapped_tags) > 0:
-            for platform_resource in mapped_tags:
-                if (
-                    platform_resource.resource_info
-                    and platform_resource.resource_info.value
-                ):
-                    lake_formation_tag_platform_resource = (
-                        LakeFormationTagPlatformResource(
-                            **platform_resource.resource_info.value.as_pydantic_object(
-                                LakeFormationTagPlatformResource
-                            ).dict()
-                        )
-                    )
-                    if (
-                        lake_formation_tag_platform_resource.id.platform_instance
-                        == tag_sync_context.platform_instance
-                        and lake_formation_tag_platform_resource.id.catalog
-                        == tag_sync_context.catalog
-                    ):
-                        lake_formation_tag_id = lake_formation_tag_platform_resource.id
-                        lake_formation_tag_id.exists_in_lake_formation = True
-                        lake_formation_tag_id.persisted = True
-                        return lake_formation_tag_id
-                else:
-                    logger.warning(
-                        f"Platform resource {platform_resource} does not have a resource_info value"
-                    )
-                    continue
-
-            # If we reach here, it means we did not find a mapped tag for the URN
-            logger.info(
-                f"No mapped tag found for URN {urn} with platform instance {tag_sync_context.platform_instance}. Creating a new LakeFormationTagPlatformResourceId."
-            )
         return None
 
     @classmethod
     def from_datahub_urn(
         cls,
         urn: str,
-        platform_resource_repository: PlatformResourceRepository,
+        platform_resource_repository: "GluePlatformResourceRepository",
         tag_sync_context: LakeFormationTagSyncContext,
     ) -> "LakeFormationTagPlatformResourceId":
         """
@@ -188,11 +177,17 @@ class LakeFormationTagPlatformResourceId(BaseModel, ExternalEntityId):
                 logger.info(
                     f"Tag {new_tag_id} already exists in platform resource repository with {resource_key}"
                 )
-                new_tag_id.exists_in_lake_formation = (
-                    True  # TODO: Check if this is a safe assumption
+                # Create a new ID with the correct state instead of mutating
+                return LakeFormationTagPlatformResourceId(
+                    tag_key=new_tag_id.tag_key,
+                    tag_value=new_tag_id.tag_value,
+                    platform_instance=new_tag_id.platform_instance,
+                    catalog=new_tag_id.catalog,
+                    exists_in_lake_formation=True,  # This tag exists in Lake Formation
+                    persisted=new_tag_id.persisted,
                 )
             return new_tag_id
-        raise ValueError(f"Unable to create SnowflakeTagId from DataHub URN: {urn}")
+        raise ValueError(f"Unable to create LakeFormationTagId from DataHub URN: {urn}")
 
     @classmethod
     def generate_tag_id(
@@ -223,7 +218,7 @@ class LakeFormationTagPlatformResourceId(BaseModel, ExternalEntityId):
         )
 
 
-class LakeFormationTagPlatformResource(BaseModel, ExternalEntity):
+class LakeFormationTagPlatformResource(ExternalEntity):
     datahub_urns: LinkedResourceSet
     managed_by_datahub: bool
     id: LakeFormationTagPlatformResourceId
@@ -246,46 +241,29 @@ class LakeFormationTagPlatformResource(BaseModel, ExternalEntity):
         )
 
     @classmethod
-    def get_from_datahub(
+    def create_default(
         cls,
-        lake_formation_tag_id: LakeFormationTagPlatformResourceId,
-        platform_resource_repository: PlatformResourceRepository,
-        managed_by_datahub: bool = False,
+        entity_id: ExternalEntityId,
+        managed_by_datahub: bool,
     ) -> "LakeFormationTagPlatformResource":
-        # Search for linked DataHub URNs
-        platform_resources = [
-            r
-            for r in platform_resource_repository.search_by_filter(
-                ElasticDocumentQuery.create_from(
-                    (
-                        PlatformResourceSearchFields.RESOURCE_TYPE,
-                        str(LakeFormationTagPlatformResourceId._RESOURCE_TYPE()),
-                    ),
-                    (
-                        PlatformResourceSearchFields.PRIMARY_KEY,
-                        f"{lake_formation_tag_id.tag_key}/{lake_formation_tag_id.tag_value}",
-                    ),
-                )
-            )
-        ]
-        for platform_resource in platform_resources:
-            if (
-                platform_resource.resource_info
-                and platform_resource.resource_info.value
-            ):
-                lf_tag = LakeFormationTagPlatformResource(
-                    **platform_resource.resource_info.value.as_pydantic_object(
-                        LakeFormationTagPlatformResource
-                    ).dict()
-                )
-                if (
-                    lf_tag.id.platform_instance
-                    == lake_formation_tag_id.platform_instance
-                    and lf_tag.id.catalog == lake_formation_tag_id.catalog
-                ):
-                    return lf_tag
+        """Create a default Lake Formation tag entity when none found in DataHub."""
+        # Type narrowing: we know this will be a LakeFormationTagPlatformResourceId
+        assert isinstance(entity_id, LakeFormationTagPlatformResourceId), (
+            f"Expected LakeFormationTagPlatformResourceId, got {type(entity_id)}"
+        )
+
+        # Create a new entity ID with correct default state instead of mutating
+        default_entity_id = LakeFormationTagPlatformResourceId(
+            tag_key=entity_id.tag_key,
+            tag_value=entity_id.tag_value,
+            platform_instance=entity_id.platform_instance,
+            catalog=entity_id.catalog,
+            exists_in_lake_formation=False,  # New entities don't exist in Lake Formation yet
+            persisted=False,  # New entities are not persisted yet
+        )
+
         return cls(
-            id=lake_formation_tag_id,
+            id=default_entity_id,
             datahub_urns=LinkedResourceSet(urns=[]),
             managed_by_datahub=managed_by_datahub,
             allowed_values=None,

datahub/ingestion/source/common/subtypes.py

@@ -30,6 +30,7 @@ class DatasetSubTypes(StrEnum):
     NEO4J_NODE = "Neo4j Node"
     NEO4J_RELATIONSHIP = "Neo4j Relationship"
     SNOWFLAKE_STREAM = "Snowflake Stream"
+    DYNAMIC_TABLE = "Dynamic Table"
     API_ENDPOINT = "API Endpoint"
     SLACK_CHANNEL = "Slack Channel"
     PROJECTIONS = "Projections"

datahub/ingestion/source/hex/api.py

@@ -375,6 +375,7 @@ class HexApi:
                 description=hex_item.description,
                 created_at=hex_item.created_at,
                 last_edited_at=hex_item.last_edited_at,
+                last_published_at=hex_item.last_published_at,
                 status=status,
                 categories=categories,
                 collections=collections,
@@ -389,6 +390,7 @@ class HexApi:
                 description=hex_item.description,
                 created_at=hex_item.created_at,
                 last_edited_at=hex_item.last_edited_at,
+                last_published_at=hex_item.last_published_at,
                 status=status,
                 categories=categories,
                 collections=collections,

datahub/ingestion/source/hex/mapper.py

@@ -122,7 +122,7 @@ class Mapper:
             lastModified=self._change_audit_stamps(
                 created_at=project.created_at, last_edited_at=project.last_edited_at
             ),
-            externalUrl=f"{self._base_url}/{self._workspace_name}/hex/{project.id}",
+            externalUrl=self._get_project_or_component_external_url(project),
             customProperties=dict(id=project.id),
             datasetEdges=self._dataset_edges(project.upstream_datasets),
             # TODO: support schema field upstream, maybe InputFields?
@@ -173,7 +173,7 @@ class Mapper:
             lastModified=self._change_audit_stamps(
                 created_at=component.created_at, last_edited_at=component.last_edited_at
             ),
-            externalUrl=f"{self._base_url}/{self._workspace_name}/hex/{component.id}",
+            externalUrl=self._get_project_or_component_external_url(component),
             customProperties=dict(id=component.id),
         )
 
@@ -242,6 +242,20 @@ class Mapper:
         assert isinstance(dashboard_urn, DashboardUrn)
         return dashboard_urn
 
+    def _get_project_or_component_external_url(
+        self,
+        project_or_component: Union[Project, Component],
+    ) -> Optional[str]:
+        if project_or_component.last_published_at is None:
+            return (
+                f"{self._base_url}/{self._workspace_name}/hex/{project_or_component.id}"
+            )
+        else:
+            # published Projects/Components have a different URL that everybody, not just editors, can access
+            return (
+                f"{self._base_url}/{self._workspace_name}/app/{project_or_component.id}"
+            )
+
     def _change_audit_stamps(
         self, created_at: Optional[datetime], last_edited_at: Optional[datetime]
     ) -> ChangeAuditStampsClass:

datahub/ingestion/source/hex/model.py

@@ -46,6 +46,7 @@ class Project:
     title: str
     description: Optional[str]
     last_edited_at: Optional[datetime] = None
+    last_published_at: Optional[datetime] = None
     created_at: Optional[datetime] = None
     status: Optional[Status] = None
     categories: Optional[List[Category]] = None  # TODO: emit category description!
@@ -67,6 +68,7 @@ class Component:
     title: str
     description: Optional[str]
     last_edited_at: Optional[datetime] = None
+    last_published_at: Optional[datetime] = None
     created_at: Optional[datetime] = None
     status: Optional[Status] = None
     categories: Optional[List[Category]] = None

datahub/ingestion/source/looker/looker_common.py

@@ -379,6 +379,14 @@ class ExploreUpstreamViewField:
                 : -(len(self.field.field_group_variant.lower()) + 1)
             ]
 
+        # Validate that field_name is not empty to prevent invalid schema field URNs
+        if not field_name or not field_name.strip():
+            logger.warning(
+                f"Empty field name detected for field '{self.field.name}' in explore '{self.explore.name}'. "
+                f"Skipping field to prevent invalid schema field URN generation."
+            )
+            return None
+
         assert view_name  # for lint false positive
 
         project_include: ProjectInclude = ProjectInclude(
@@ -1351,7 +1359,25 @@ class LookerExplore:
             fine_grained_lineages = []
             if config.extract_column_level_lineage:
                 for field in self.fields or []:
+                    # Skip creating fine-grained lineage for empty field names to prevent invalid schema field URNs
+                    if not field.name or not field.name.strip():
+                        logger.warning(
+                            f"Skipping fine-grained lineage for field with empty name in explore '{self.name}'"
+                        )
+                        continue
+
                     for upstream_column_ref in field.upstream_fields:
+                        # Skip creating fine-grained lineage for empty column names to prevent invalid schema field URNs
+                        if (
+                            not upstream_column_ref.column
+                            or not upstream_column_ref.column.strip()
+                        ):
+                            logger.warning(
+                                f"Skipping some fine-grained lineage for field '{field.name}' in explore '{self.name}' "
+                                f"due to empty upstream column name in table '{upstream_column_ref.table}'"
+                            )
+                            continue
+
                         fine_grained_lineages.append(
                             FineGrainedLineageClass(
                                 upstreamType=FineGrainedLineageUpstreamType.FIELD_SET,

datahub/ingestion/source/snowflake/constants.py

@@ -55,6 +55,7 @@ class SnowflakeObjectDomain(StrEnum):
     ICEBERG_TABLE = "iceberg table"
     STREAM = "stream"
     PROCEDURE = "procedure"
+    DYNAMIC_TABLE = "dynamic table"
 
 
 GENERIC_PERMISSION_ERROR_KEY = "permission-error"

datahub/ingestion/source/snowflake/snowflake_query.py

@@ -8,7 +8,7 @@ from datahub.ingestion.source.snowflake.snowflake_config import (
 )
 from datahub.utilities.prefix_batch_builder import PrefixGroup
 
-SHOW_VIEWS_MAX_PAGE_SIZE = 10000
+SHOW_COMMAND_MAX_PAGE_SIZE = 10000
 SHOW_STREAM_MAX_PAGE_SIZE = 10000
 
 
@@ -38,12 +38,23 @@ class SnowflakeQuery:
         SnowflakeObjectDomain.MATERIALIZED_VIEW.capitalize(),
         SnowflakeObjectDomain.ICEBERG_TABLE.capitalize(),
         SnowflakeObjectDomain.STREAM.capitalize(),
+        SnowflakeObjectDomain.DYNAMIC_TABLE.capitalize(),
     }
 
     ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER = "({})".format(
         ",".join(f"'{domain}'" for domain in ACCESS_HISTORY_TABLE_VIEW_DOMAINS)
     )
 
+    # Domains that can be downstream tables in lineage
+    DOWNSTREAM_TABLE_DOMAINS = {
+        SnowflakeObjectDomain.TABLE.capitalize(),
+        SnowflakeObjectDomain.DYNAMIC_TABLE.capitalize(),
+    }
+
+    DOWNSTREAM_TABLE_DOMAINS_FILTER = "({})".format(
+        ",".join(f"'{domain}'" for domain in DOWNSTREAM_TABLE_DOMAINS)
+    )
+
     @staticmethod
     def current_account() -> str:
         return "select CURRENT_ACCOUNT()"
@@ -235,7 +246,7 @@ class SnowflakeQuery:
     @staticmethod
     def show_views_for_database(
         db_name: str,
-        limit: int = SHOW_VIEWS_MAX_PAGE_SIZE,
+        limit: int = SHOW_COMMAND_MAX_PAGE_SIZE,
         view_pagination_marker: Optional[str] = None,
     ) -> str:
         # While there is an information_schema.views view, that only shows the view definition if the role
@@ -244,7 +255,7 @@ class SnowflakeQuery:
 
         # SHOW VIEWS can return a maximum of 10000 rows.
         # https://docs.snowflake.com/en/sql-reference/sql/show-views#usage-notes
-        assert limit <= SHOW_VIEWS_MAX_PAGE_SIZE
+        assert limit <= SHOW_COMMAND_MAX_PAGE_SIZE
 
         # To work around this, we paginate through the results using the FROM clause.
         from_clause = (
@@ -686,7 +697,7 @@ WHERE table_schema='{schema_name}' AND {extra_clause}"""
                 AND t.query_start_time >= to_timestamp_ltz({start_time_millis}, 3)
                 AND t.query_start_time < to_timestamp_ltz({end_time_millis}, 3)
                 AND upstream_table_domain in {allowed_upstream_table_domains}
-                AND downstream_table_domain = '{SnowflakeObjectDomain.TABLE.capitalize()}'
+                AND downstream_table_domain in {SnowflakeQuery.DOWNSTREAM_TABLE_DOMAINS_FILTER}
                 {("AND " + upstream_sql_filter) if upstream_sql_filter else ""}
             ),
         column_upstream_jobs AS (
@@ -843,7 +854,7 @@ WHERE table_schema='{schema_name}' AND {extra_clause}"""
                 AND t.query_start_time >= to_timestamp_ltz({start_time_millis}, 3)
                 AND t.query_start_time < to_timestamp_ltz({end_time_millis}, 3)
                 AND upstream_table_domain in {allowed_upstream_table_domains}
-                AND downstream_table_domain = '{SnowflakeObjectDomain.TABLE.capitalize()}'
+                AND downstream_table_domain in {SnowflakeQuery.DOWNSTREAM_TABLE_DOMAINS_FILTER}
                 {("AND " + upstream_sql_filter) if upstream_sql_filter else ""}
             ),
         table_upstream_jobs_unique AS (
@@ -940,3 +951,37 @@ WHERE table_schema='{schema_name}' AND {extra_clause}"""
             f"""FROM '{stream_pagination_marker}'""" if stream_pagination_marker else ""
         )
         return f"""SHOW STREAMS IN DATABASE "{db_name}" LIMIT {limit} {from_clause};"""
+
+    @staticmethod
+    def show_dynamic_tables_for_database(
+        db_name: str,
+        limit: int = SHOW_COMMAND_MAX_PAGE_SIZE,
+        dynamic_table_pagination_marker: Optional[str] = None,
+    ) -> str:
+        """Get dynamic table definitions using SHOW DYNAMIC TABLES."""
+        assert limit <= SHOW_COMMAND_MAX_PAGE_SIZE
+
+        from_clause = (
+            f"""FROM '{dynamic_table_pagination_marker}'"""
+            if dynamic_table_pagination_marker
+            else ""
+        )
+        return f"""\
+    SHOW DYNAMIC TABLES IN DATABASE "{db_name}"
+    LIMIT {limit} {from_clause};
+    """
+
+    @staticmethod
+    def get_dynamic_table_graph_history(db_name: str) -> str:
+        """Get dynamic table dependency information from information schema."""
+        return f"""
+            SELECT   
+                name,
+                inputs,
+                target_lag_type,
+                target_lag_sec,
+                scheduling_state,
+                alter_trigger
+            FROM TABLE("{db_name}".INFORMATION_SCHEMA.DYNAMIC_TABLE_GRAPH_HISTORY())
+            ORDER BY name
+        """