acryl-datahub 1.2.0.5rc1__py3-none-any.whl → 1.2.0.6rc1__py3-none-any.whl

{acryl_datahub-1.2.0.5rc1.dist-info → acryl_datahub-1.2.0.6rc1.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.2.0.5rc1.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.2.0.6rc1.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=KSlHYPz315hvUomSrL-xCymHj7Nx74tIb2-JrBfK6_s,323
+datahub/_version.py,sha256=UpTxFRBPZj7BDFXx0obVG8eaUPk2TF8H9bqlPMVnt2A,323
 datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -148,7 +148,7 @@ datahub/ingestion/api/incremental_properties_helper.py,sha256=KzdxdrQtaMV2XMHfPs
 datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py,sha256=3lLdkkxVqE9MVc26cdXImPeWy16az5BwgcorWxeBV50,1759
 datahub/ingestion/api/pipeline_run_listener.py,sha256=5uBP__LbMQxJ2utlf07cIzQINqPbUOKiZyOJta6a0og,713
 datahub/ingestion/api/registry.py,sha256=LbdZr89465Lj7ptQRVB4vI1JR1igWABvQFj9-WX63bI,7454
-datahub/ingestion/api/report.py,sha256=jdAfqkAn1tlwBJBmC_t0-6KzKhKBxO8N3y_NziyBXSU,17889
+datahub/ingestion/api/report.py,sha256=-xduHhIRUgf5G51mUb3uTi6GBxVli6ZK25AS5ikXuII,18312
 datahub/ingestion/api/report_helpers.py,sha256=WbUC1kQeaKqIagGV3XzfPmPs7slAT1mfNY4og2BH2A8,994
 datahub/ingestion/api/sink.py,sha256=GZt48PV56FAhNoma-V5EwwRZvezhb40YH_zprm8_Yo0,4961
 datahub/ingestion/api/source.py,sha256=uf0fNbiOy0bS_aKFOcNv6NvuZe0LSDIDdNza9hraP7s,21857
@@ -400,7 +400,7 @@ datahub/ingestion/source/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 datahub/ingestion/source/metadata/business_glossary.py,sha256=T_RJHst6iQRghJNmLLPeSBMEDsbEKf3yBldOAgMcGuo,19666
 datahub/ingestion/source/metadata/lineage.py,sha256=PA4JwSeQ-30XFMN4O5tPwIu-hZF1e-xMZ_CnEUE2c-Q,9595
 datahub/ingestion/source/mock_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=jVJjqOzBN9JteLxcFRklg2eG9RmVdEozCxFYPArzjX8,19361
+datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=3i3SdBp267cZRszhmD_JWJLTGIot2FI8REFpjJQ4jD8,19822
 datahub/ingestion/source/mock_data/datahub_mock_data_report.py,sha256=sV_H7JgcuVbrpIBqtGse_BBigMdqP32ZXuanpeXmwVI,331
 datahub/ingestion/source/mock_data/table_naming_helper.py,sha256=zJtEBSJGDvVr-kiKjK7LbHAifK3sfE786M3yO--Bn2o,3493
 datahub/ingestion/source/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -481,12 +481,12 @@ datahub/ingestion/source/snowflake/constants.py,sha256=prPRPVgDajTeEbPhXB6a9gVW5
 datahub/ingestion/source/snowflake/oauth_config.py,sha256=ol9D3RmruGStJAeL8PYSQguSqcD2HfkjPkMF2AB_eZs,1277
 datahub/ingestion/source/snowflake/oauth_generator.py,sha256=fu2VnREGuJXeTqIV2jx4TwieVnznf83HQkrE0h2DGGM,3423
 datahub/ingestion/source/snowflake/snowflake_assertion.py,sha256=_l3k4aI9wvioE81xxdeizJn9nJCZ_nMIXgk9N6pEk5o,4803
-datahub/ingestion/source/snowflake/snowflake_config.py,sha256=6uR7eZDmEfLpfs_QYNJGTTZI6N2VOvjqXKmG_7c6-9Q,22291
+datahub/ingestion/source/snowflake/snowflake_config.py,sha256=3oakxSKMoZRw6J-15mIIXzutypvdCnwPZG3wvKEobh0,22825
 datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=3-nP3HHCblUnUHYo_fvFp5VOAteCtR4GNjaUEvyNTNQ,18175
 datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
 datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=a2vDWZNthV3AqD3Y_Rd4lHSZbaiyuEc--WXWcJqcV0k,21711
 datahub/ingestion/source/snowflake/snowflake_profiler.py,sha256=PmQi-qDlRhdJ-PsJ7x-EScIiswWRAxDDOKHydvN3mTY,7404
-datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=deJwFf9Bg3bphMi6WNfPon59hFSLYJZ4ByQA4pUkTkU,42926
+datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=Oie4qRiKfqyn80nwr2c0unjdOnOV-GSefgR9gV0B1tI,44865
 datahub/ingestion/source/snowflake/snowflake_query.py,sha256=iMIbT5iQzy6mxPLNkjbup_C6sLa2482K-QI7vaVZ8Ac,39677
 datahub/ingestion/source/snowflake/snowflake_report.py,sha256=O-465aBA8uaYZ6WepP7i6cgK6Q1jXJPjDA1j9C8klus,6762
 datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=x1gN8dmZZlNOANju0RnGQWqtBpm5UaNJDlGHBFL2LsE,33963
@@ -496,7 +496,7 @@ datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=WJfsP8w3HceUkM6GK
 datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=eA9xh-G1Ydr1OwUUtrbXUWp26hE1jF0zvyKNky_i_nQ,8887
 datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=mM0v9b4PHRJAT-SdRids3wdzc5O96gWCCww3e42itV8,24982
 datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=kL9W_PrJuP8QPgCe9dSjWkMvDzW2EcJciLQRSvrkGiI,14346
-datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=lUSAAZ3FzJEgLpLYfoLzCigMegbS5RClTg4Dhd4WxRI,35334
+datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=bIjbw6oI3zyGSBxhLmtpylE1rc2nyzE02xgvME2g0qQ,35421
 datahub/ingestion/source/snowflake/stored_proc_lineage.py,sha256=rOb78iHiWiK8v8WdVs1xDwVut4Y0OHmszej6IopQfCo,5341
 datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/sql/athena.py,sha256=TPKwL9oRiZlVnqIsOSBWUEwyvoW-1ssXvY4PfjxOR6g,28175
@@ -1104,8 +1104,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.2.0.5rc1.dist-info/METADATA,sha256=vAftVMr-wWAhe89fUtDvsCm9BjFdT92jn4Ea4-cSPV4,182056
-acryl_datahub-1.2.0.5rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.2.0.5rc1.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
-acryl_datahub-1.2.0.5rc1.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.2.0.5rc1.dist-info/RECORD,,
+acryl_datahub-1.2.0.6rc1.dist-info/METADATA,sha256=d_JiiaPjg1a5n4u__cmZ1gBZ7FpY1qf6flPFgzZM2D8,182056
+acryl_datahub-1.2.0.6rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.2.0.6rc1.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
+acryl_datahub-1.2.0.6rc1.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.2.0.6rc1.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.2.0.5rc1"
+__version__ = "1.2.0.6rc1"
 
 
 def is_dev_mode() -> bool:

datahub/ingestion/api/report.py

@@ -23,6 +23,7 @@ from datahub.ingestion.autogenerated.lineage_helper import is_lineage_aspect
 from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
 from datahub.metadata.schema_classes import (
     MetadataChangeProposalClass,
+    StatusClass,
     SubTypesClass,
     UpstreamLineageClass,
 )
@@ -161,6 +162,7 @@ class SourceReportSubtypes:
     entity_type: str
     subType: str = field(default="unknown")
     aspects: Dict[str, int] = field(default_factory=dict)
+    soft_deleted: bool = field(default=False)
 
 
 class ReportAttribute(BaseModel):
@@ -217,6 +219,7 @@ class ExamplesReport(Report, Closeable):
                 "entityType": lambda val: val.entity_type,
                 "subTypes": lambda val: val.subType,
                 "aspects": lambda val: json.dumps(val.aspects),
+                "soft_deleted": lambda val: val.soft_deleted,
             },
         )
 
@@ -376,6 +379,13 @@ class ExamplesReport(Report, Closeable):
                 subType=sub_type,
                 aspects=aspects_dict,
             )
+        if (
+            isinstance(mcp.aspect, StatusClass)
+            and mcp is not None
+            and mcp.aspect is not None
+        ):
+            self._file_based_dict[urn].soft_deleted = mcp.aspect.removed
+            self._file_based_dict.mark_dirty(urn)
 
     def _store_workunit_data(self, wu: MetadataWorkUnit) -> None:
         urn = wu.get_urn()
@@ -401,7 +411,8 @@ class ExamplesReport(Report, Closeable):
         query = """
         SELECT entityType, subTypes, aspects, count(*) as count
         FROM urn_aspects 
-        group by entityType, subTypes, aspects
+        WHERE soft_deleted = 0
+        GROUP BY entityType, subTypes, aspects
         """
 
         entity_subtype_aspect_counts: Dict[str, Dict[str, Dict[str, int]]] = (
@@ -410,6 +421,7 @@ class ExamplesReport(Report, Closeable):
         entity_subtype_aspect_counts_exist: Dict[str, Dict[str, Dict[str, int]]] = (
             defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
         )
+
         for row in self._file_based_dict.sql_query(query):
             entity_type = row["entityType"]
             sub_type = row["subTypes"]

datahub/ingestion/source/mock_data/datahub_mock_data.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, Iterable, List, Optional, Tuple
+from typing import Dict, Iterable, List, Optional, Tuple, Union
 
 from pydantic import Field
 
@@ -14,6 +14,7 @@ from datahub.ingestion.api.decorators import (
     support_status,
 )
 from datahub.ingestion.api.source import Source, SourceReport, StructuredLogCategory
+from datahub.ingestion.api.source_helpers import AutoSystemMetadata, auto_workunit
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import DatasetSubTypes
 from datahub.ingestion.source.mock_data.datahub_mock_data_report import (
@@ -31,6 +32,7 @@ from datahub.metadata.schema_classes import (
     UpstreamClass,
     UpstreamLineageClass,
 )
+from datahub.sdk.entity import Entity
 from datahub.utilities.str_enum import StrEnum
 
 logger = logging.getLogger(__name__)
@@ -165,6 +167,14 @@ class DataHubMockDataSource(Source):
         self.report = DataHubMockDataReport()
 
     def get_workunits(self) -> Iterable[MetadataWorkUnit]:
+        workunit_processors = [AutoSystemMetadata(self.ctx).stamp]
+        return self._apply_workunit_processors(
+            workunit_processors, auto_workunit(self.get_workunits_internal())
+        )
+
+    def get_workunits_internal(
+        self,
+    ) -> Iterable[Union[MetadataWorkUnit, MetadataChangeProposalWrapper, Entity]]:
         # We don't want any implicit aspects to be produced
         # so we are not using get_workunits_internal
 

datahub/ingestion/source/snowflake/snowflake_config.py

@@ -356,11 +356,18 @@ class SnowflakeV2Config(
 
     pushdown_deny_usernames: List[str] = Field(
         default=[],
-        description="List of snowflake usernames which will not be considered for lineage/usage/queries extraction. "
+        description="List of snowflake usernames (SQL LIKE patterns, e.g., 'SERVICE_%', '%_PROD', 'TEST_USER') which will NOT be considered for lineage/usage/queries extraction. "
         "This is primarily useful for improving performance by filtering out users with extremely high query volumes. "
         "Only applicable if `use_queries_v2` is enabled.",
     )
 
+    pushdown_allow_usernames: List[str] = Field(
+        default=[],
+        description="List of snowflake usernames (SQL LIKE patterns, e.g., 'ANALYST_%', '%_USER', 'MAIN_ACCOUNT') which WILL be considered for lineage/usage/queries extraction. "
+        "This is primarily useful for improving performance by filtering in only specific users. "
+        "Only applicable if `use_queries_v2` is enabled. If not specified, all users not in deny list are included.",
+    )
+
     push_down_database_pattern_access_history: bool = Field(
         default=False,
         description="If enabled, pushes down database pattern filtering to the access_history table for improved performance. "

datahub/ingestion/source/snowflake/snowflake_queries.py

@@ -89,10 +89,17 @@ class SnowflakeQueriesExtractorConfig(ConfigModel):
 
     pushdown_deny_usernames: List[str] = pydantic.Field(
         default=[],
-        description="List of snowflake usernames which will not be considered for lineage/usage/queries extraction. "
+        description="List of snowflake usernames (SQL LIKE patterns, e.g., 'SERVICE_%', '%_PROD', 'TEST_USER') which will NOT be considered for lineage/usage/queries extraction. "
         "This is primarily useful for improving performance by filtering out users with extremely high query volumes.",
     )
 
+    pushdown_allow_usernames: List[str] = pydantic.Field(
+        default=[],
+        description="List of snowflake usernames (SQL LIKE patterns, e.g., 'ANALYST_%', '%_USER', 'MAIN_ACCOUNT') which WILL be considered for lineage/usage/queries extraction. "
+        "This is primarily useful for improving performance by filtering in only specific users. "
+        "If not specified, all users not in deny list are included.",
+    )
+
     user_email_pattern: AllowDenyPattern = pydantic.Field(
         default=AllowDenyPattern.allow_all(),
         description="Regex patterns for user emails to filter in usage.",
@@ -396,6 +403,7 @@ class SnowflakeQueriesExtractor(SnowflakeStructuredReportMixin, Closeable):
             end_time=self.config.window.end_time,
             bucket_duration=self.config.window.bucket_duration,
             deny_usernames=self.config.pushdown_deny_usernames,
+            allow_usernames=self.config.pushdown_allow_usernames,
             dedup_strategy=self.config.query_dedup_strategy,
             database_pattern=self.filters.filter_config.database_pattern
             if self.config.push_down_database_pattern_access_history
@@ -740,7 +748,8 @@ class QueryLogQueryBuilder:
         start_time: datetime,
         end_time: datetime,
         bucket_duration: BucketDuration,
-        deny_usernames: Optional[List[str]],
+        deny_usernames: Optional[List[str]] = None,
+        allow_usernames: Optional[List[str]] = None,
         max_tables_per_query: int = 20,
         dedup_strategy: QueryDedupStrategyType = QueryDedupStrategyType.STANDARD,
         database_pattern: Optional[AllowDenyPattern] = None,
@@ -753,10 +762,7 @@ class QueryLogQueryBuilder:
         self.max_tables_per_query = max_tables_per_query
         self.dedup_strategy = dedup_strategy
 
-        self.users_filter = "TRUE"
-        if deny_usernames:
-            user_not_in = ",".join(f"'{user.upper()}'" for user in deny_usernames)
-            self.users_filter = f"user_name NOT IN ({user_not_in})"
+        self.users_filter = self._build_user_filter(deny_usernames, allow_usernames)
 
         self.access_history_database_filter = (
             self._build_access_history_database_filter_condition(
@@ -767,6 +773,43 @@ class QueryLogQueryBuilder:
         self.time_bucket_size = bucket_duration.value
         assert self.time_bucket_size in ("HOUR", "DAY", "MONTH")
 
+    def _build_user_filter(
+        self,
+        deny_usernames: Optional[List[str]] = None,
+        allow_usernames: Optional[List[str]] = None,
+    ) -> str:
+        """
+        Build user filter SQL condition based on deny and allow username patterns.
+
+        Args:
+            deny_usernames: List of username patterns to exclude (SQL LIKE patterns)
+            allow_usernames: List of username patterns to include (SQL LIKE patterns)
+
+        Returns:
+            SQL WHERE condition string for filtering users
+        """
+        user_filters = []
+
+        if deny_usernames:
+            deny_conditions = []
+            for pattern in deny_usernames:
+                # Escape single quotes for SQL safety
+                escaped_pattern = pattern.replace("'", "''")
+                deny_conditions.append(f"user_name NOT ILIKE '{escaped_pattern}'")
+            if deny_conditions:
+                user_filters.append(f"({' AND '.join(deny_conditions)})")
+
+        if allow_usernames:
+            allow_conditions = []
+            for pattern in allow_usernames:
+                # Escape single quotes for SQL safety
+                escaped_pattern = pattern.replace("'", "''")
+                allow_conditions.append(f"user_name ILIKE '{escaped_pattern}'")
+            if allow_conditions:
+                user_filters.append(f"({' OR '.join(allow_conditions)})")
+
+        return " AND ".join(user_filters) if user_filters else "TRUE"
+
     def _build_access_history_database_filter_condition(
         self,
         database_pattern: Optional[AllowDenyPattern],

datahub/ingestion/source/snowflake/snowflake_v2.py

@@ -602,6 +602,7 @@ class SnowflakeV2Source(
                         include_query_usage_statistics=self.config.include_query_usage_statistics,
                         user_email_pattern=self.config.user_email_pattern,
                         pushdown_deny_usernames=self.config.pushdown_deny_usernames,
+                        pushdown_allow_usernames=self.config.pushdown_allow_usernames,
                         query_dedup_strategy=self.config.query_dedup_strategy,
                         push_down_database_pattern_access_history=self.config.push_down_database_pattern_access_history,
                         additional_database_names_allowlist=self.config.additional_database_names_allowlist,