acryl-datahub 1.2.0.3rc2__py3-none-any.whl → 1.2.0.4rc2__py3-none-any.whl

datahub/ingestion/source/looker/looker_liquid_tag.py

@@ -1,5 +1,5 @@
 from functools import lru_cache
-from typing import ClassVar, Optional, TextIO
+from typing import ClassVar, Optional, TextIO, Type
 
 from liquid import Environment
 from liquid.ast import Node
@@ -20,16 +20,27 @@ class CustomTagException(Exception):
 class ConditionNode(Node):
     def __init__(self, tok: Token, sql_or_lookml_reference: str, filter_name: str):
         self.tok = tok
-
         self.sql_or_lookml_reference = sql_or_lookml_reference
-
         self.filter_name = filter_name
 
     def render_to_output(self, context: Context, buffer: TextIO) -> Optional[bool]:
         # This implementation will make sure that sql parse work correctly if looker condition tag
         # is used in lookml sql field
         buffer.write(f"{self.sql_or_lookml_reference}='dummy_value'")
+        return True
 
+
+class IncrementConditionNode(Node):
+    def __init__(self, tok: Token, sql_or_lookml_reference: str):
+        self.tok = tok
+        self.sql_or_lookml_reference = sql_or_lookml_reference
+
+    def render_to_output(self, context: Context, buffer: TextIO) -> Optional[bool]:
+        # For incrementcondition, we need to generate a condition that would be used
+        # in incremental PDT updates. This typically involves date/time comparisons.
+        # We'll render it as a date comparison with a placeholder value
+        # See details in Looker documentation for incrementcondition tag -> cloud.google.com/looker/docs/reference/param-view-increment-key
+        buffer.write(f"{self.sql_or_lookml_reference} > '2023-01-01'")
         return True
 
 
@@ -44,7 +55,6 @@ class ConditionTag(Tag):
     This class render the below tag as order.region='ap-south-1' if order_region is provided in config.liquid_variables
     as order_region: 'ap-south-1'
         {% condition order_region %} order.region {% endcondition %}
-
     """
 
     TAG_START: ClassVar[str] = "condition"
@@ -79,7 +89,48 @@ class ConditionTag(Tag):
         )
 
 
-custom_tags = [ConditionTag]
+class IncrementConditionTag(Tag):
+    """
+    IncrementConditionTag is the equivalent implementation of looker's custom liquid tag "incrementcondition".
+    Refer doc: https://cloud.google.com/looker/docs/incremental-pdts#using_the_incrementcondition_tag
+
+    This tag is used for incremental PDTs to determine which records should be updated.
+    It typically works with date/time fields to filter data that has changed since the last update.
+
+    Example usage in Looker:
+        {% incrementcondition created_at %} order.created_at {% endincrementcondition %}
+
+    This would generate SQL like: order.created_at > '2023-01-01 00:00:00'
+    """
+
+    TAG_START: ClassVar[str] = "incrementcondition"
+    TAG_END: ClassVar[str] = "endincrementcondition"
+    name: str = "incrementcondition"
+
+    def __init__(self, env: Environment):
+        super().__init__(env)
+        self.parser = get_parser(self.env)
+
+    def parse(self, stream: TokenStream) -> Node:
+        expect(stream, TOKEN_TAG, value=IncrementConditionTag.TAG_START)
+
+        start_token = stream.current
+
+        stream.next_token()
+        expect(stream, TOKEN_LITERAL)
+        sql_or_lookml_reference: str = stream.current.value.strip()
+
+        stream.next_token()
+        expect(stream, TOKEN_TAG, value=IncrementConditionTag.TAG_END)
+
+        return IncrementConditionNode(
+            tok=start_token,
+            sql_or_lookml_reference=sql_or_lookml_reference,
+        )
+
+
+# Updated custom_tags list to include both tags
+custom_tags: list[Type[Tag]] = [ConditionTag, IncrementConditionTag]
 
 
 @string_filter

datahub/ingestion/source/mock_data/datahub_mock_data.py

@@ -13,7 +13,7 @@ from datahub.ingestion.api.decorators import (
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.source import Source, SourceReport
+from datahub.ingestion.api.source import Source, SourceReport, StructuredLogCategory
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import DatasetSubTypes
 from datahub.ingestion.source.mock_data.datahub_mock_data_report import (
@@ -35,6 +35,8 @@ from datahub.utilities.str_enum import StrEnum
 
 logger = logging.getLogger(__name__)
 
+PLATFORM_NAME = "fake"
+
 
 class SubTypePattern(StrEnum):
     ALTERNATING = "alternating"
@@ -137,6 +139,10 @@ class DataHubMockDataConfig(ConfigModel):
         default=0,
         description="Number of warnings to add in report for testing",
     )
+    num_info: int = Field(
+        default=0,
+        description="Number of info to add in report for testing",
+    )
 
     gen_1: LineageConfigGen1 = Field(
         default_factory=LineageConfigGen1,
@@ -144,7 +150,7 @@ class DataHubMockDataConfig(ConfigModel):
     )
 
 
-@platform_name("DataHubMockData")
+@platform_name(PLATFORM_NAME)
 @config_class(DataHubMockDataConfig)
 @support_status(SupportStatus.TESTING)
 class DataHubMockDataSource(Source):
@@ -159,6 +165,9 @@ class DataHubMockDataSource(Source):
         self.report = DataHubMockDataReport()
 
     def get_workunits(self) -> Iterable[MetadataWorkUnit]:
+        # We don't want any implicit aspects to be produced
+        # so we are not using get_workunits_internal
+
         if self.config.throw_uncaught_exceptions:
             raise Exception("This is a test exception")
 
@@ -176,10 +185,17 @@ class DataHubMockDataSource(Source):
                     message="This is test warning",
                     title="Test Warning",
                     context=f"This is test warning {i}",
+                    log_category=StructuredLogCategory.LINEAGE,
+                )
+
+        if self.config.num_info > 0:
+            for i in range(self.config.num_info):
+                self.report.info(
+                    message="This is test info",
+                    title="Test Info",
+                    context=f"This is test info {i}",
                 )
 
-        # We don't want any implicit aspects to be produced
-        # so we are not using get_workunits_internal
         if self.config.gen_1.enabled:
             for wu in self._data_gen_1():
                 if self.report.first_urn_seen is None:
@@ -309,7 +325,7 @@ class DataHubMockDataSource(Source):
             table_level, table_index, subtype_pattern, subtype_types, level_subtypes
         )
 
-        urn = make_dataset_urn(platform="fake", name=table_name)
+        urn = make_dataset_urn(platform=PLATFORM_NAME, name=table_name)
         mcp = MetadataChangeProposalWrapper(
             entityUrn=urn,
             entityType="dataset",
@@ -433,7 +449,7 @@ class DataHubMockDataSource(Source):
 
     def _get_status_aspect(self, table: str) -> MetadataWorkUnit:
         urn = make_dataset_urn(
-            platform="fake",
+            platform=PLATFORM_NAME,
             name=table,
         )
         mcp = MetadataChangeProposalWrapper(
@@ -448,7 +464,7 @@ class DataHubMockDataSource(Source):
     ) -> MetadataWorkUnit:
         mcp = MetadataChangeProposalWrapper(
             entityUrn=make_dataset_urn(
-                platform="fake",
+                platform=PLATFORM_NAME,
                 name=downstream_table,
             ),
             entityType="dataset",
@@ -456,7 +472,7 @@ class DataHubMockDataSource(Source):
                 upstreams=[
                     UpstreamClass(
                         dataset=make_dataset_urn(
-                            platform="fake",
+                            platform=PLATFORM_NAME,
                             name=upstream_table,
                         ),
                         type=DatasetLineageTypeClass.TRANSFORMED,
@@ -468,7 +484,7 @@ class DataHubMockDataSource(Source):
 
     def _get_profile_aspect(self, table: str) -> MetadataWorkUnit:
         urn = make_dataset_urn(
-            platform="fake",
+            platform=PLATFORM_NAME,
             name=table,
         )
         mcp = MetadataChangeProposalWrapper(
@@ -485,7 +501,7 @@ class DataHubMockDataSource(Source):
 
     def _get_usage_aspect(self, table: str) -> MetadataWorkUnit:
         urn = make_dataset_urn(
-            platform="fake",
+            platform=PLATFORM_NAME,
             name=table,
         )
         mcp = MetadataChangeProposalWrapper(

datahub/ingestion/source/powerbi/powerbi.py

@@ -1226,7 +1226,10 @@ class Mapper:
 @platform_name("PowerBI")
 @config_class(PowerBiDashboardSourceConfig)
 @support_status(SupportStatus.CERTIFIED)
-@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(
+    SourceCapability.CONTAINERS,
+    "Enabled by default",
+)
 @capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
 @capability(SourceCapability.OWNERSHIP, "Enabled by default")
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")

datahub/ingestion/source/powerbi_report_server/report_server_domain.py

@@ -27,10 +27,8 @@ class CatalogItem(BaseModel):
     is_favorite: bool = Field(alias="IsFavorite")
     user_info: Any = Field(None, alias="UserInfo")
     display_name: Optional[str] = Field(None, alias="DisplayName")
-    has_data_sources: bool = Field(default=False, alias="HasDataSources")
-    data_sources: Optional[List["DataSource"]] = Field(
-        default_factory=list, alias="DataSources"
-    )
+    has_data_sources: bool = Field(False, alias="HasDataSources")
+    data_sources: Optional[List["DataSource"]] = Field(None, alias="DataSources")
 
     @validator("display_name", always=True)
     def validate_diplay_name(cls, value, values):

datahub/ingestion/source/redshift/datashares.py

@@ -26,7 +26,7 @@ from datahub.utilities.search_utils import LogicalOperator
 
 class OutboundSharePlatformResource(BaseModel):
     namespace: str
-    platform_instance: Optional[str]
+    platform_instance: Optional[str] = None
     env: str
     source_database: str
     share_name: str

datahub/ingestion/source/redshift/redshift.py

@@ -132,6 +132,7 @@ logger: logging.Logger = logging.getLogger(__name__)
     "Enabled by default",
     subtype_modifier=[
         SourceCapabilityModifier.DATABASE,
+        SourceCapabilityModifier.SCHEMA,
     ],
 )
 @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")

datahub/ingestion/source/salesforce.py

@@ -549,6 +549,14 @@ class SalesforceApi:
     capability_name=SourceCapability.TAGS,
     description="Enabled by default",
 )
+@capability(
+    capability_name=SourceCapability.LINEAGE_COARSE,
+    description="Extract table-level lineage for Salesforce objects",
+    subtype_modifier=[
+        SourceCapabilityModifier.SALESFORCE_CUSTOM_OBJECT,
+        SourceCapabilityModifier.SALESFORCE_STANDARD_OBJECT,
+    ],
+)
 class SalesforceSource(StatefulIngestionSourceBase):
     def __init__(self, config: SalesforceConfig, ctx: PipelineContext) -> None:
         super().__init__(config, ctx)

datahub/ingestion/source/slack/slack.py

@@ -203,38 +203,31 @@ class SlackSourceConfig(
         description="Bot token for the Slack workspace. Needs `users:read`, `users:read.email`, `users.profile:read`, and `team:read` scopes.",
     )
     enrich_user_metadata: bool = Field(
-        type=bool,
-        default=True,
+        True,
         description="When enabled, will enrich provisioned DataHub users' metadata with information from Slack.",
     )
     ingest_users: bool = Field(
-        type=bool,
-        default=True,
+        True,
         description="Whether to ingest users. When set to true, will ingest all users in the Slack workspace (as platform resources) to simplify user enrichment after they are provisioned on DataHub.",
     )
     api_requests_per_min: int = Field(
-        type=int,
-        default=10,
+        10,
         description="Number of API requests per minute. Low-level config. Do not tweak unless you are facing any issues.",
     )
     ingest_public_channels: bool = Field(
-        type=bool,
-        default=False,
+        False,
         description="Whether to ingest public channels. If set to true needs `channels:read` scope.",
     )
     channels_iteration_limit: int = Field(
-        type=int,
-        default=200,
+        200,
         description="Limit the number of channels to be ingested in a iteration. Low-level config. Do not tweak unless you are facing any issues.",
     )
     channel_min_members: int = Field(
-        type=int,
-        default=2,
+        2,
         description="Ingest channels with at least this many members.",
     )
     should_ingest_archived_channels: bool = Field(
-        type=bool,
-        default=False,
+        False,
         description="Whether to ingest archived channels.",
     )
 

datahub/ingestion/source/snowflake/snowflake_lineage_v2.py

@@ -72,7 +72,7 @@ class ColumnUpstreamJob(BaseModel):
 
 
 class ColumnUpstreamLineage(BaseModel):
-    column_name: Optional[str]
+    column_name: Optional[str] = None
     upstreams: List[ColumnUpstreamJob] = Field(default_factory=list)
 
 
@@ -91,9 +91,9 @@ class Query(BaseModel):
 class UpstreamLineageEdge(BaseModel):
     DOWNSTREAM_TABLE_NAME: str
     DOWNSTREAM_TABLE_DOMAIN: str
-    UPSTREAM_TABLES: Optional[List[UpstreamTableNode]]
-    UPSTREAM_COLUMNS: Optional[List[ColumnUpstreamLineage]]
-    QUERIES: Optional[List[Query]]
+    UPSTREAM_TABLES: Optional[List[UpstreamTableNode]] = None
+    UPSTREAM_COLUMNS: Optional[List[ColumnUpstreamLineage]] = None
+    QUERIES: Optional[List[Query]] = None
 
     _json_upstream_tables = pydantic_parse_json("UPSTREAM_TABLES")
     _json_upstream_columns = pydantic_parse_json("UPSTREAM_COLUMNS")

datahub/ingestion/source/sql/hive_metastore.py

@@ -27,6 +27,7 @@ from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import (
     DatasetContainerSubTypes,
     DatasetSubTypes,
+    SourceCapabilityModifier,
 )
 from datahub.ingestion.source.sql.sql_common import (
     SQLAlchemySource,
@@ -168,6 +169,13 @@ class HiveMetastore(BasicSQLAlchemyConfig):
 @capability(
     SourceCapability.LINEAGE_COARSE, "View lineage is not supported", supported=False
 )
+@capability(
+    SourceCapability.CONTAINERS,
+    "Enabled by default",
+    subtype_modifier=[
+        SourceCapabilityModifier.CATALOG,
+    ],
+)
 class HiveMetastoreSource(SQLAlchemySource):
     """
     This plugin extracts the following:

datahub/ingestion/source/sql/teradata.py

@@ -42,6 +42,7 @@ from datahub.ingestion.api.decorators import (
 )
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.graph.client import DataHubGraph
+from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
 from datahub.ingestion.source.sql.sql_common import register_custom_type
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
 from datahub.ingestion.source.sql.sql_report import SQLSourceReport
@@ -539,7 +540,13 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
 @config_class(TeradataConfig)
 @support_status(SupportStatus.TESTING)
 @capability(SourceCapability.DOMAINS, "Enabled by default")
-@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(
+    SourceCapability.CONTAINERS,
+    "Enabled by default",
+    subtype_modifier=[
+        SourceCapabilityModifier.DATABASE,
+    ],
+)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(
     SourceCapability.DELETION_DETECTION,

datahub/ingestion/source/sql/trino.py

@@ -36,6 +36,7 @@ from datahub.ingestion.api.decorators import (
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.extractor import schema_util
 from datahub.ingestion.source.common.data_reader import DataReader
+from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
 from datahub.ingestion.source.sql.sql_common import (
     SQLAlchemySource,
     SqlWorkUnit,
@@ -249,6 +250,14 @@ class TrinoConfig(BasicSQLAlchemyConfig):
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
+@capability(
+    SourceCapability.LINEAGE_COARSE,
+    "Extract table-level lineage",
+    subtype_modifier=[
+        SourceCapabilityModifier.TABLE,
+        SourceCapabilityModifier.VIEW,
+    ],
+)
 class TrinoSource(SQLAlchemySource):
     """
 

datahub/ingestion/source/unity/config.py

@@ -8,7 +8,7 @@ import pydantic
 from pydantic import Field
 from typing_extensions import Literal
 
-from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from datahub.configuration.common import AllowDenyPattern, ConfigEnum, ConfigModel
 from datahub.configuration.source_common import (
     DatasetSourceConfigMixin,
     LowerCaseDatasetUrnConfigMixin,
@@ -36,6 +36,12 @@ from datahub.utilities.global_warning_util import add_global_warning
 logger = logging.getLogger(__name__)
 
 
+class LineageDataSource(ConfigEnum):
+    AUTO = "AUTO"
+    SYSTEM_TABLES = "SYSTEM_TABLES"
+    API = "API"
+
+
 class UnityCatalogProfilerConfig(ConfigModel):
     method: str = Field(
         description=(
@@ -243,6 +249,21 @@ class UnityCatalogSourceConfig(
         description="Option to enable/disable lineage generation. Currently we have to call a rest call per column to get column level lineage due to the Databrick api which can slow down ingestion. ",
     )
 
+    lineage_data_source: LineageDataSource = pydantic.Field(
+        default=LineageDataSource.AUTO,
+        description=(
+            "Source for lineage data extraction. Options: "
+            f"'{LineageDataSource.AUTO.value}' - Use system tables when SQL warehouse is available, fallback to API; "
+            f"'{LineageDataSource.SYSTEM_TABLES.value}' - Force use of system.access.table_lineage and system.access.column_lineage tables (requires SQL warehouse); "
+            f"'{LineageDataSource.API.value}' - Force use of REST API endpoints for lineage data"
+        ),
+    )
+
+    ignore_start_time_lineage: bool = pydantic.Field(
+        default=False,
+        description="Option to ignore the start_time and retrieve all available lineage. When enabled, the start_time filter will be set to zero to extract all lineage events regardless of the configured time window.",
+    )
+
     column_lineage_column_limit: int = pydantic.Field(
         default=300,
         description="Limit the number of columns to get column level lineage. ",
@@ -362,6 +383,20 @@ class UnityCatalogSourceConfig(
 
         return values
 
+    @pydantic.root_validator(skip_on_failure=True)
+    def validate_lineage_data_source_with_warehouse(
+        cls, values: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        lineage_data_source = values.get("lineage_data_source", LineageDataSource.AUTO)
+        warehouse_id = values.get("warehouse_id")
+
+        if lineage_data_source == LineageDataSource.SYSTEM_TABLES and not warehouse_id:
+            raise ValueError(
+                f"lineage_data_source='{LineageDataSource.SYSTEM_TABLES.value}' requires warehouse_id to be set"
+            )
+
+        return values
+
     @pydantic.validator("schema_pattern", always=True)
     def schema_pattern_should__always_deny_information_schema(
         cls, v: AllowDenyPattern