acryl-datahub 1.1.1rc4__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl

datahub/ingestion/source/feast.py

@@ -1,5 +1,6 @@
+import pathlib
 from dataclasses import dataclass
-from typing import Dict, Iterable, List, Optional, Tuple, Union
+from typing import ClassVar, Dict, Iterable, List, Optional, Tuple, Union
 
 import feast.types
 from feast import (
@@ -97,7 +98,7 @@ class FeastRepositorySourceConfig(
     StatefulIngestionConfigBase,
 ):
     path: str = Field(description="Path to Feast repository")
-    fs_yaml_file: Optional[str] = Field(
+    fs_yaml_file: Optional[pathlib.Path] = Field(
         default=None,
         description="Path to the `feature_store.yaml` file used to configure the feature store",
     )
@@ -142,17 +143,14 @@ class FeastRepositorySource(StatefulIngestionSourceBase):
     - Column types associated with each entity and feature
     """
 
-    platform = "feast"
-    source_config: FeastRepositorySourceConfig
-    report: StaleEntityRemovalSourceReport
-    feature_store: FeatureStore
+    platform: ClassVar[str] = "feast"
 
     def __init__(self, config: FeastRepositorySourceConfig, ctx: PipelineContext):
         super().__init__(config, ctx)
-        self.source_config = config
-        self.ctx = ctx
-        self.report = StaleEntityRemovalSourceReport()
-        self.feature_store = FeatureStore(
+        self.source_config: FeastRepositorySourceConfig = config
+        self.ctx: PipelineContext = ctx
+        self.report: StaleEntityRemovalSourceReport = StaleEntityRemovalSourceReport()
+        self.feature_store: FeatureStore = FeatureStore(
             repo_path=self.source_config.path,
             fs_yaml_file=self.source_config.fs_yaml_file,
         )

datahub/ingestion/source/file.py

@@ -18,7 +18,9 @@ from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
+    SourceCapability,
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
@@ -187,6 +189,7 @@ class FileSourceReport(StaleEntityRemovalSourceReport):
 @platform_name("Metadata File")
 @config_class(FileSourceConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class GenericFileSource(StatefulIngestionSourceBase, TestableSource):
     """
     This plugin pulls metadata from a previously generated file.

datahub/ingestion/source/fivetran/config.py

@@ -29,6 +29,7 @@ from datahub.ingestion.source.state.stale_entity_removal_handler import (
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionConfigBase,
 )
+from datahub.ingestion.source.unity.connection import UnityCatalogConnectionConfig
 from datahub.utilities.lossy_collections import LossyList
 from datahub.utilities.perf_timer import PerfTimer
 
@@ -56,8 +57,8 @@ class Constant:
     STATUS = "status"
     USER_ID = "user_id"
     EMAIL = "email"
-    CONNECTOR_ID = "connector_id"
-    CONNECTOR_NAME = "connector_name"
+    CONNECTOR_ID = "connection_id"
+    CONNECTOR_NAME = "connection_name"
     CONNECTOR_TYPE_ID = "connector_type_id"
     PAUSED = "paused"
     SYNC_FREQUENCY = "sync_frequency"
@@ -67,13 +68,22 @@ class Constant:
     SUCCESSFUL = "SUCCESSFUL"
     FAILURE_WITH_TASK = "FAILURE_WITH_TASK"
     CANCELED = "CANCELED"
+    GOOGLE_SHEETS_CONNECTOR_TYPE = "google_sheets"
 
 
+# Key: Connector Type, Value: Platform ID/Name
 KNOWN_DATA_PLATFORM_MAPPING = {
+    "google_cloud_postgresql": "postgres",
     "postgres": "postgres",
     "snowflake": "snowflake",
+    Constant.GOOGLE_SHEETS_CONNECTOR_TYPE: Constant.GOOGLE_SHEETS_CONNECTOR_TYPE,
 }
 
+# Note: (As of Oct 2025) Fivetran Platform Connector has stale lineage metadata for Google Sheets column data (deleted/renamed).
+# Ref: https://fivetran.com/docs/connectors/files/google-sheets#deletingdata
+# TODO: Remove Google Sheets connector type from DISABLE_LINEAGE_FOR_CONNECTOR_TYPES
+DISABLE_COL_LINEAGE_FOR_CONNECTOR_TYPES = [Constant.GOOGLE_SHEETS_CONNECTOR_TYPE]
+
 
 class SnowflakeDestinationConfig(SnowflakeConnectionConfig):
     database: str = Field(description="The fivetran connector log database.")
@@ -84,10 +94,34 @@ class BigQueryDestinationConfig(BigQueryConnectionConfig):
     dataset: str = Field(description="The fivetran connector log dataset.")
 
 
+class DatabricksDestinationConfig(UnityCatalogConnectionConfig):
+    catalog: str = Field(description="The fivetran connector log catalog.")
+    log_schema: str = Field(description="The fivetran connector log schema.")
+
+    @pydantic.validator("warehouse_id")
+    def warehouse_id_should_not_be_empty(cls, warehouse_id: Optional[str]) -> str:
+        if warehouse_id is None or (warehouse_id and warehouse_id.strip() == ""):
+            raise ValueError("Fivetran requires warehouse_id to be set")
+        return warehouse_id
+
+
+class FivetranAPIConfig(ConfigModel):
+    api_key: str = Field(description="Fivetran API key")
+    api_secret: str = Field(description="Fivetran API secret")
+    base_url: str = Field(
+        default="https://api.fivetran.com", description="Fivetran API base URL"
+    )
+    request_timeout_sec: int = Field(
+        default=30, description="Request timeout in seconds"
+    )
+
+
 class FivetranLogConfig(ConfigModel):
-    destination_platform: Literal["snowflake", "bigquery"] = pydantic.Field(
-        default="snowflake",
-        description="The destination platform where fivetran connector log tables are dumped.",
+    destination_platform: Literal["snowflake", "bigquery", "databricks"] = (
+        pydantic.Field(
+            default="snowflake",
+            description="The destination platform where fivetran connector log tables are dumped.",
+        )
     )
     snowflake_destination_config: Optional[SnowflakeDestinationConfig] = pydantic.Field(
         default=None,
@@ -97,11 +131,17 @@ class FivetranLogConfig(ConfigModel):
         default=None,
         description="If destination platform is 'bigquery', provide bigquery configuration.",
     )
+    databricks_destination_config: Optional[DatabricksDestinationConfig] = (
+        pydantic.Field(
+            default=None,
+            description="If destination platform is 'databricks', provide databricks configuration.",
+        )
+    )
     _rename_destination_config = pydantic_renamed_field(
         "destination_config", "snowflake_destination_config"
     )
 
-    @root_validator(pre=True)
+    @root_validator(skip_on_failure=True)
     def validate_destination_platfrom_and_config(cls, values: Dict) -> Dict:
         destination_platform = values["destination_platform"]
         if destination_platform == "snowflake":
@@ -114,6 +154,11 @@ class FivetranLogConfig(ConfigModel):
                 raise ValueError(
                     "If destination platform is 'bigquery', user must provide bigquery destination configuration in the recipe."
                 )
+        elif destination_platform == "databricks":
+            if "databricks_destination_config" not in values:
+                raise ValueError(
+                    "If destination platform is 'databricks', user must provide databricks destination configuration in the recipe."
+                )
         else:
             raise ValueError(
                 f"Destination platform '{destination_platform}' is not yet supported."
@@ -137,6 +182,7 @@ class MetadataExtractionPerfReport(Report):
 @dataclasses.dataclass
 class FivetranSourceReport(StaleEntityRemovalSourceReport):
     connectors_scanned: int = 0
+    fivetran_rest_api_call_count: int = 0
     filtered_connectors: LossyList[str] = dataclasses.field(default_factory=LossyList)
     metadata_extraction_perf: MetadataExtractionPerfReport = dataclasses.field(
         default_factory=MetadataExtractionPerfReport
@@ -148,6 +194,9 @@ class FivetranSourceReport(StaleEntityRemovalSourceReport):
     def report_connectors_dropped(self, connector: str) -> None:
         self.filtered_connectors.append(connector)
 
+    def report_fivetran_rest_api_call_count(self) -> None:
+        self.fivetran_rest_api_call_count += 1
+
 
 class PlatformDetail(ConfigModel):
     platform: Optional[str] = pydantic.Field(
@@ -194,7 +243,7 @@ class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin
 
     # Configuration for stateful ingestion
     stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field(
-        default=None, description="Airbyte Stateful Ingestion Config."
+        default=None, description="Fivetran Stateful Ingestion Config."
     )
 
     # Fivetran connector all sources to platform instance mapping
@@ -208,6 +257,16 @@ class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin
         description="A mapping of destination id to its platform/instance/env details.",
     )
 
+    """
+    Use Fivetran REST API to get :
+    - Google Sheets Connector details and emit related entities
+    Fivetran Platform Connector syncs limited information about the Google Sheets Connector.
+    """
+    api_config: Optional[FivetranAPIConfig] = Field(
+        default=None,
+        description="Fivetran REST API configuration, used to provide wider support for connections.",
+    )
+
     @pydantic.root_validator(pre=True)
     def compat_sources_to_database(cls, values: Dict) -> Dict:
         if "sources_to_database" in values:

datahub/ingestion/source/fivetran/fivetran.py

@@ -1,8 +1,9 @@
 import logging
-from typing import Dict, Iterable, List, Optional
+from typing import Dict, Iterable, List, Optional, Union
+from urllib.parse import urlparse
 
 import datahub.emitter.mce_builder as builder
-from datahub.api.entities.datajob import DataFlow, DataJob
+from datahub.api.entities.datajob import DataJob as DataJobV1
 from datahub.api.entities.dataprocess.dataprocess_instance import (
     DataProcessInstance,
     InstanceRunResult,
@@ -16,8 +17,13 @@ from datahub.ingestion.api.decorators import (
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceReport
+from datahub.ingestion.api.source import (
+    MetadataWorkUnitProcessor,
+    SourceReport,
+    StructuredLogCategory,
+)
 from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.common.subtypes import DatasetSubTypes
 from datahub.ingestion.source.fivetran.config import (
     KNOWN_DATA_PLATFORM_MAPPING,
     Constant,
@@ -31,27 +37,39 @@ from datahub.ingestion.source.fivetran.fivetran_query import (
     MAX_JOBS_PER_CONNECTOR,
     MAX_TABLE_LINEAGE_PER_CONNECTOR,
 )
+from datahub.ingestion.source.fivetran.fivetran_rest_api import FivetranAPIClient
+from datahub.ingestion.source.fivetran.response_models import FivetranConnectionDetails
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalHandler,
 )
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionSourceBase,
 )
+from datahub.metadata.com.linkedin.pegasus2avro.common import AuditStamp
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     FineGrainedLineage,
     FineGrainedLineageDownstreamType,
     FineGrainedLineageUpstreamType,
+    UpstreamLineage,
+)
+from datahub.metadata.schema_classes import (
+    DatasetLineageTypeClass,
+    UpstreamClass,
 )
-from datahub.utilities.urns.data_flow_urn import DataFlowUrn
-from datahub.utilities.urns.dataset_urn import DatasetUrn
+from datahub.metadata.urns import CorpUserUrn, DataFlowUrn, DatasetUrn
+from datahub.sdk.dataflow import DataFlow
+from datahub.sdk.datajob import DataJob
+from datahub.sdk.dataset import Dataset
+from datahub.sdk.entity import Entity
 
 # Logger instance
 logger = logging.getLogger(__name__)
+CORPUSER_DATAHUB = "urn:li:corpuser:datahub"
 
 
 @platform_name("Fivetran")
 @config_class(FivetranSourceConfig)
-@support_status(SupportStatus.INCUBATING)
+@support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(
     SourceCapability.LINEAGE_FINE,
@@ -60,7 +78,6 @@ logger = logging.getLogger(__name__)
 class FivetranSource(StatefulIngestionSourceBase):
     """
     This plugin extracts fivetran users, connectors, destinations and sync history.
-    This plugin is in beta and has only been tested on Snowflake connector.
     """
 
     config: FivetranSourceConfig
@@ -71,12 +88,16 @@ class FivetranSource(StatefulIngestionSourceBase):
         super().__init__(config, ctx)
         self.config = config
         self.report = FivetranSourceReport()
-
         self.audit_log = FivetranLogAPI(self.config.fivetran_log_config)
+        self.api_client: Optional[FivetranAPIClient] = None
+        self._connection_details_cache: Dict[str, FivetranConnectionDetails] = {}
+
+        if self.config.api_config:
+            self.api_client = FivetranAPIClient(self.config.api_config)
 
     def _extend_lineage(self, connector: Connector, datajob: DataJob) -> Dict[str, str]:
-        input_dataset_urn_list: List[DatasetUrn] = []
-        output_dataset_urn_list: List[DatasetUrn] = []
+        input_dataset_urn_list: List[Union[str, DatasetUrn]] = []
+        output_dataset_urn_list: List[Union[str, DatasetUrn]] = []
         fine_grained_lineage: List[FineGrainedLineage] = []
 
         # TODO: Once Fivetran exposes the database via the API, we shouldn't ask for it via config.
@@ -94,8 +115,10 @@ class FivetranSource(StatefulIngestionSourceBase):
                 self.report.info(
                     title="Guessing source platform for lineage",
                     message="We encountered a connector type that we don't fully support yet. "
-                    "We will attempt to guess the platform based on the connector type.",
-                    context=f"{connector.connector_name} (connector_id: {connector.connector_id}, connector_type: {connector.connector_type})",
+                    "We will attempt to guess the platform based on the connector type. "
+                    "Note that we use connector_id as the key not connector_name which you may see in the UI of Fivetran. ",
+                    context=f"connector_name: {connector.connector_name} (connector_id: {connector.connector_id}, connector_type: {connector.connector_type})",
+                    log_category=StructuredLogCategory.LINEAGE,
                 )
                 source_details.platform = connector.connector_type
 
@@ -124,17 +147,43 @@ class FivetranSource(StatefulIngestionSourceBase):
                 if source_details.include_schema_in_urn
                 else lineage.source_table.split(".", 1)[1]
             )
-            input_dataset_urn = DatasetUrn.create_from_ids(
-                platform_id=source_details.platform,
-                table_name=(
-                    f"{source_details.database.lower()}.{source_table}"
-                    if source_details.database
-                    else source_table
-                ),
-                env=source_details.env,
-                platform_instance=source_details.platform_instance,
-            )
-            input_dataset_urn_list.append(input_dataset_urn)
+            input_dataset_urn: Optional[DatasetUrn] = None
+            # Special Handling for Google Sheets Connectors
+            if connector.connector_type == Constant.GOOGLE_SHEETS_CONNECTOR_TYPE:
+                # Get Google Sheet dataset details from Fivetran API
+                # This is cached in the api_client
+                gsheets_conn_details: Optional[FivetranConnectionDetails] = (
+                    self._get_connection_details_by_id(connector.connector_id)
+                )
+
+                if gsheets_conn_details:
+                    input_dataset_urn = DatasetUrn.create_from_ids(
+                        platform_id=Constant.GOOGLE_SHEETS_CONNECTOR_TYPE,
+                        table_name=self._get_gsheet_named_range_dataset_id(
+                            gsheets_conn_details
+                        ),
+                        env=source_details.env,
+                    )
+                else:
+                    self.report.warning(
+                        title="Failed to extract lineage for Google Sheets Connector",
+                        message="Unable to extract lineage for Google Sheets Connector, as the connector details are not available from Fivetran API.",
+                        context=f"{connector.connector_name} (connector_id: {connector.connector_id})",
+                    )
+            else:
+                input_dataset_urn = DatasetUrn.create_from_ids(
+                    platform_id=source_details.platform,
+                    table_name=(
+                        f"{source_details.database.lower()}.{source_table}"
+                        if source_details.database
+                        else source_table
+                    ),
+                    env=source_details.env,
+                    platform_instance=source_details.platform_instance,
+                )
+
+            if input_dataset_urn:
+                input_dataset_urn_list.append(input_dataset_urn)
 
             destination_table = (
                 lineage.destination_table
@@ -178,9 +227,9 @@ class FivetranSource(StatefulIngestionSourceBase):
                         )
                     )
 
-        datajob.inlets.extend(input_dataset_urn_list)
-        datajob.outlets.extend(output_dataset_urn_list)
-        datajob.fine_grained_lineages.extend(fine_grained_lineage)
+        datajob.set_inlets(input_dataset_urn_list)
+        datajob.set_outlets(output_dataset_urn_list)
+        datajob.set_fine_grained_lineages(fine_grained_lineage)
 
         return dict(
             **{
@@ -197,10 +246,10 @@ class FivetranSource(StatefulIngestionSourceBase):
 
     def _generate_dataflow_from_connector(self, connector: Connector) -> DataFlow:
         return DataFlow(
-            orchestrator=Constant.ORCHESTRATOR,
-            id=connector.connector_id,
+            platform=Constant.ORCHESTRATOR,
+            name=connector.connector_id,
             env=self.config.env,
-            name=connector.connector_name,
+            display_name=connector.connector_name,
             platform_instance=self.config.platform_instance,
         )
 
@@ -213,11 +262,11 @@ class FivetranSource(StatefulIngestionSourceBase):
         )
         owner_email = self.audit_log.get_user_email(connector.user_id)
         datajob = DataJob(
-            id=connector.connector_id,
+            name=connector.connector_id,
             flow_urn=dataflow_urn,
             platform_instance=self.config.platform_instance,
-            name=connector.connector_name,
-            owners={owner_email} if owner_email else set(),
+            display_name=connector.connector_name,
+            owners=[CorpUserUrn(owner_email)] if owner_email else None,
         )
 
         # Map connector source and destination table with dataset entity
@@ -232,21 +281,90 @@ class FivetranSource(StatefulIngestionSourceBase):
             "sync_frequency": str(connector.sync_frequency),
             "destination_id": connector.destination_id,
         }
-        datajob.properties = {
-            **connector_properties,
-            **lineage_properties,
-        }
+
+        datajob.set_custom_properties({**connector_properties, **lineage_properties})
 
         return datajob
 
     def _generate_dpi_from_job(self, job: Job, datajob: DataJob) -> DataProcessInstance:
+        # hack: convert to old instance for DataProcessInstance.from_datajob compatibility
+        datajob_v1 = DataJobV1(
+            id=datajob.name,
+            flow_urn=datajob.flow_urn,
+            platform_instance=self.config.platform_instance,
+            name=datajob.name,
+            inlets=datajob.inlets,
+            outlets=datajob.outlets,
+            fine_grained_lineages=datajob.fine_grained_lineages,
+        )
         return DataProcessInstance.from_datajob(
-            datajob=datajob,
+            datajob=datajob_v1,
             id=job.job_id,
             clone_inlets=True,
             clone_outlets=True,
         )
 
+    def _get_connection_details_by_id(
+        self, connection_id: str
+    ) -> Optional[FivetranConnectionDetails]:
+        if self.api_client is None:
+            self.report.warning(
+                title="Fivetran API client is not initialized",
+                message="Google Sheets Connector details cannot be extracted, as Fivetran API client is not initialized.",
+                context=f"connector_id: {connection_id}",
+            )
+            return None
+
+        if connection_id in self._connection_details_cache:
+            return self._connection_details_cache[connection_id]
+
+        try:
+            self.report.report_fivetran_rest_api_call_count()
+            conn_details = self.api_client.get_connection_details_by_id(connection_id)
+            # Update Cache
+            if conn_details:
+                self._connection_details_cache[connection_id] = conn_details
+
+            return conn_details
+        except Exception as e:
+            self.report.warning(
+                title="Failed to get connection details for Google Sheets Connector",
+                message=f"Exception occurred while getting connection details from Fivetran API. {e}",
+                context=f"connector_id: {connection_id}",
+            )
+            return None
+
+    def _get_gsheet_sheet_id_from_url(
+        self, gsheets_conn_details: FivetranConnectionDetails
+    ) -> str:
+        # Extracting the sheet_id (1A82PdLAE7NXLLb5JcLPKeIpKUMytXQba5Z-Ei-mbXLo) from the sheet_id url
+        # "https://docs.google.com/spreadsheets/d/1A82PdLAE7NXLLb5JcLPKeIpKUMytXQba5Z-Ei-mbXLo/edit?gid=0#gid=0",
+        try:
+            parsed = urlparse(gsheets_conn_details.config.sheet_id)
+            # Example: https://docs.google.com/spreadsheets/d/<spreadsheetId>/edit
+            parts = parsed.path.split("/")
+            return parts[3] if len(parts) > 2 else ""
+        except Exception as e:
+            logger.warning(
+                f"Failed to extract sheet_id from the sheet_id url: {gsheets_conn_details.config.sheet_id}, {e}"
+            )
+
+        return ""
+
+    def _get_gsheet_named_range_dataset_id(
+        self, gsheets_conn_details: FivetranConnectionDetails
+    ) -> str:
+        sheet_id = self._get_gsheet_sheet_id_from_url(gsheets_conn_details)
+        named_range_id = (
+            f"{sheet_id}.{gsheets_conn_details.config.named_range}"
+            if sheet_id
+            else gsheets_conn_details.config.named_range
+        )
+        logger.debug(
+            f"Using gsheet_named_range_dataset_id: {named_range_id} for connector: {gsheets_conn_details.id}"
+        )
+        return named_range_id
+
     def _get_dpi_workunits(
         self, job: Job, dpi: DataProcessInstance
     ) -> Iterable[MetadataWorkUnit]:
@@ -278,17 +396,83 @@ class FivetranSource(StatefulIngestionSourceBase):
 
     def _get_connector_workunits(
         self, connector: Connector
-    ) -> Iterable[MetadataWorkUnit]:
+    ) -> Iterable[Union[MetadataWorkUnit, Entity]]:
         self.report.report_connectors_scanned()
+
+        """
+        -------------------------------------------------------
+        Special Handling for Google Sheets Connectors
+        -------------------------------------------------------
+        Google Sheets source is not supported by Datahub yet.   
+        As a workaround, we are emitting a dataset entity for the Google Sheet
+        and adding it to the lineage. This workaround needs to be removed once 
+        Datahub supports Google Sheets source natively.
+        -------------------------------------------------------
+        """
+        if connector.connector_type == Constant.GOOGLE_SHEETS_CONNECTOR_TYPE:
+            # Get Google Sheet dataset details from Fivetran API
+            gsheets_conn_details: Optional[FivetranConnectionDetails] = (
+                self._get_connection_details_by_id(connector.connector_id)
+            )
+
+            if gsheets_conn_details:
+                gsheets_dataset = Dataset(
+                    name=self._get_gsheet_sheet_id_from_url(gsheets_conn_details),
+                    platform=Constant.GOOGLE_SHEETS_CONNECTOR_TYPE,
+                    env=self.config.env,
+                    display_name=self._get_gsheet_sheet_id_from_url(
+                        gsheets_conn_details
+                    ),
+                    external_url=gsheets_conn_details.config.sheet_id,
+                    created=gsheets_conn_details.created_at,
+                    last_modified=gsheets_conn_details.source_sync_details.last_synced,
+                    subtype=DatasetSubTypes.GOOGLE_SHEETS,
+                    custom_properties={
+                        "ingested_by": "fivetran source",
+                        "connector_id": gsheets_conn_details.id,
+                    },
+                )
+                gsheets_named_range_dataset = Dataset(
+                    name=self._get_gsheet_named_range_dataset_id(gsheets_conn_details),
+                    platform=Constant.GOOGLE_SHEETS_CONNECTOR_TYPE,
+                    env=self.config.env,
+                    display_name=gsheets_conn_details.config.named_range,
+                    external_url=gsheets_conn_details.config.sheet_id,
+                    created=gsheets_conn_details.created_at,
+                    last_modified=gsheets_conn_details.source_sync_details.last_synced,
+                    subtype=DatasetSubTypes.GOOGLE_SHEETS_NAMED_RANGE,
+                    custom_properties={
+                        "ingested_by": "fivetran source",
+                        "connector_id": gsheets_conn_details.id,
+                    },
+                    upstreams=UpstreamLineage(
+                        upstreams=[
+                            UpstreamClass(
+                                dataset=str(gsheets_dataset.urn),
+                                type=DatasetLineageTypeClass.VIEW,
+                                auditStamp=AuditStamp(
+                                    time=int(
+                                        gsheets_conn_details.created_at.timestamp()
+                                        * 1000
+                                    ),
+                                    actor=CORPUSER_DATAHUB,
+                                ),
+                            )
+                        ],
+                        fineGrainedLineages=None,
+                    ),
+                )
+
+                yield gsheets_dataset
+                yield gsheets_named_range_dataset
+
         # Create dataflow entity with same name as connector name
         dataflow = self._generate_dataflow_from_connector(connector)
-        for mcp in dataflow.generate_mcp():
-            yield mcp.as_workunit()
+        yield dataflow
 
         # Map Fivetran's connector entity with Datahub's datajob entity
         datajob = self._generate_datajob_from_connector(connector)
-        for mcp in datajob.generate_mcp(materialize_iolets=False):
-            yield mcp.as_workunit()
+        yield datajob
 
         # Map Fivetran's job/sync history entity with Datahub's data process entity
         if len(connector.jobs) >= MAX_JOBS_PER_CONNECTOR:
@@ -310,7 +494,7 @@ class FivetranSource(StatefulIngestionSourceBase):
             ).workunit_processor,
         ]
 
-    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+    def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:
         """
         Datahub Ingestion framework invoke this method
         """