acryl-datahub 1.1.1rc4__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl

datahub/ingestion/source/fivetran/fivetran_log_api.py

@@ -9,6 +9,7 @@ from sqlalchemy import create_engine
 
 from datahub.configuration.common import AllowDenyPattern, ConfigurationError
 from datahub.ingestion.source.fivetran.config import (
+    DISABLE_COL_LINEAGE_FOR_CONNECTOR_TYPES,
     Constant,
     FivetranLogConfig,
     FivetranSourceReport,
@@ -69,9 +70,23 @@ class FivetranLogAPI:
                 fivetran_log_query.set_schema(bigquery_destination_config.dataset)
 
                 # The "database" should be the BigQuery project name.
-                fivetran_log_database = engine.execute(
-                    "SELECT @@project_id"
-                ).fetchone()[0]
+                result = engine.execute("SELECT @@project_id").fetchone()
+                if result is None:
+                    raise ValueError("Failed to retrieve BigQuery project ID")
+                fivetran_log_database = result[0]
+        elif destination_platform == "databricks":
+            databricks_destination_config = (
+                self.fivetran_log_config.databricks_destination_config
+            )
+            if databricks_destination_config is not None:
+                engine = create_engine(
+                    databricks_destination_config.get_sql_alchemy_url(
+                        databricks_destination_config.catalog
+                    ),
+                    **databricks_destination_config.get_options(),
+                )
+                fivetran_log_query.set_schema(databricks_destination_config.log_schema)
+                fivetran_log_database = databricks_destination_config.catalog
         else:
             raise ConfigurationError(
                 f"Destination platform '{destination_platform}' is not yet supported."
@@ -98,7 +113,11 @@ class FivetranLogAPI:
         """
         Returns dict of column lineage metadata with key as (<SOURCE_TABLE_ID>, <DESTINATION_TABLE_ID>)
         """
-        all_column_lineage = defaultdict(list)
+        all_column_lineage: Dict[Tuple[str, str], List] = defaultdict(list)
+
+        if not connector_ids:
+            return dict(all_column_lineage)
+
         column_lineage_result = self._query(
             self.fivetran_log_query.get_column_lineage_query(
                 connector_ids=connector_ids
@@ -116,7 +135,11 @@ class FivetranLogAPI:
         """
         Returns dict of table lineage metadata with key as 'CONNECTOR_ID'
         """
-        connectors_table_lineage_metadata = defaultdict(list)
+        connectors_table_lineage_metadata: Dict[str, List] = defaultdict(list)
+
+        if not connector_ids:
+            return dict(connectors_table_lineage_metadata)
+
         table_lineage_result = self._query(
             self.fivetran_log_query.get_table_lineage_query(connector_ids=connector_ids)
         )
@@ -232,9 +255,15 @@ class FivetranLogAPI:
         return self._get_users().get(user_id)
 
     def _fill_connectors_lineage(self, connectors: List[Connector]) -> None:
-        connector_ids = [connector.connector_id for connector in connectors]
-        table_lineage_metadata = self._get_table_lineage_metadata(connector_ids)
-        column_lineage_metadata = self._get_column_lineage_metadata(connector_ids)
+        # Create 2 filtered connector_ids lists - one for table lineage and one for column lineage
+        tll_connector_ids: List[str] = []
+        cll_connector_ids: List[str] = []
+        for connector in connectors:
+            tll_connector_ids.append(connector.connector_id)
+            if connector.connector_type not in DISABLE_COL_LINEAGE_FOR_CONNECTOR_TYPES:
+                cll_connector_ids.append(connector.connector_id)
+        table_lineage_metadata = self._get_table_lineage_metadata(tll_connector_ids)
+        column_lineage_metadata = self._get_column_lineage_metadata(cll_connector_ids)
         for connector in connectors:
             connector.lineage = self._extract_connector_lineage(
                 table_lineage_result=table_lineage_metadata.get(connector.connector_id),

datahub/ingestion/source/fivetran/fivetran_query.py

@@ -6,6 +6,21 @@ MAX_COLUMN_LINEAGE_PER_CONNECTOR = 1000
 MAX_JOBS_PER_CONNECTOR = 500
 
 
+"""
+------------------------------------------------------------------------------------------------------------
+Fivetran Platform Connector Handling
+------------------------------------------------------------------------------------------------------------
+Current Query Change Log: August 2025 (See: https://fivetran.com/docs/changelog/2025/august-2025)
+
+All queries have to be updated as per Fivetran Platform Connector release if any. We expect customers
+and fivetran to keep platform connector configured for DataHub with auto sync enabled to get latest changes.
+
+References:
+- Fivetran Release Notes: https://fivetran.com/docs/changelog (Look for "Fivetran Platform Connector")
+- Latest Platform Connector Schema: https://fivetran.com/docs/logs/fivetran-platform?erdModal=open
+"""
+
+
 class FivetranLogQuery:
     # Note: All queries are written in Snowflake SQL.
     # They will be transpiled to the target database's SQL dialect at runtime.
@@ -18,22 +33,29 @@ class FivetranLogQuery:
         return f"use database {db_name}"
 
     def set_schema(self, schema_name: str) -> None:
-        self.schema_clause = f"{schema_name}."
+        """
+        Using Snowflake quoted identifiers convention
+
+        Add double quotes around an identifier
+        Use two quotes to use the double quote character inside a quoted identifier
+        """
+        schema_name = schema_name.replace('"', '""')
+        self.schema_clause = f'"{schema_name}".'
 
     def get_connectors_query(self) -> str:
         return f"""\
 SELECT
-  connector_id,
+  connection_id,
   connecting_user_id,
   connector_type_id,
-  connector_name,
+  connection_name,
   paused,
   sync_frequency,
   destination_id
-FROM {self.schema_clause}connector
+FROM {self.schema_clause}connection
 WHERE
   _fivetran_deleted = FALSE
-QUALIFY ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY _fivetran_synced DESC) = 1
+QUALIFY ROW_NUMBER() OVER (PARTITION BY connection_id ORDER BY _fivetran_synced DESC) = 1
 """
 
     def get_users_query(self) -> str:
@@ -56,20 +78,20 @@ FROM {self.schema_clause}user
         return f"""\
 WITH ranked_syncs AS (
     SELECT
-        connector_id,
+        connection_id,
         sync_id,
         MAX(CASE WHEN message_event = 'sync_start' THEN time_stamp END) as start_time,
         MAX(CASE WHEN message_event = 'sync_end' THEN time_stamp END) as end_time,
         MAX(CASE WHEN message_event = 'sync_end' THEN message_data END) as end_message_data,
-        ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY MAX(time_stamp) DESC) as rn
+        ROW_NUMBER() OVER (PARTITION BY connection_id ORDER BY MAX(time_stamp) DESC) as rn
     FROM {self.schema_clause}log
     WHERE message_event in ('sync_start', 'sync_end')
     AND time_stamp > CURRENT_TIMESTAMP - INTERVAL '{syncs_interval} days'
-    AND connector_id IN ({formatted_connector_ids})
-    GROUP BY connector_id, sync_id
+    AND connection_id IN ({formatted_connector_ids})
+    GROUP BY connection_id, sync_id
 )
 SELECT
-    connector_id,
+    connection_id,
     sync_id,
     start_time,
     end_time,
@@ -78,7 +100,7 @@ FROM ranked_syncs
 WHERE rn <= {MAX_JOBS_PER_CONNECTOR}
     AND start_time IS NOT NULL
     AND end_time IS NOT NULL
-ORDER BY connector_id, end_time DESC
+ORDER BY connection_id, end_time DESC
 """
 
     def get_table_lineage_query(self, connector_ids: List[str]) -> str:
@@ -90,7 +112,7 @@ SELECT
     *
 FROM (
     SELECT
-        stm.connector_id as connector_id,
+        stm.connection_id as connection_id,
         stm.id as source_table_id,
         stm.name as source_table_name,
         ssm.name as source_schema_name,
@@ -98,18 +120,18 @@ FROM (
         dtm.name as destination_table_name,
         dsm.name as destination_schema_name,
         tl.created_at as created_at,
-        ROW_NUMBER() OVER (PARTITION BY stm.connector_id, stm.id, dtm.id ORDER BY tl.created_at DESC) as table_combo_rn
+        ROW_NUMBER() OVER (PARTITION BY stm.connection_id, stm.id, dtm.id ORDER BY tl.created_at DESC) as table_combo_rn
     FROM {self.schema_clause}table_lineage as tl
-    JOIN {self.schema_clause}source_table_metadata as stm on tl.source_table_id = stm.id
-    JOIN {self.schema_clause}destination_table_metadata as dtm on tl.destination_table_id = dtm.id
-    JOIN {self.schema_clause}source_schema_metadata as ssm on stm.schema_id = ssm.id
-    JOIN {self.schema_clause}destination_schema_metadata as dsm on dtm.schema_id = dsm.id
-    WHERE stm.connector_id IN ({formatted_connector_ids})
+    JOIN {self.schema_clause}source_table as stm on tl.source_table_id = stm.id -- stm: source_table_metadata
+    JOIN {self.schema_clause}destination_table as dtm on tl.destination_table_id = dtm.id -- dtm: destination_table_metadata
+    JOIN {self.schema_clause}source_schema as ssm on stm.schema_id = ssm.id -- ssm: source_schema_metadata
+    JOIN {self.schema_clause}destination_schema as dsm on dtm.schema_id = dsm.id -- dsm: destination_schema_metadata
+    WHERE stm.connection_id IN ({formatted_connector_ids})
 )
 -- Ensure that we only get back one entry per source and destination pair.
 WHERE table_combo_rn = 1
-QUALIFY ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY created_at DESC) <= {MAX_TABLE_LINEAGE_PER_CONNECTOR}
-ORDER BY connector_id, created_at DESC
+QUALIFY ROW_NUMBER() OVER (PARTITION BY connection_id ORDER BY created_at DESC) <= {MAX_TABLE_LINEAGE_PER_CONNECTOR}
+ORDER BY connection_id, created_at DESC
 """
 
     def get_column_lineage_query(self, connector_ids: List[str]) -> str:
@@ -124,25 +146,25 @@ SELECT
     destination_column_name
 FROM (
     SELECT
-        stm.connector_id as connector_id,
+        stm.connection_id as connection_id,
         scm.table_id as source_table_id,
         dcm.table_id as destination_table_id,
         scm.name as source_column_name,
         dcm.name as destination_column_name,
         cl.created_at as created_at,
-        ROW_NUMBER() OVER (PARTITION BY stm.connector_id, cl.source_column_id, cl.destination_column_id ORDER BY cl.created_at DESC) as column_combo_rn
+        ROW_NUMBER() OVER (PARTITION BY stm.connection_id, cl.source_column_id, cl.destination_column_id ORDER BY cl.created_at DESC) as column_combo_rn
     FROM {self.schema_clause}column_lineage as cl
-    JOIN {self.schema_clause}source_column_metadata as scm
+    JOIN {self.schema_clause}source_column as scm -- scm: source_column_metadata
       ON cl.source_column_id = scm.id
-    JOIN {self.schema_clause}destination_column_metadata as dcm
+    JOIN {self.schema_clause}destination_column as dcm -- dcm: destination_column_metadata
       ON cl.destination_column_id = dcm.id
-    -- Only joining source_table_metadata to get the connector_id.
-    JOIN {self.schema_clause}source_table_metadata as stm
+    -- Only joining source_table to get the connection_id.
+    JOIN {self.schema_clause}source_table as stm -- stm: source_table_metadata
       ON scm.table_id = stm.id
-    WHERE stm.connector_id IN ({formatted_connector_ids})
+    WHERE stm.connection_id IN ({formatted_connector_ids})
 )
 -- Ensure that we only get back one entry per (connector, source column, destination column) pair.
 WHERE column_combo_rn = 1
-QUALIFY ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY created_at DESC) <= {MAX_COLUMN_LINEAGE_PER_CONNECTOR}
-ORDER BY connector_id, created_at DESC
+QUALIFY ROW_NUMBER() OVER (PARTITION BY connection_id ORDER BY created_at DESC) <= {MAX_COLUMN_LINEAGE_PER_CONNECTOR}
+ORDER BY connection_id, created_at DESC
 """

datahub/ingestion/source/fivetran/fivetran_rest_api.py

@@ -0,0 +1,65 @@
+import logging
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+
+from datahub.ingestion.source.fivetran.config import (
+    FivetranAPIConfig,
+)
+from datahub.ingestion.source.fivetran.response_models import FivetranConnectionDetails
+
+logger = logging.getLogger(__name__)
+
+# Retry configuration constants
+RETRY_MAX_TIMES = 3
+RETRY_STATUS_CODES = [429, 500, 502, 503, 504]
+RETRY_BACKOFF_FACTOR = 1
+RETRY_ALLOWED_METHODS = ["GET"]
+
+
+class FivetranAPIClient:
+    """Client for interacting with the Fivetran REST API."""
+
+    def __init__(self, config: FivetranAPIConfig) -> None:
+        self.config = config
+        self._session = self._create_session()
+
+    def _create_session(self) -> requests.Session:
+        """
+        Create a session with retry logic and basic authentication
+        """
+        requests_session = requests.Session()
+
+        # Configure retry strategy for transient failures
+        retry_strategy = Retry(
+            total=RETRY_MAX_TIMES,
+            backoff_factor=RETRY_BACKOFF_FACTOR,
+            status_forcelist=RETRY_STATUS_CODES,
+            allowed_methods=RETRY_ALLOWED_METHODS,
+            raise_on_status=True,
+        )
+
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        requests_session.mount("http://", adapter)
+        requests_session.mount("https://", adapter)
+
+        # Set up basic authentication
+        requests_session.auth = (self.config.api_key, self.config.api_secret)
+        requests_session.headers.update(
+            {
+                "Content-Type": "application/json",
+                "Accept": "application/json",
+            }
+        )
+        return requests_session
+
+    def get_connection_details_by_id(
+        self, connection_id: str
+    ) -> FivetranConnectionDetails:
+        """Get details for a specific connection."""
+        connection_details = self._session.get(
+            f"{self.config.base_url}/v1/connections/{connection_id}",
+            timeout=self.config.request_timeout_sec,
+        )
+        return FivetranConnectionDetails(**connection_details.json().get("data", {}))

datahub/ingestion/source/fivetran/response_models.py

@@ -0,0 +1,97 @@
+import datetime
+from typing import Dict, List
+
+from pydantic import BaseModel
+
+
+class FivetranConnectionWarnings(BaseModel):
+    code: str  # Warning Code
+    message: str  # Warning Message
+    details: Dict  # Warning Details
+
+
+class FivetranConnectionStatus(BaseModel):
+    setup_state: str  # Setup State
+    schema_status: str  # Schema Status
+    sync_state: str  # Sync State
+    update_state: str  # Update State
+    is_historical_sync: bool  # Is Historical Sync
+    warnings: List[FivetranConnectionWarnings]  # Warnings
+
+
+class FivetranConnectionConfig(BaseModel):
+    # Note: Connection Config is different for different connectors
+    auth_type: str  # Auth Type
+    sheet_id: str  # Sheet ID - URL to the Google Sheet
+    named_range: str  # Named Range
+
+
+class FivetranConnectionSourceSyncDetails(BaseModel):
+    last_synced: datetime.datetime  # Last Synced
+
+
+class FivetranConnectionDetails(BaseModel):
+    """
+    Note: This reponse class only captures fields that are relevant to the Google Sheets Connector
+    """
+
+    id: str  # Source ID
+    group_id: str  # Destination ID
+    service: str  # Connector Type
+    created_at: datetime.datetime
+    succeeded_at: datetime.datetime
+    paused: bool  # Paused Status
+    sync_frequency: int  # Sync Frequency (minutes)
+    status: FivetranConnectionStatus  # Status
+    config: FivetranConnectionConfig  # Connection Config
+    source_sync_details: FivetranConnectionSourceSyncDetails  # Source Sync Details
+
+    """
+    # Sample Response for Google Sheets Connector
+    {
+        "code": "Success",
+        "data": {
+            "id": "dialectical_remindful",
+            "group_id": "empties_classification",
+            "service": "google_sheets",
+            "service_version": 1,
+            "schema": "fivetran_google_sheets.fivetran_google_sheets",
+            "connected_by": "sewn_restrained",
+            "created_at": "2025-10-06T17:53:01.554289Z",
+            "succeeded_at": "2025-10-06T22:55:45.275000Z",
+            "failed_at": null,
+            "paused": true,
+            "pause_after_trial": false,
+            "sync_frequency": 360,
+            "data_delay_threshold": 0,
+            "data_delay_sensitivity": "NORMAL",
+            "private_link_id": null,
+            "networking_method": "Directly",
+            "proxy_agent_id": null,
+            "schedule_type": "auto",
+            "status": {
+                "setup_state": "connected",
+                "schema_status": "ready",
+                "sync_state": "paused",
+                "update_state": "on_schedule",
+                "is_historical_sync": false,
+                "tasks": [],
+                "warnings": [
+                    {
+                        "code": "snowflake_discontinuing_password_auth",
+                        "message": "Snowflake is discontinuing username/password authentication",
+                        "details": {}
+                    }
+                ]
+            },
+            "config": {
+                "auth_type": "ServiceAccount",
+                "sheet_id": "https://docs.google.com/spreadsheets/d/1A82PdLAE7NXLLb5JcLPKeIpKUMytXQba5Z-Ei-mbXLo/edit?gid=0#gid=0",
+                "named_range": "Fivetran_Test_Range"
+            },
+            "source_sync_details": {
+                "last_synced": "2025-10-06T22:55:27.371Z"
+            }
+        }
+    }
+    """

datahub/ingestion/source/gc/datahub_gc.py

@@ -34,7 +34,6 @@ from datahub.ingestion.source.gc.soft_deleted_entity_cleanup import (
     SoftDeletedEntitiesCleanupConfig,
     SoftDeletedEntitiesReport,
 )
-from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 
 logger = logging.getLogger(__name__)
 
@@ -87,7 +86,6 @@ class DataHubGcSourceReport(
     DataProcessCleanupReport,
     SoftDeletedEntitiesReport,
     DatahubExecutionRequestCleanupReport,
-    IngestionStageReport,
 ):
     expired_tokens_revoked: int = 0
 

datahub/ingestion/source/gcs/gcs_source.py

@@ -16,6 +16,7 @@ from datahub.ingestion.api.decorators import (
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceCapability
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.aws.aws_common import AwsConnectionConfig
+from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
 from datahub.ingestion.source.data_lake_common.config import PathSpecsConfigMixin
 from datahub.ingestion.source.data_lake_common.data_lake_utils import PLATFORM_GCS
 from datahub.ingestion.source.data_lake_common.object_store import (
@@ -36,6 +37,8 @@ from datahub.ingestion.source.state.stateful_ingestion_base import (
 
 logger: logging.Logger = logging.getLogger(__name__)
 
+GCS_ENDPOINT_URL = "https://storage.googleapis.com"
+
 
 class HMACKey(ConfigModel):
     hmac_access_id: str = Field(description="Access ID")
@@ -82,7 +85,14 @@ class GCSSourceReport(DataLakeSourceReport):
 @platform_name("Google Cloud Storage", id=PLATFORM_GCS)
 @config_class(GCSSourceConfig)
 @support_status(SupportStatus.INCUBATING)
-@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(
+    SourceCapability.CONTAINERS,
+    "Enabled by default",
+    subtype_modifier=[
+        SourceCapabilityModifier.GCS_BUCKET,
+        SourceCapabilityModifier.FOLDER,
+    ],
+)
 @capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
 @capability(SourceCapability.DATA_PROFILING, "Not supported", supported=False)
 class GCSSource(StatefulIngestionSourceBase):
@@ -104,7 +114,7 @@ class GCSSource(StatefulIngestionSourceBase):
         s3_config = DataLakeSourceConfig(
             path_specs=s3_path_specs,
             aws_config=AwsConnectionConfig(
-                aws_endpoint_url="https://storage.googleapis.com",
+                aws_endpoint_url=GCS_ENDPOINT_URL,
                 aws_access_key_id=self.config.credential.hmac_access_id,
                 aws_secret_access_key=self.config.credential.hmac_access_secret.get_secret_value(),
                 aws_region="auto",
@@ -112,15 +122,26 @@ class GCSSource(StatefulIngestionSourceBase):
             env=self.config.env,
             max_rows=self.config.max_rows,
             number_of_files_to_sample=self.config.number_of_files_to_sample,
+            platform=PLATFORM_GCS,  # Ensure GCS platform is used for correct container subtypes
+            platform_instance=self.config.platform_instance,
         )
         return s3_config
 
     def create_equivalent_s3_path_specs(self):
         s3_path_specs = []
         for path_spec in self.config.path_specs:
+            # PathSpec modifies the passed-in include to add /** to the end if
+            # autodetecting partitions. Remove that, otherwise creating a new
+            # PathSpec will complain.
+            # TODO: this should be handled inside PathSpec, which probably shouldn't
+            # modify its input.
+            include = path_spec.include
+            if include.endswith("{table}/**") and not path_spec.allow_double_stars:
+                include = include.removesuffix("**")
+
             s3_path_specs.append(
                 PathSpec(
-                    include=path_spec.include.replace("gs://", "s3://"),
+                    include=include.replace("gs://", "s3://"),
                     exclude=(
                         [exc.replace("gs://", "s3://") for exc in path_spec.exclude]
                         if path_spec.exclude
@@ -131,6 +152,11 @@ class GCSSource(StatefulIngestionSourceBase):
                     table_name=path_spec.table_name,
                     enable_compression=path_spec.enable_compression,
                     sample_files=path_spec.sample_files,
+                    allow_double_stars=path_spec.allow_double_stars,
+                    autodetect_partitions=path_spec.autodetect_partitions,
+                    include_hidden_folders=path_spec.include_hidden_folders,
+                    tables_filter_pattern=path_spec.tables_filter_pattern,
+                    traversal_method=path_spec.traversal_method,
                 )
             )
 
@@ -138,7 +164,9 @@ class GCSSource(StatefulIngestionSourceBase):
 
     def create_equivalent_s3_source(self, ctx: PipelineContext) -> S3Source:
         config = self.create_equivalent_s3_config()
-        s3_source = S3Source(config, PipelineContext(ctx.run_id))
+        # Create a new context for S3 source without graph to avoid duplicate checkpointer registration
+        s3_ctx = PipelineContext(run_id=ctx.run_id, pipeline_name=ctx.pipeline_name)
+        s3_source = S3Source(config, s3_ctx)
         return self.s3_source_overrides(s3_source)
 
     def s3_source_overrides(self, source: S3Source) -> S3Source: