acryl-datahub 1.0.0rc18__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl

datahub/ingestion/source/fivetran/fivetran_log_api.py

@@ -9,6 +9,7 @@ from sqlalchemy import create_engine
 
 from datahub.configuration.common import AllowDenyPattern, ConfigurationError
 from datahub.ingestion.source.fivetran.config import (
+    DISABLE_COL_LINEAGE_FOR_CONNECTOR_TYPES,
     Constant,
     FivetranLogConfig,
     FivetranSourceReport,
@@ -54,7 +55,7 @@ class FivetranLogAPI:
                         snowflake_destination_config.database,
                     )
                 )
-                fivetran_log_query.set_db(
+                fivetran_log_query.set_schema(
                     snowflake_destination_config.log_schema,
                 )
                 fivetran_log_database = snowflake_destination_config.database
@@ -66,8 +67,26 @@ class FivetranLogAPI:
                 engine = create_engine(
                     bigquery_destination_config.get_sql_alchemy_url(),
                 )
-                fivetran_log_query.set_db(bigquery_destination_config.dataset)
-                fivetran_log_database = bigquery_destination_config.dataset
+                fivetran_log_query.set_schema(bigquery_destination_config.dataset)
+
+                # The "database" should be the BigQuery project name.
+                result = engine.execute("SELECT @@project_id").fetchone()
+                if result is None:
+                    raise ValueError("Failed to retrieve BigQuery project ID")
+                fivetran_log_database = result[0]
+        elif destination_platform == "databricks":
+            databricks_destination_config = (
+                self.fivetran_log_config.databricks_destination_config
+            )
+            if databricks_destination_config is not None:
+                engine = create_engine(
+                    databricks_destination_config.get_sql_alchemy_url(
+                        databricks_destination_config.catalog
+                    ),
+                    **databricks_destination_config.get_options(),
+                )
+                fivetran_log_query.set_schema(databricks_destination_config.log_schema)
+                fivetran_log_database = databricks_destination_config.catalog
         else:
             raise ConfigurationError(
                 f"Destination platform '{destination_platform}' is not yet supported."
@@ -94,7 +113,11 @@ class FivetranLogAPI:
         """
         Returns dict of column lineage metadata with key as (<SOURCE_TABLE_ID>, <DESTINATION_TABLE_ID>)
         """
-        all_column_lineage = defaultdict(list)
+        all_column_lineage: Dict[Tuple[str, str], List] = defaultdict(list)
+
+        if not connector_ids:
+            return dict(all_column_lineage)
+
         column_lineage_result = self._query(
             self.fivetran_log_query.get_column_lineage_query(
                 connector_ids=connector_ids
@@ -112,7 +135,11 @@ class FivetranLogAPI:
         """
         Returns dict of table lineage metadata with key as 'CONNECTOR_ID'
         """
-        connectors_table_lineage_metadata = defaultdict(list)
+        connectors_table_lineage_metadata: Dict[str, List] = defaultdict(list)
+
+        if not connector_ids:
+            return dict(connectors_table_lineage_metadata)
+
         table_lineage_result = self._query(
             self.fivetran_log_query.get_table_lineage_query(connector_ids=connector_ids)
         )
@@ -190,7 +217,7 @@ class FivetranLogAPI:
         jobs: List[Job] = []
         if connector_sync_log is None:
             return jobs
-        for sync_id in connector_sync_log.keys():
+        for sync_id in connector_sync_log:
             if len(connector_sync_log[sync_id]) != 2:
                 # If both sync-start and sync-end event log not present for this sync that means sync is still in progress
                 continue
@@ -228,9 +255,15 @@ class FivetranLogAPI:
         return self._get_users().get(user_id)
 
     def _fill_connectors_lineage(self, connectors: List[Connector]) -> None:
-        connector_ids = [connector.connector_id for connector in connectors]
-        table_lineage_metadata = self._get_table_lineage_metadata(connector_ids)
-        column_lineage_metadata = self._get_column_lineage_metadata(connector_ids)
+        # Create 2 filtered connector_ids lists - one for table lineage and one for column lineage
+        tll_connector_ids: List[str] = []
+        cll_connector_ids: List[str] = []
+        for connector in connectors:
+            tll_connector_ids.append(connector.connector_id)
+            if connector.connector_type not in DISABLE_COL_LINEAGE_FOR_CONNECTOR_TYPES:
+                cll_connector_ids.append(connector.connector_id)
+        table_lineage_metadata = self._get_table_lineage_metadata(tll_connector_ids)
+        column_lineage_metadata = self._get_column_lineage_metadata(cll_connector_ids)
         for connector in connectors:
             connector.lineage = self._extract_connector_lineage(
                 table_lineage_result=table_lineage_metadata.get(connector.connector_id),

datahub/ingestion/source/fivetran/fivetran_query.py

@@ -6,34 +6,56 @@ MAX_COLUMN_LINEAGE_PER_CONNECTOR = 1000
 MAX_JOBS_PER_CONNECTOR = 500
 
 
+"""
+------------------------------------------------------------------------------------------------------------
+Fivetran Platform Connector Handling
+------------------------------------------------------------------------------------------------------------
+Current Query Change Log: August 2025 (See: https://fivetran.com/docs/changelog/2025/august-2025)
+
+All queries have to be updated as per Fivetran Platform Connector release if any. We expect customers
+and fivetran to keep platform connector configured for DataHub with auto sync enabled to get latest changes.
+
+References:
+- Fivetran Release Notes: https://fivetran.com/docs/changelog (Look for "Fivetran Platform Connector")
+- Latest Platform Connector Schema: https://fivetran.com/docs/logs/fivetran-platform?erdModal=open
+"""
+
+
 class FivetranLogQuery:
     # Note: All queries are written in Snowflake SQL.
     # They will be transpiled to the target database's SQL dialect at runtime.
 
     def __init__(self) -> None:
         # Select query db clause
-        self.db_clause: str = ""
-
-    def set_db(self, db_name: str) -> None:
-        self.db_clause = f"{db_name}."
+        self.schema_clause: str = ""
 
     def use_database(self, db_name: str) -> str:
         return f"use database {db_name}"
 
+    def set_schema(self, schema_name: str) -> None:
+        """
+        Using Snowflake quoted identifiers convention
+
+        Add double quotes around an identifier
+        Use two quotes to use the double quote character inside a quoted identifier
+        """
+        schema_name = schema_name.replace('"', '""')
+        self.schema_clause = f'"{schema_name}".'
+
     def get_connectors_query(self) -> str:
         return f"""\
 SELECT
-  connector_id,
+  connection_id,
   connecting_user_id,
   connector_type_id,
-  connector_name,
+  connection_name,
   paused,
   sync_frequency,
   destination_id
-FROM {self.db_clause}connector
+FROM {self.schema_clause}connection
 WHERE
   _fivetran_deleted = FALSE
-QUALIFY ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY _fivetran_synced DESC) = 1
+QUALIFY ROW_NUMBER() OVER (PARTITION BY connection_id ORDER BY _fivetran_synced DESC) = 1
 """
 
     def get_users_query(self) -> str:
@@ -42,7 +64,7 @@ SELECT id as user_id,
 given_name,
 family_name,
 email
-FROM {self.db_clause}user
+FROM {self.schema_clause}user
 """
 
     def get_sync_logs_query(
@@ -56,20 +78,20 @@ FROM {self.db_clause}user
         return f"""\
 WITH ranked_syncs AS (
     SELECT
-        connector_id,
+        connection_id,
         sync_id,
         MAX(CASE WHEN message_event = 'sync_start' THEN time_stamp END) as start_time,
         MAX(CASE WHEN message_event = 'sync_end' THEN time_stamp END) as end_time,
         MAX(CASE WHEN message_event = 'sync_end' THEN message_data END) as end_message_data,
-        ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY MAX(time_stamp) DESC) as rn
-    FROM {self.db_clause}log
+        ROW_NUMBER() OVER (PARTITION BY connection_id ORDER BY MAX(time_stamp) DESC) as rn
+    FROM {self.schema_clause}log
     WHERE message_event in ('sync_start', 'sync_end')
     AND time_stamp > CURRENT_TIMESTAMP - INTERVAL '{syncs_interval} days'
-    AND connector_id IN ({formatted_connector_ids})
-    GROUP BY connector_id, sync_id
+    AND connection_id IN ({formatted_connector_ids})
+    GROUP BY connection_id, sync_id
 )
 SELECT
-    connector_id,
+    connection_id,
     sync_id,
     start_time,
     end_time,
@@ -78,7 +100,7 @@ FROM ranked_syncs
 WHERE rn <= {MAX_JOBS_PER_CONNECTOR}
     AND start_time IS NOT NULL
     AND end_time IS NOT NULL
-ORDER BY connector_id, end_time DESC
+ORDER BY connection_id, end_time DESC
 """
 
     def get_table_lineage_query(self, connector_ids: List[str]) -> str:
@@ -90,7 +112,7 @@ SELECT
     *
 FROM (
     SELECT
-        stm.connector_id as connector_id,
+        stm.connection_id as connection_id,
         stm.id as source_table_id,
         stm.name as source_table_name,
         ssm.name as source_schema_name,
@@ -98,18 +120,18 @@ FROM (
         dtm.name as destination_table_name,
         dsm.name as destination_schema_name,
         tl.created_at as created_at,
-        ROW_NUMBER() OVER (PARTITION BY stm.connector_id, stm.id, dtm.id ORDER BY tl.created_at DESC) as table_combo_rn
-    FROM {self.db_clause}table_lineage as tl
-    JOIN {self.db_clause}source_table_metadata as stm on tl.source_table_id = stm.id
-    JOIN {self.db_clause}destination_table_metadata as dtm on tl.destination_table_id = dtm.id
-    JOIN {self.db_clause}source_schema_metadata as ssm on stm.schema_id = ssm.id
-    JOIN {self.db_clause}destination_schema_metadata as dsm on dtm.schema_id = dsm.id
-    WHERE stm.connector_id IN ({formatted_connector_ids})
+        ROW_NUMBER() OVER (PARTITION BY stm.connection_id, stm.id, dtm.id ORDER BY tl.created_at DESC) as table_combo_rn
+    FROM {self.schema_clause}table_lineage as tl
+    JOIN {self.schema_clause}source_table as stm on tl.source_table_id = stm.id -- stm: source_table_metadata
+    JOIN {self.schema_clause}destination_table as dtm on tl.destination_table_id = dtm.id -- dtm: destination_table_metadata
+    JOIN {self.schema_clause}source_schema as ssm on stm.schema_id = ssm.id -- ssm: source_schema_metadata
+    JOIN {self.schema_clause}destination_schema as dsm on dtm.schema_id = dsm.id -- dsm: destination_schema_metadata
+    WHERE stm.connection_id IN ({formatted_connector_ids})
 )
 -- Ensure that we only get back one entry per source and destination pair.
 WHERE table_combo_rn = 1
-QUALIFY ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY created_at DESC) <= {MAX_TABLE_LINEAGE_PER_CONNECTOR}
-ORDER BY connector_id, created_at DESC
+QUALIFY ROW_NUMBER() OVER (PARTITION BY connection_id ORDER BY created_at DESC) <= {MAX_TABLE_LINEAGE_PER_CONNECTOR}
+ORDER BY connection_id, created_at DESC
 """
 
     def get_column_lineage_query(self, connector_ids: List[str]) -> str:
@@ -124,25 +146,25 @@ SELECT
     destination_column_name
 FROM (
     SELECT
-        stm.connector_id as connector_id,
+        stm.connection_id as connection_id,
         scm.table_id as source_table_id,
         dcm.table_id as destination_table_id,
         scm.name as source_column_name,
         dcm.name as destination_column_name,
         cl.created_at as created_at,
-        ROW_NUMBER() OVER (PARTITION BY stm.connector_id, cl.source_column_id, cl.destination_column_id ORDER BY cl.created_at DESC) as column_combo_rn
-    FROM {self.db_clause}column_lineage as cl
-    JOIN {self.db_clause}source_column_metadata as scm
+        ROW_NUMBER() OVER (PARTITION BY stm.connection_id, cl.source_column_id, cl.destination_column_id ORDER BY cl.created_at DESC) as column_combo_rn
+    FROM {self.schema_clause}column_lineage as cl
+    JOIN {self.schema_clause}source_column as scm -- scm: source_column_metadata
       ON cl.source_column_id = scm.id
-    JOIN {self.db_clause}destination_column_metadata as dcm
+    JOIN {self.schema_clause}destination_column as dcm -- dcm: destination_column_metadata
       ON cl.destination_column_id = dcm.id
-    -- Only joining source_table_metadata to get the connector_id.
-    JOIN {self.db_clause}source_table_metadata as stm
+    -- Only joining source_table to get the connection_id.
+    JOIN {self.schema_clause}source_table as stm -- stm: source_table_metadata
       ON scm.table_id = stm.id
-    WHERE stm.connector_id IN ({formatted_connector_ids})
+    WHERE stm.connection_id IN ({formatted_connector_ids})
 )
 -- Ensure that we only get back one entry per (connector, source column, destination column) pair.
 WHERE column_combo_rn = 1
-QUALIFY ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY created_at DESC) <= {MAX_COLUMN_LINEAGE_PER_CONNECTOR}
-ORDER BY connector_id, created_at DESC
+QUALIFY ROW_NUMBER() OVER (PARTITION BY connection_id ORDER BY created_at DESC) <= {MAX_COLUMN_LINEAGE_PER_CONNECTOR}
+ORDER BY connection_id, created_at DESC
 """

datahub/ingestion/source/fivetran/fivetran_rest_api.py

@@ -0,0 +1,65 @@
+import logging
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+
+from datahub.ingestion.source.fivetran.config import (
+    FivetranAPIConfig,
+)
+from datahub.ingestion.source.fivetran.response_models import FivetranConnectionDetails
+
+logger = logging.getLogger(__name__)
+
+# Retry configuration constants
+RETRY_MAX_TIMES = 3
+RETRY_STATUS_CODES = [429, 500, 502, 503, 504]
+RETRY_BACKOFF_FACTOR = 1
+RETRY_ALLOWED_METHODS = ["GET"]
+
+
+class FivetranAPIClient:
+    """Client for interacting with the Fivetran REST API."""
+
+    def __init__(self, config: FivetranAPIConfig) -> None:
+        self.config = config
+        self._session = self._create_session()
+
+    def _create_session(self) -> requests.Session:
+        """
+        Create a session with retry logic and basic authentication
+        """
+        requests_session = requests.Session()
+
+        # Configure retry strategy for transient failures
+        retry_strategy = Retry(
+            total=RETRY_MAX_TIMES,
+            backoff_factor=RETRY_BACKOFF_FACTOR,
+            status_forcelist=RETRY_STATUS_CODES,
+            allowed_methods=RETRY_ALLOWED_METHODS,
+            raise_on_status=True,
+        )
+
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        requests_session.mount("http://", adapter)
+        requests_session.mount("https://", adapter)
+
+        # Set up basic authentication
+        requests_session.auth = (self.config.api_key, self.config.api_secret)
+        requests_session.headers.update(
+            {
+                "Content-Type": "application/json",
+                "Accept": "application/json",
+            }
+        )
+        return requests_session
+
+    def get_connection_details_by_id(
+        self, connection_id: str
+    ) -> FivetranConnectionDetails:
+        """Get details for a specific connection."""
+        connection_details = self._session.get(
+            f"{self.config.base_url}/v1/connections/{connection_id}",
+            timeout=self.config.request_timeout_sec,
+        )
+        return FivetranConnectionDetails(**connection_details.json().get("data", {}))

datahub/ingestion/source/fivetran/response_models.py

@@ -0,0 +1,97 @@
+import datetime
+from typing import Dict, List
+
+from pydantic import BaseModel
+
+
+class FivetranConnectionWarnings(BaseModel):
+    code: str  # Warning Code
+    message: str  # Warning Message
+    details: Dict  # Warning Details
+
+
+class FivetranConnectionStatus(BaseModel):
+    setup_state: str  # Setup State
+    schema_status: str  # Schema Status
+    sync_state: str  # Sync State
+    update_state: str  # Update State
+    is_historical_sync: bool  # Is Historical Sync
+    warnings: List[FivetranConnectionWarnings]  # Warnings
+
+
+class FivetranConnectionConfig(BaseModel):
+    # Note: Connection Config is different for different connectors
+    auth_type: str  # Auth Type
+    sheet_id: str  # Sheet ID - URL to the Google Sheet
+    named_range: str  # Named Range
+
+
+class FivetranConnectionSourceSyncDetails(BaseModel):
+    last_synced: datetime.datetime  # Last Synced
+
+
+class FivetranConnectionDetails(BaseModel):
+    """
+    Note: This reponse class only captures fields that are relevant to the Google Sheets Connector
+    """
+
+    id: str  # Source ID
+    group_id: str  # Destination ID
+    service: str  # Connector Type
+    created_at: datetime.datetime
+    succeeded_at: datetime.datetime
+    paused: bool  # Paused Status
+    sync_frequency: int  # Sync Frequency (minutes)
+    status: FivetranConnectionStatus  # Status
+    config: FivetranConnectionConfig  # Connection Config
+    source_sync_details: FivetranConnectionSourceSyncDetails  # Source Sync Details
+
+    """
+    # Sample Response for Google Sheets Connector
+    {
+        "code": "Success",
+        "data": {
+            "id": "dialectical_remindful",
+            "group_id": "empties_classification",
+            "service": "google_sheets",
+            "service_version": 1,
+            "schema": "fivetran_google_sheets.fivetran_google_sheets",
+            "connected_by": "sewn_restrained",
+            "created_at": "2025-10-06T17:53:01.554289Z",
+            "succeeded_at": "2025-10-06T22:55:45.275000Z",
+            "failed_at": null,
+            "paused": true,
+            "pause_after_trial": false,
+            "sync_frequency": 360,
+            "data_delay_threshold": 0,
+            "data_delay_sensitivity": "NORMAL",
+            "private_link_id": null,
+            "networking_method": "Directly",
+            "proxy_agent_id": null,
+            "schedule_type": "auto",
+            "status": {
+                "setup_state": "connected",
+                "schema_status": "ready",
+                "sync_state": "paused",
+                "update_state": "on_schedule",
+                "is_historical_sync": false,
+                "tasks": [],
+                "warnings": [
+                    {
+                        "code": "snowflake_discontinuing_password_auth",
+                        "message": "Snowflake is discontinuing username/password authentication",
+                        "details": {}
+                    }
+                ]
+            },
+            "config": {
+                "auth_type": "ServiceAccount",
+                "sheet_id": "https://docs.google.com/spreadsheets/d/1A82PdLAE7NXLLb5JcLPKeIpKUMytXQba5Z-Ei-mbXLo/edit?gid=0#gid=0",
+                "named_range": "Fivetran_Test_Range"
+            },
+            "source_sync_details": {
+                "last_synced": "2025-10-06T22:55:27.371Z"
+            }
+        }
+    }
+    """

datahub/ingestion/source/gc/datahub_gc.py

@@ -34,7 +34,6 @@ from datahub.ingestion.source.gc.soft_deleted_entity_cleanup import (
     SoftDeletedEntitiesCleanupConfig,
     SoftDeletedEntitiesReport,
 )
-from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 
 logger = logging.getLogger(__name__)
 
@@ -87,7 +86,6 @@ class DataHubGcSourceReport(
     DataProcessCleanupReport,
     SoftDeletedEntitiesReport,
     DatahubExecutionRequestCleanupReport,
-    IngestionStageReport,
 ):
     expired_tokens_revoked: int = 0