openmetadata-ingestion 1.10.2.0__py3-none-any.whl → 1.10.4.0__py3-none-any.whl

metadata/ingestion/source/pipeline/fivetran/client.py

@@ -42,6 +42,10 @@ class FivetranClient:
             auth_header="Authorization",
             auth_token=lambda: (api_token[2:-1], 0),
             auth_token_mode="Basic",
+            retry=20,
+            retry_wait=60,
+            retry_codes=[429],
+            limit_codes=[],
         )
         self.client = REST(client_config)
 
@@ -88,15 +92,15 @@ class FivetranClient:
         Method returns destination details
         """
         response = self.client.get(f"/connectors/{connector_id}/schemas")
-        return response.get("data", {}).get("schemas", [])
+        return response.get("data", {}).get("schemas", {})
 
     def get_connector_column_lineage(
         self, connector_id: str, schema_name: str, table_name: str
     ) -> dict:
         """
-        Method returns destination details
+        Method returns column lineage details for a table
         """
         response: Optional[Response] = self.client.get(
             f"/connectors/{connector_id}/schemas/{schema_name}/tables/{table_name}/columns"
         )
-        return response.get("data", {}).get("columns", [])
+        return response.get("data", {}).get("columns", {})

metadata/ingestion/source/pipeline/fivetran/metadata.py

@@ -13,7 +13,7 @@ Fivetran source to extract metadata
 """
 
 import traceback
-from typing import Iterable, List, Optional, Union, cast
+from typing import Iterable, List, Optional, cast
 
 from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
@@ -22,7 +22,6 @@ from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.pipeline.fivetranConnection import (
     FivetranConnection,
 )
-from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
@@ -45,8 +44,6 @@ from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.pipeline.fivetran.client import FivetranClient
 from metadata.ingestion.source.pipeline.fivetran.models import FivetranPipelineDetails
-from metadata.ingestion.source.pipeline.openlineage.models import TableDetails
-from metadata.ingestion.source.pipeline.openlineage.utils import FQNNotFoundException
 from metadata.ingestion.source.pipeline.pipeline_service import PipelineServiceSource
 from metadata.utils import fqn
 from metadata.utils.logger import ingestion_logger
@@ -117,48 +114,59 @@ class FivetranSource(PipelineServiceSource):
         """Method to get task & pipeline status"""
 
     def fetch_column_lineage(
-        self, pipeline_details: FivetranPipelineDetails, schema, schema_data, table
+        self,
+        pipeline_details: FivetranPipelineDetails,
+        schema_name: str,
+        schema_data: dict,
+        table_name: str,
+        from_table_entity: Table,
+        to_table_entity: Table,
     ) -> List[Optional[ColumnLineage]]:
-        col_details = self.client.get_connector_column_lineage(
-            pipeline_details.connector_id, schema_name=schema, table_name=table
-        )
+        """
+        Fetch column-level lineage between source and destination tables in a Fivetran connector.
+
+        This method retrieves column mappings from Fivetran and creates ColumnLineage objects
+        for each enabled column transformation, mapping source columns to their corresponding
+        destination columns.
+
+        :param pipeline_details: FivetranPipelineDetails containing connector information
+        :param schema_name: Name of the source schema
+        :param schema_data: Dictionary containing schema configuration data
+        :param table_name: Name of the source table
+        :param from_table_entity: Source Table entity from OpenMetadata
+        :param to_table_entity: Destination Table entity from OpenMetadata
+        :return: List of ColumnLineage objects representing column-to-column mappings, empty list if none found
+        """
+        pipeline_name = self.get_pipeline_name(pipeline_details)
+
         col_lineage_arr = []
-        try:
-            from_entity_fqn: Optional[str] = self._get_table_fqn_from_om(
-                table_details=TableDetails(schema=schema, name=table)
+        for column_name, column_data in self.client.get_connector_column_lineage(
+            pipeline_details.connector_id,
+            schema_name=schema_name,
+            table_name=table_name,
+        ).items():
+            if not column_data.get("enabled"):
+                logger.debug(
+                    f"Skipping column [{schema_name}.{table_name}.{column_name}] for pipeline"
+                    f" [{pipeline_name}] lineage - column is disabled"
+                )
+                continue
+
+            from_col = get_column_fqn(
+                table_entity=from_table_entity, column=column_name
             )
-            to_entity_fqn: Optional[str] = self._get_table_fqn_from_om(
-                table_details=TableDetails(
-                    schema=schema_data.get("name_in_destination"),
-                    name=schema_data["tables"][table]["name_in_destination"],
+            to_col = get_column_fqn(
+                table_entity=to_table_entity,
+                column=column_data.get("name_in_destination"),
+            )
+            col_lineage_arr.append(
+                ColumnLineage(
+                    fromColumns=[from_col],
+                    toColumn=to_col,
+                    function=None,
                 )
             )
-        except FQNNotFoundException:
-            to_entity_fqn = ""
-            from_entity_fqn = ""
 
-        if from_entity_fqn and to_entity_fqn:
-            to_table_entity = self.metadata.get_by_name(entity=Table, fqn=to_entity_fqn)
-            from_table_entity = self.metadata.get_by_name(
-                entity=Table, fqn=from_entity_fqn
-            )
-            for key, value in col_details.items():
-                if value["enabled"] == True:
-                    if from_table_entity and to_table_entity:
-                        from_col = get_column_fqn(
-                            table_entity=from_table_entity, column=key
-                        )
-                        to_col = get_column_fqn(
-                            table_entity=to_table_entity,
-                            column=value.get("name_in_destination"),
-                        )
-                    col_lineage_arr.append(
-                        ColumnLineage(
-                            toColumn=to_col,
-                            fromColumns=[from_col],
-                            function=None,
-                        )
-                    )
         return col_lineage_arr if col_lineage_arr else []
 
     def yield_pipeline_lineage_details(
@@ -170,66 +178,90 @@ class FivetranSource(PipelineServiceSource):
         :return: Lineage from inlets and outlets
         """
         self.client = cast(FivetranClient, self.client)
-        source_service = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=pipeline_details.source.get("schema")
-        )
-        if not source_service:
-            es_resp: Union[
-                List[DatabaseService], None
-            ] = self.metadata.es_search_from_fqn(
-                DatabaseService, pipeline_details.source.get("schema", "")
-            )
-            if es_resp and len(es_resp) > 0 and es_resp[0].fullyQualifiedName:
-                source_service = self.metadata.get_by_name(
-                    entity=DatabaseService,
-                    fqn=(es_resp[0].fullyQualifiedName.root),
-                )
-        destination_service = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=pipeline_details.group.get("name")
-        )
+        pipeline_name = self.get_pipeline_name(pipeline_details)
 
-        if not source_service or not destination_service:
-            return
+        source_database_name = pipeline_details.source.get("config", {}).get("database")
+        destination_database_name = pipeline_details.destination.get("config", {}).get(
+            "database"
+        )
 
-        for schema, schema_data in self.client.get_connector_schema_details(
+        for schema_name, schema_data in self.client.get_connector_schema_details(
             connector_id=pipeline_details.source.get("id")
         ).items():
+            if not schema_data.get("enabled"):
+                logger.debug(
+                    f"Skipping schema [{schema_name}] for pipeline [{pipeline_name}] lineage"
+                    " - schema is disabled"
+                )
+                continue
+
+            source_schema_name = schema_name
+            destination_schema_name = schema_data.get("name_in_destination")
+
+            for table_name, table_data in schema_data.get("tables", {}).items():
+                if not table_data.get("enabled"):
+                    logger.debug(
+                        f"Skipping table [{schema_name}].[{table_name}] for pipeline [{pipeline_name}]"
+                        " lineage - table is disabled"
+                    )
+                    continue
+
+                source_table_name = table_name
+                destination_table_name = table_data.get("name_in_destination")
+
+                from_fqn = None
+                from_entity = None
+                for db_service_name in self.get_db_service_names() or "*":
+                    from_fqn = fqn.build(
+                        metadata=self.metadata,
+                        entity_type=Table,
+                        table_name=source_table_name,
+                        database_name=source_database_name,
+                        schema_name=source_schema_name,
+                        service_name=db_service_name,
+                    )
+                    from_entity = self.metadata.get_by_name(entity=Table, fqn=from_fqn)
+                    if from_entity:
+                        break
+
+                if not from_entity:
+                    logger.debug(
+                        f"Lineage skipped for pipeline [{pipeline_name}]"
+                        f" since source table [{from_fqn}] not found."
+                    )
+                    continue
+
+                to_fqn = None
+                to_entity = None
+                for db_service_name in self.get_db_service_names() or "*":
+                    to_fqn = fqn.build(
+                        self.metadata,
+                        Table,
+                        table_name=destination_table_name,
+                        database_name=destination_database_name,
+                        schema_name=destination_schema_name,
+                        service_name=db_service_name,
+                    )
+                    to_entity = self.metadata.get_by_name(entity=Table, fqn=to_fqn)
+                    if to_entity:
+                        break
+
+                if not to_entity:
+                    logger.debug(
+                        f"Lineage skipped for pipeline [{pipeline_name}]"
+                        f" since destination table [{to_fqn}] not found."
+                    )
+                    continue
 
-            for table in schema_data.get("tables", {}).keys():
                 col_lineage_arr = self.fetch_column_lineage(
                     pipeline_details=pipeline_details,
-                    schema=schema,
+                    schema_name=schema_name,
                     schema_data=schema_data,
-                    table=table,
+                    table_name=table_name,
+                    from_table_entity=from_entity,
+                    to_table_entity=to_entity,
                 )
 
-                from_fqn = fqn.build(
-                    metadata=self.metadata,
-                    entity_type=Table,
-                    table_name=table,
-                    database_name=pipeline_details.source.get("config", {}).get(
-                        "database"
-                    ),
-                    schema_name=schema,
-                    service_name=pipeline_details.source.get("schema"),
-                )
-
-                to_fqn = fqn.build(
-                    self.metadata,
-                    Table,
-                    table_name=table,
-                    database_name=pipeline_details.destination.get("config", {}).get(
-                        "database"
-                    ),
-                    schema_name=f"{pipeline_details.source.get('schema')}_{schema}",
-                    service_name=pipeline_details.group.get("name"),
-                )
-
-                from_entity = self.metadata.get_by_name(entity=Table, fqn=from_fqn)
-                to_entity = self.metadata.get_by_name(entity=Table, fqn=to_fqn)
-                if not from_entity or not to_entity:
-                    logger.info(f"Lineage Skipped for {from_fqn} - {to_fqn}")
-                    continue
                 pipeline_fqn = fqn.build(
                     metadata=self.metadata,
                     entity_type=Pipeline,

metadata/ingestion/source/storage/gcs/connection.py

@@ -117,16 +117,26 @@ class Tester:
             raise SourceConnectionException("No buckets found in provided projects")
         for bucket_test in self.bucket_tests:
             client = self.client.storage_client.clients[bucket_test.project_id]
-            blob = next(client.list_blobs(bucket_test.bucket_name))
-            bucket_test.blob_name = blob.name
+            try:
+                blob = next(client.list_blobs(bucket_test.bucket_name))
+                bucket_test.blob_name = blob.name
+            except StopIteration:
+                # Empty bucket - this is valid, we can list blobs
+                # even if there are none
+                logger.debug(
+                    f"Bucket {bucket_test.bucket_name} is empty, but list "
+                    "permission is working correctly"
+                )
 
     def get_blob(self):
         if not self.bucket_tests:
             raise SourceConnectionException("No buckets found in provided projects")
         for bucket_test in self.bucket_tests:
-            client = self.client.storage_client.clients[bucket_test.project_id]
-            bucket = client.get_bucket(bucket_test.bucket_name)
-            bucket.get_blob(bucket_test.blob_name)
+            # Only test get_blob if we found a blob in the list_blobs test
+            if bucket_test.blob_name:
+                client = self.client.storage_client.clients[bucket_test.project_id]
+                bucket = client.get_bucket(bucket_test.bucket_name)
+                bucket.get_blob(bucket_test.blob_name)
 
     def get_metrics(self):
         for project_id in self.client.storage_client.clients.keys():

metadata/ingestion/source/storage/storage_service.py

@@ -275,6 +275,7 @@ class StorageServiceSource(TopologyRunnerMixin, Source, ABC):
                 partitionColumns=entry.partitionColumns,
                 separator=entry.separator,
                 depth=entry.depth,
+                unstructuredFormats=entry.unstructuredFormats,
             )
             for entry in manifest.entries
             if entry.containerName == container_name