openmetadata-ingestion 1.9.4.0__py3-none-any.whl → 1.9.6.0__py3-none-any.whl

metadata/ingestion/source/storage/gcs/metadata.py

@@ -12,9 +12,10 @@
 import json
 import secrets
 import traceback
+from copy import deepcopy
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import Dict, Iterable, List, Optional
+from typing import Dict, Iterable, List, Optional, Tuple
 
 from google.cloud.exceptions import NotFound
 from google.cloud.monitoring_v3.types import TimeInterval
@@ -42,6 +43,7 @@ from metadata.generated.schema.metadataIngestion.storage.containerMetadataConfig
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
@@ -63,6 +65,8 @@ from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
+WILD_CARD = "*"
+
 
 class GCSMetric(Enum):
     NUMBER_OF_OBJECTS = "storage.googleapis.com/storage/object_count"
@@ -82,6 +86,7 @@ class GcsSource(StorageServiceSource):
             for project_id, client in self.gcs_clients.storage_client.clients.items()
         }
         self._bucket_cache: Dict[str, Container] = {}
+        self._unstructured_container_cache: Dict[str, Tuple[str, str]] = {}
 
     @classmethod
     def create(
@@ -115,6 +120,10 @@ class GcsSource(StorageServiceSource):
                     entity=Container, fqn=container_fqn
                 )
                 self._bucket_cache[bucket_name] = container_entity
+                self._unstructured_container_cache[container_fqn] = (
+                    container_entity.id.root,
+                    "",  # Empty string for bucket root (no relative path)
+                )
                 parent_entity: EntityReference = EntityReference(
                     id=self._bucket_cache[bucket_name].id.root, type="container"
                 )
@@ -132,6 +141,11 @@ class GcsSource(StorageServiceSource):
                             entries=manifest_entries_for_current_bucket,
                             parent=parent_entity,
                         )
+                        yield from self._generate_unstructured_containers(
+                            bucket_response=bucket_response,
+                            entries=manifest_entries_for_current_bucket,
+                            parent=parent_entity,
+                        )
                         # nothing else do to for the current bucket, skipping to the next
                         continue
                 # If no global file, or no valid entries in the manifest, check for bucket level metadata file
@@ -142,6 +156,14 @@ class GcsSource(StorageServiceSource):
                         entries=metadata_config.entries,
                         parent=parent_entity,
                     )
+                    yield from self._generate_unstructured_containers(
+                        bucket_response=bucket_response,
+                        entries=metadata_config.entries,
+                        parent=parent_entity,
+                    )
+
+                # clean up the cache after each bucket
+                self._unstructured_container_cache.clear()
 
             except ValidationError as err:
                 self.status.failed(
@@ -164,12 +186,12 @@ class GcsSource(StorageServiceSource):
         self, container_details: GCSContainerDetails
     ) -> Iterable[Either[CreateContainerRequest]]:
         container_request = CreateContainerRequest(
-            name=container_details.name,
+            name=EntityName(container_details.name),
             prefix=container_details.prefix,
             numberOfObjects=container_details.number_of_objects,
             size=container_details.size,
             dataModel=container_details.data_model,
-            service=self.context.get().objectstore_service,
+            service=FullyQualifiedEntityName(self.context.get().objectstore_service),
             parent=container_details.parent,
             sourceUrl=container_details.sourceUrl,
             fileFormats=container_details.file_formats,
@@ -178,6 +200,33 @@ class GcsSource(StorageServiceSource):
         yield Either(right=container_request)
         self.register_record(container_request=container_request)
 
+    def get_size(
+        self, bucket_name: str, project_id: str, file_path: str
+    ) -> Optional[float]:
+        """
+        Method to get the size of the file
+        """
+        try:
+            client = self.gcs_clients.storage_client.clients[project_id]
+            bucket = client.get_bucket(bucket_name)
+            blob = bucket.blob(file_path)
+            blob.reload()
+            return blob.size
+        except Exception as exc:
+            logger.debug(f"Failed to get size of file due to {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
+    def is_valid_unstructured_file(self, accepted_extensions: List, key: str) -> bool:
+        if WILD_CARD in accepted_extensions:
+            return True
+
+        for ext in accepted_extensions:
+            if key.endswith(ext):
+                return True
+
+        return False
+
     def _generate_container_details(
         self,
         bucket_response: GCSBucketResponse,
@@ -185,6 +234,10 @@ class GcsSource(StorageServiceSource):
         parent: Optional[EntityReference] = None,
     ) -> Optional[GCSContainerDetails]:
         bucket_name = bucket_response.name
+
+        if not metadata_entry.structureFormat:
+            return None
+
         sample_key = self._get_sample_file_path(
             bucket=bucket_response, metadata_entry=metadata_entry
         )
@@ -226,33 +279,80 @@ class GcsSource(StorageServiceSource):
                     sourceUrl=self._get_object_source_url(
                         bucket=bucket_response,
                         prefix=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                        is_file=False,  # Structured containers are directories
                     ),
                 )
         return None
 
+    def _generate_structured_containers_by_depth(
+        self,
+        bucket_response: GCSBucketResponse,
+        metadata_entry: MetadataEntry,
+        parent: Optional[EntityReference] = None,
+    ) -> Iterable[GCSContainerDetails]:
+        try:
+            prefix = self._get_sample_file_prefix(metadata_entry=metadata_entry)
+            if prefix:
+                client = self.gcs_clients.storage_client.clients[
+                    bucket_response.project_id
+                ]
+                response = client.list_blobs(
+                    bucket_response.name,
+                    prefix=prefix,
+                    max_results=1000,
+                )
+                # total depth is depth of prefix + depth of the metadata entry
+                total_depth = metadata_entry.depth + len(prefix[:-1].split("/"))
+                candidate_keys = {
+                    "/".join(entry.name.split("/")[:total_depth]) + "/"
+                    for entry in response
+                    if entry and entry.name and len(entry.name.split("/")) > total_depth
+                }
+                for key in candidate_keys:
+                    metadata_entry_copy = deepcopy(metadata_entry)
+                    metadata_entry_copy.dataPath = key.strip(KEY_SEPARATOR)
+                    structured_container: Optional[
+                        GCSContainerDetails
+                    ] = self._generate_container_details(
+                        bucket_response=bucket_response,
+                        metadata_entry=metadata_entry_copy,
+                        parent=parent,
+                    )
+                    if structured_container:
+                        yield structured_container
+        except Exception as err:
+            logger.warning(
+                f"Error while generating structured containers by depth for {metadata_entry.dataPath} - {err}"
+            )
+            logger.debug(traceback.format_exc())
+
     def _generate_structured_containers(
         self,
         bucket_response: GCSBucketResponse,
         entries: List[MetadataEntry],
         parent: Optional[EntityReference] = None,
-    ) -> List[GCSContainerDetails]:
-        result: List[GCSContainerDetails] = []
+    ) -> Iterable[GCSContainerDetails]:
         for metadata_entry in entries:
             logger.info(
                 f"Extracting metadata from path {metadata_entry.dataPath.strip(KEY_SEPARATOR)} "
                 f"and generating structured container"
             )
-            structured_container: Optional[
-                GCSContainerDetails
-            ] = self._generate_container_details(
-                bucket_response=bucket_response,
-                metadata_entry=metadata_entry,
-                parent=parent,
-            )
-            if structured_container:
-                result.append(structured_container)
-
-        return result
+            if metadata_entry.depth == 0:
+                structured_container: Optional[
+                    GCSContainerDetails
+                ] = self._generate_container_details(
+                    bucket_response=bucket_response,
+                    metadata_entry=metadata_entry,
+                    parent=parent,
+                )
+                if structured_container:
+                    yield structured_container
+            else:
+                yield from self._generate_structured_containers_by_depth(
+                    bucket_response=bucket_response,
+                    metadata_entry=metadata_entry,
+                    parent=parent,
+                )
 
     def _fetch_bucket(self, bucket_name: str) -> GCSBucketResponse:
         for project_id, client in self.gcs_clients.storage_client.clients.items():
@@ -409,31 +509,190 @@ class GcsSource(StorageServiceSource):
         Method to get the source url of GCS bucket
         """
         try:
-            return (
-                f"https://console.cloud.google.com/storage/browser/{bucket.name}"
-                f";tab=objects?project={bucket.project_id}"
-            )
+            return f"https://console.cloud.google.com/storage/browser/{bucket.name}?project={bucket.project_id}"
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Unable to get source url: {exc}")
         return None
 
     def _get_object_source_url(
-        self, bucket: GCSBucketResponse, prefix: str
+        self, bucket: GCSBucketResponse, prefix: str, is_file: bool = False
     ) -> Optional[str]:
         """
-        Method to get the source url of GCS object
+        Method to get the source url of GCS object or directory
         """
         try:
-            return (
-                f"https://console.cloud.google.com/storage/browser/_details/{bucket.name}/{prefix}"
-                f";tab=live_object?project={bucket.project_id}"
-            )
+            # Remove trailing slash from prefix if present
+            clean_prefix = prefix.rstrip("/")
+
+            if is_file:
+                # For files, use the _details path with tab=live_object
+                return f"https://console.cloud.google.com/storage/browser/_details/{bucket.name}/{clean_prefix};tab=live_object"
+            else:
+                # For directories/prefixes, use the browser view
+                return f"https://console.cloud.google.com/storage/browser/{bucket.name}/{clean_prefix}?project={bucket.project_id}"
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Unable to get source url: {exc}")
         return None
 
+    def _yield_parents_of_unstructured_container(
+        self,
+        bucket_name: str,
+        project_id: str,
+        list_of_parent: List[str],
+        parent: Optional[EntityReference] = None,
+    ):
+        relative_path = ""  # Path relative to bucket for URLs
+        sub_parent = parent
+        for i in range(len(list_of_parent) - 1):
+            container_fqn = fqn._build(  # pylint: disable=protected-access
+                *(
+                    self.context.get().objectstore_service,
+                    bucket_name,
+                    *list_of_parent[: i + 1],
+                )
+            )
+            if container_fqn in self._unstructured_container_cache:
+                parent_id, relative_path = self._unstructured_container_cache[
+                    container_fqn
+                ]
+                sub_parent = EntityReference(id=parent_id, type="container")
+                continue
+
+            # Build the relative path (no gs:// prefix)
+            current_relative_path = KEY_SEPARATOR.join(list_of_parent[: i + 1])
+            yield GCSContainerDetails(
+                name=list_of_parent[i],
+                prefix=KEY_SEPARATOR + current_relative_path,
+                file_formats=[],
+                parent=sub_parent,
+                fullPath=self._get_full_path(bucket_name, current_relative_path),
+                sourceUrl=self._get_object_source_url(
+                    bucket=GCSBucketResponse(
+                        name=bucket_name, project_id=project_id, creation_date=None
+                    ),
+                    prefix=current_relative_path,
+                    is_file=False,  # Parent containers are directories
+                ),
+            )
+            container_entity = self.metadata.get_by_name(
+                entity=Container, fqn=container_fqn
+            )
+            relative_path = current_relative_path
+            self._unstructured_container_cache[container_fqn] = (
+                container_entity.id.root,
+                relative_path,
+            )
+            sub_parent = EntityReference(id=container_entity.id.root, type="container")
+
+    def _yield_nested_unstructured_containers(
+        self,
+        bucket_response: GCSBucketResponse,
+        metadata_entry: MetadataEntry,
+        parent: Optional[EntityReference] = None,
+    ):
+        bucket_name = bucket_response.name
+        client = self.gcs_clients.storage_client.clients[bucket_response.project_id]
+        response = client.list_blobs(
+            bucket_name,
+            prefix=metadata_entry.dataPath,
+            max_results=1000,
+        )
+        candidate_keys = [
+            entry.name
+            for entry in response
+            if entry and entry.name and not entry.name.endswith("/")
+        ]
+        for key in candidate_keys:
+            if self.is_valid_unstructured_file(metadata_entry.unstructuredFormats, key):
+                logger.debug(
+                    f"Extracting metadata from path {key.strip(KEY_SEPARATOR)} "
+                    f"and generating unstructured container"
+                )
+                list_of_parent = key.strip(KEY_SEPARATOR).split(KEY_SEPARATOR)
+                yield from self._yield_parents_of_unstructured_container(
+                    bucket_name, bucket_response.project_id, list_of_parent, parent
+                )
+                parent_fqn = fqn._build(  # pylint: disable=protected-access
+                    *(
+                        self.context.get().objectstore_service,
+                        bucket_name,
+                        *list_of_parent[:-1],
+                    )
+                )
+                parent_id, parent_relative_path = self._unstructured_container_cache[
+                    parent_fqn
+                ]
+                container_fqn = fqn._build(  # pylint: disable=protected-access
+                    *(
+                        self.context.get().objectstore_service,
+                        bucket_name,
+                        *list_of_parent,
+                    )
+                )
+                size = self.get_size(bucket_name, bucket_response.project_id, key)
+                yield GCSContainerDetails(
+                    name=list_of_parent[-1],
+                    prefix=KEY_SEPARATOR + parent_relative_path
+                    if parent_relative_path
+                    else KEY_SEPARATOR,
+                    file_formats=[],
+                    size=size,
+                    container_fqn=container_fqn,
+                    leaf_container=True,
+                    parent=EntityReference(id=parent_id, type="container"),
+                    fullPath=self._get_full_path(bucket_name, key),
+                    sourceUrl=self._get_object_source_url(
+                        bucket=bucket_response,
+                        prefix=key,  # Use the full key path for the file
+                        is_file=True,  # Leaf containers are files
+                    ),
+                )
+
+    def _generate_unstructured_containers(
+        self,
+        bucket_response: GCSBucketResponse,
+        entries: List[MetadataEntry],
+        parent: Optional[EntityReference] = None,
+    ) -> Iterable[GCSContainerDetails]:
+        bucket_name = bucket_response.name
+        for metadata_entry in entries:
+            if metadata_entry.structureFormat:
+                continue
+            if metadata_entry.unstructuredFormats:
+                yield from self._yield_nested_unstructured_containers(
+                    bucket_response=bucket_response,
+                    metadata_entry=metadata_entry,
+                    parent=parent,
+                )
+            else:
+                logger.debug(
+                    f"Extracting metadata from path {metadata_entry.dataPath.strip(KEY_SEPARATOR)} "
+                    f"and generating unstructured container"
+                )
+                prefix = (
+                    f"{KEY_SEPARATOR}{metadata_entry.dataPath.strip(KEY_SEPARATOR)}"
+                )
+                yield GCSContainerDetails(
+                    name=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                    prefix=prefix,
+                    file_formats=[],
+                    data_model=None,
+                    parent=parent,
+                    size=self.get_size(
+                        bucket_name=bucket_name,
+                        project_id=bucket_response.project_id,
+                        file_path=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                    ),
+                    fullPath=self._get_full_path(bucket_name, prefix),
+                    sourceUrl=self._get_object_source_url(
+                        bucket=bucket_response,
+                        prefix=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                        is_file=True,  # Individual unstructured files
+                    ),
+                )
+
     def _load_metadata_file(
         self, bucket: GCSBucketResponse
     ) -> Optional[StorageContainerConfig]:

metadata/ingestion/source/storage/gcs/models.py

@@ -53,21 +53,21 @@ class GCSContainerDetails(BaseModel):
     description: Optional[basic.Markdown] = Field(
         None, description="Description of the container instance."
     )
-    number_of_objects: float = Field(
-        ...,
+    number_of_objects: Optional[float] = Field(
+        None,
         description="Total nr. of objects",
     )
-    size: float = Field(
-        ...,
+    size: Optional[float] = Field(
+        None,
         description="Total size in bytes of all objects",
         title="Total size(bytes) of objects",
     )
     file_formats: Optional[List[FileFormat]] = Field(
-        ...,
+        None,
         description="File formats",
     )
     data_model: Optional[ContainerDataModel] = Field(
-        ...,
+        None,
         description="Data Model of the container",
     )
     creation_date: Optional[str] = Field(
@@ -84,3 +84,9 @@ class GCSContainerDetails(BaseModel):
     fullPath: Optional[str] = Field(
         None, description="Full path of the container/file."
     )
+    container_fqn: Optional[str] = Field(
+        None, description="Fully qualified name of the container."
+    )
+    leaf_container: Optional[bool] = Field(
+        None, description="Whether this is a leaf container."
+    )

metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py

@@ -85,10 +85,28 @@ class DatabricksProfilerInterface(SQAProfilerInterface):
             result += "`.`".join(splitted_result)
         return result
 
+    def visit_table(self, *args, **kwargs):
+        result = super(  # pylint: disable=bad-super-call
+            HiveCompiler, self
+        ).visit_table(*args, **kwargs)
+        # Handle table references with hyphens in database/schema names
+        # Format: `database`.`schema`.`table` for Unity Catalog/Databricks
+        if "." in result and not result.startswith("`"):
+            parts = result.split(".")
+            quoted_parts = []
+            for part in parts:
+                if "-" in part and not (part.startswith("`") and part.endswith("`")):
+                    quoted_parts.append(f"`{part}`")
+                else:
+                    quoted_parts.append(part)
+            result = ".".join(quoted_parts)
+        return result
+
     def __init__(self, service_connection_config, **kwargs):
         super().__init__(service_connection_config=service_connection_config, **kwargs)
         self.set_catalog(self.session)
         HiveCompiler.visit_column = DatabricksProfilerInterface.visit_column
+        HiveCompiler.visit_table = DatabricksProfilerInterface.visit_table
 
     def _get_struct_columns(self, columns: List[OMColumn], parent: str):
         """Get struct columns"""

metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py

@@ -14,6 +14,8 @@ Interfaces with database for all database engine
 supporting sqlalchemy abstraction layer
 """
 
+from sqlalchemy import event
+from sqlalchemy.orm import scoped_session, sessionmaker
 
 from metadata.ingestion.source.database.databricks.connection import (
     get_connection as databricks_get_connection,
@@ -26,5 +28,17 @@ from metadata.profiler.interface.sqlalchemy.databricks.profiler_interface import
 class UnityCatalogProfilerInterface(DatabricksProfilerInterface):
     def create_session(self):
         self.connection = databricks_get_connection(self.service_connection_config)
-        super().create_session()
-        self.set_catalog(self.session)
+
+        # Create custom session factory with after_begin event to set catalog
+        session_maker = sessionmaker(bind=self.connection)
+
+        @event.listens_for(session_maker, "after_begin")
+        def set_catalog(session, transaction, connection):
+            # Safely quote the catalog name to prevent SQL injection
+            quoted_catalog = connection.dialect.identifier_preparer.quote(
+                self.service_connection_config.catalog
+            )
+            connection.execute(f"USE CATALOG {quoted_catalog};")
+
+        self.session_factory = scoped_session(session_maker)
+        self.session = self.session_factory()

metadata/profiler/metrics/system/databricks/system.py

@@ -24,7 +24,7 @@ SYSTEM_QUERY = textwrap.dedent(
         '{database}' AS database,
         '{schema}' AS schema,
         '{table}' AS table
-    FROM (DESCRIBE HISTORY {database}.{schema}.{table})
+    FROM (DESCRIBE HISTORY `{database}`.`{schema}`.`{table}`)
     WHERE operation IN ({operations}) AND timestamp > DATEADD(day, -1, CURRENT_TIMESTAMP())
     """
 )

metadata/sampler/sqlalchemy/databricks/sampler.py

@@ -11,7 +11,8 @@
 """
 Helper module to handle data sampling for the profiler
 """
-from sqlalchemy import Column, text
+from sqlalchemy import Column, event, text
+from sqlalchemy.orm import scoped_session, sessionmaker
 
 from metadata.ingestion.source.database.databricks.connection import (
     get_connection as databricks_get_connection,
@@ -25,6 +26,17 @@ class DatabricksSamplerInterface(SQASampler):
         """Initialize with a single Databricks connection"""
         super().__init__(*args, **kwargs)
         self.connection = databricks_get_connection(self.service_connection_config)
+        session_maker = sessionmaker(bind=self.connection)
+
+        @event.listens_for(session_maker, "after_begin")
+        def set_catalog(session, transaction, connection):
+            # Safely quote the catalog name to prevent SQL injection
+            quoted_catalog = connection.dialect.identifier_preparer.quote(
+                self.service_connection_config.catalog
+            )
+            connection.execute(f"USE CATALOG {quoted_catalog};")
+
+        self.session_factory = scoped_session(session_maker)
 
     def get_client(self):
         """client is the session for SQA"""

metadata/sampler/sqlalchemy/unitycatalog/sampler.py

@@ -14,47 +14,14 @@ Interfaces with database for all database engine
 supporting sqlalchemy abstraction layer
 """
 
-from sqlalchemy import Column, text
 
-from metadata.ingestion.source.database.databricks.connection import (
-    get_connection as databricks_get_connection,
-)
-from metadata.profiler.orm.types.custom_array import CustomArray
-from metadata.sampler.sqlalchemy.sampler import SQASampler
+from metadata.sampler.sqlalchemy.databricks.sampler import DatabricksSamplerInterface
 
 
-class UnityCatalogSamplerInterface(SQASampler):
+class UnityCatalogSamplerInterface(DatabricksSamplerInterface):
+    """
+    Unity Catalog Sampler Interface
+    """
+
     def __init__(self, *args, **kwargs):
-        """Initialize with a single Databricks connection"""
         super().__init__(*args, **kwargs)
-        self.connection = databricks_get_connection(self.service_connection_config)
-
-    def get_client(self):
-        """client is the session for SQA"""
-        client = super().get_client()
-        self.set_catalog(client)
-        return client
-
-    def _handle_array_column(self, column: Column) -> bool:
-        """Check if a column is an array type"""
-        return isinstance(column.type, CustomArray)
-
-    def _get_slice_expression(self, column: Column):
-        """Generate SQL expression to slice array elements at query level
-
-        Args:
-            column_name: Name of the column
-            max_elements: Maximum number of elements to extract
-
-        Returns:
-            SQL expression string for array slicing
-        """
-        max_elements = self._get_max_array_elements()
-        return text(
-            f"""
-        CASE 
-            WHEN `{column.name}` IS NULL THEN NULL
-            ELSE slice(`{column.name}`, 1, {max_elements})
-        END AS `{column._label}`
-        """
-        )

metadata/utils/bigquery_utils.py

@@ -14,9 +14,7 @@ Utils module of BigQuery
 """
 
 from copy import deepcopy
-from typing import List, Optional
-
-from google.cloud import bigquery
+from typing import TYPE_CHECKING, List, Optional
 
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
@@ -34,6 +32,9 @@ from metadata.utils.credentials import (
     get_gcp_impersonate_credentials,
 )
 
+if TYPE_CHECKING:
+    from google.cloud import bigquery
+
 
 def get_bigquery_client(
     project_id: Optional[str] = None,
@@ -42,7 +43,7 @@ def get_bigquery_client(
     quota_project_id: Optional[str] = None,
     scopes: Optional[List[str]] = None,
     lifetime: Optional[int] = 3600,
-) -> bigquery.Client:
+) -> "bigquery.Client":
     """Get a BigQuery client
 
     Args:
@@ -65,6 +66,8 @@ def get_bigquery_client(
             scopes=scopes,
             lifetime=lifetime,
         )
+    from google.cloud import bigquery  # pylint: disable=import-outside-toplevel
+
     return bigquery.Client(
         credentials=credentials, project=project_id, location=location
     )