openmetadata-ingestion 1.9.5.0__py3-none-any.whl → 1.9.7.0__py3-none-any.whl

metadata/ingestion/source/storage/gcs/metadata.py

@@ -12,9 +12,10 @@
 import json
 import secrets
 import traceback
+from copy import deepcopy
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import Dict, Iterable, List, Optional
+from typing import Dict, Iterable, List, Optional, Tuple
 
 from google.cloud.exceptions import NotFound
 from google.cloud.monitoring_v3.types import TimeInterval
@@ -42,6 +43,7 @@ from metadata.generated.schema.metadataIngestion.storage.containerMetadataConfig
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
@@ -63,6 +65,8 @@ from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
+WILD_CARD = "*"
+
 
 class GCSMetric(Enum):
     NUMBER_OF_OBJECTS = "storage.googleapis.com/storage/object_count"
@@ -82,6 +86,7 @@ class GcsSource(StorageServiceSource):
             for project_id, client in self.gcs_clients.storage_client.clients.items()
         }
         self._bucket_cache: Dict[str, Container] = {}
+        self._unstructured_container_cache: Dict[str, Tuple[str, str]] = {}
 
     @classmethod
     def create(
@@ -115,6 +120,10 @@ class GcsSource(StorageServiceSource):
                     entity=Container, fqn=container_fqn
                 )
                 self._bucket_cache[bucket_name] = container_entity
+                self._unstructured_container_cache[container_fqn] = (
+                    container_entity.id.root,
+                    "",  # Empty string for bucket root (no relative path)
+                )
                 parent_entity: EntityReference = EntityReference(
                     id=self._bucket_cache[bucket_name].id.root, type="container"
                 )
@@ -132,6 +141,11 @@ class GcsSource(StorageServiceSource):
                             entries=manifest_entries_for_current_bucket,
                             parent=parent_entity,
                         )
+                        yield from self._generate_unstructured_containers(
+                            bucket_response=bucket_response,
+                            entries=manifest_entries_for_current_bucket,
+                            parent=parent_entity,
+                        )
                         # nothing else do to for the current bucket, skipping to the next
                         continue
                 # If no global file, or no valid entries in the manifest, check for bucket level metadata file
@@ -142,6 +156,14 @@ class GcsSource(StorageServiceSource):
                         entries=metadata_config.entries,
                         parent=parent_entity,
                     )
+                    yield from self._generate_unstructured_containers(
+                        bucket_response=bucket_response,
+                        entries=metadata_config.entries,
+                        parent=parent_entity,
+                    )
+
+                # clean up the cache after each bucket
+                self._unstructured_container_cache.clear()
 
             except ValidationError as err:
                 self.status.failed(
@@ -164,12 +186,12 @@ class GcsSource(StorageServiceSource):
         self, container_details: GCSContainerDetails
     ) -> Iterable[Either[CreateContainerRequest]]:
         container_request = CreateContainerRequest(
-            name=container_details.name,
+            name=EntityName(container_details.name),
             prefix=container_details.prefix,
             numberOfObjects=container_details.number_of_objects,
             size=container_details.size,
             dataModel=container_details.data_model,
-            service=self.context.get().objectstore_service,
+            service=FullyQualifiedEntityName(self.context.get().objectstore_service),
             parent=container_details.parent,
             sourceUrl=container_details.sourceUrl,
             fileFormats=container_details.file_formats,
@@ -178,6 +200,33 @@ class GcsSource(StorageServiceSource):
         yield Either(right=container_request)
         self.register_record(container_request=container_request)
 
+    def get_size(
+        self, bucket_name: str, project_id: str, file_path: str
+    ) -> Optional[float]:
+        """
+        Method to get the size of the file
+        """
+        try:
+            client = self.gcs_clients.storage_client.clients[project_id]
+            bucket = client.get_bucket(bucket_name)
+            blob = bucket.blob(file_path)
+            blob.reload()
+            return blob.size
+        except Exception as exc:
+            logger.debug(f"Failed to get size of file due to {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
+    def is_valid_unstructured_file(self, accepted_extensions: List, key: str) -> bool:
+        if WILD_CARD in accepted_extensions:
+            return True
+
+        for ext in accepted_extensions:
+            if key.endswith(ext):
+                return True
+
+        return False
+
     def _generate_container_details(
         self,
         bucket_response: GCSBucketResponse,
@@ -185,6 +234,10 @@ class GcsSource(StorageServiceSource):
         parent: Optional[EntityReference] = None,
     ) -> Optional[GCSContainerDetails]:
         bucket_name = bucket_response.name
+
+        if not metadata_entry.structureFormat:
+            return None
+
         sample_key = self._get_sample_file_path(
             bucket=bucket_response, metadata_entry=metadata_entry
         )
@@ -226,33 +279,80 @@ class GcsSource(StorageServiceSource):
                     sourceUrl=self._get_object_source_url(
                         bucket=bucket_response,
                         prefix=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                        is_file=False,  # Structured containers are directories
                     ),
                 )
         return None
 
+    def _generate_structured_containers_by_depth(
+        self,
+        bucket_response: GCSBucketResponse,
+        metadata_entry: MetadataEntry,
+        parent: Optional[EntityReference] = None,
+    ) -> Iterable[GCSContainerDetails]:
+        try:
+            prefix = self._get_sample_file_prefix(metadata_entry=metadata_entry)
+            if prefix:
+                client = self.gcs_clients.storage_client.clients[
+                    bucket_response.project_id
+                ]
+                response = client.list_blobs(
+                    bucket_response.name,
+                    prefix=prefix,
+                    max_results=1000,
+                )
+                # total depth is depth of prefix + depth of the metadata entry
+                total_depth = metadata_entry.depth + len(prefix[:-1].split("/"))
+                candidate_keys = {
+                    "/".join(entry.name.split("/")[:total_depth]) + "/"
+                    for entry in response
+                    if entry and entry.name and len(entry.name.split("/")) > total_depth
+                }
+                for key in candidate_keys:
+                    metadata_entry_copy = deepcopy(metadata_entry)
+                    metadata_entry_copy.dataPath = key.strip(KEY_SEPARATOR)
+                    structured_container: Optional[
+                        GCSContainerDetails
+                    ] = self._generate_container_details(
+                        bucket_response=bucket_response,
+                        metadata_entry=metadata_entry_copy,
+                        parent=parent,
+                    )
+                    if structured_container:
+                        yield structured_container
+        except Exception as err:
+            logger.warning(
+                f"Error while generating structured containers by depth for {metadata_entry.dataPath} - {err}"
+            )
+            logger.debug(traceback.format_exc())
+
     def _generate_structured_containers(
         self,
         bucket_response: GCSBucketResponse,
         entries: List[MetadataEntry],
         parent: Optional[EntityReference] = None,
-    ) -> List[GCSContainerDetails]:
-        result: List[GCSContainerDetails] = []
+    ) -> Iterable[GCSContainerDetails]:
         for metadata_entry in entries:
             logger.info(
                 f"Extracting metadata from path {metadata_entry.dataPath.strip(KEY_SEPARATOR)} "
                 f"and generating structured container"
             )
-            structured_container: Optional[
-                GCSContainerDetails
-            ] = self._generate_container_details(
-                bucket_response=bucket_response,
-                metadata_entry=metadata_entry,
-                parent=parent,
-            )
-            if structured_container:
-                result.append(structured_container)
-
-        return result
+            if metadata_entry.depth == 0:
+                structured_container: Optional[
+                    GCSContainerDetails
+                ] = self._generate_container_details(
+                    bucket_response=bucket_response,
+                    metadata_entry=metadata_entry,
+                    parent=parent,
+                )
+                if structured_container:
+                    yield structured_container
+            else:
+                yield from self._generate_structured_containers_by_depth(
+                    bucket_response=bucket_response,
+                    metadata_entry=metadata_entry,
+                    parent=parent,
+                )
 
     def _fetch_bucket(self, bucket_name: str) -> GCSBucketResponse:
         for project_id, client in self.gcs_clients.storage_client.clients.items():
@@ -409,31 +509,190 @@ class GcsSource(StorageServiceSource):
         Method to get the source url of GCS bucket
         """
         try:
-            return (
-                f"https://console.cloud.google.com/storage/browser/{bucket.name}"
-                f";tab=objects?project={bucket.project_id}"
-            )
+            return f"https://console.cloud.google.com/storage/browser/{bucket.name}?project={bucket.project_id}"
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Unable to get source url: {exc}")
         return None
 
     def _get_object_source_url(
-        self, bucket: GCSBucketResponse, prefix: str
+        self, bucket: GCSBucketResponse, prefix: str, is_file: bool = False
     ) -> Optional[str]:
         """
-        Method to get the source url of GCS object
+        Method to get the source url of GCS object or directory
         """
         try:
-            return (
-                f"https://console.cloud.google.com/storage/browser/_details/{bucket.name}/{prefix}"
-                f";tab=live_object?project={bucket.project_id}"
-            )
+            # Remove trailing slash from prefix if present
+            clean_prefix = prefix.rstrip("/")
+
+            if is_file:
+                # For files, use the _details path with tab=live_object
+                return f"https://console.cloud.google.com/storage/browser/_details/{bucket.name}/{clean_prefix};tab=live_object"
+            else:
+                # For directories/prefixes, use the browser view
+                return f"https://console.cloud.google.com/storage/browser/{bucket.name}/{clean_prefix}?project={bucket.project_id}"
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Unable to get source url: {exc}")
         return None
 
+    def _yield_parents_of_unstructured_container(
+        self,
+        bucket_name: str,
+        project_id: str,
+        list_of_parent: List[str],
+        parent: Optional[EntityReference] = None,
+    ):
+        relative_path = ""  # Path relative to bucket for URLs
+        sub_parent = parent
+        for i in range(len(list_of_parent) - 1):
+            container_fqn = fqn._build(  # pylint: disable=protected-access
+                *(
+                    self.context.get().objectstore_service,
+                    bucket_name,
+                    *list_of_parent[: i + 1],
+                )
+            )
+            if container_fqn in self._unstructured_container_cache:
+                parent_id, relative_path = self._unstructured_container_cache[
+                    container_fqn
+                ]
+                sub_parent = EntityReference(id=parent_id, type="container")
+                continue
+
+            # Build the relative path (no gs:// prefix)
+            current_relative_path = KEY_SEPARATOR.join(list_of_parent[: i + 1])
+            yield GCSContainerDetails(
+                name=list_of_parent[i],
+                prefix=KEY_SEPARATOR + current_relative_path,
+                file_formats=[],
+                parent=sub_parent,
+                fullPath=self._get_full_path(bucket_name, current_relative_path),
+                sourceUrl=self._get_object_source_url(
+                    bucket=GCSBucketResponse(
+                        name=bucket_name, project_id=project_id, creation_date=None
+                    ),
+                    prefix=current_relative_path,
+                    is_file=False,  # Parent containers are directories
+                ),
+            )
+            container_entity = self.metadata.get_by_name(
+                entity=Container, fqn=container_fqn
+            )
+            relative_path = current_relative_path
+            self._unstructured_container_cache[container_fqn] = (
+                container_entity.id.root,
+                relative_path,
+            )
+            sub_parent = EntityReference(id=container_entity.id.root, type="container")
+
+    def _yield_nested_unstructured_containers(
+        self,
+        bucket_response: GCSBucketResponse,
+        metadata_entry: MetadataEntry,
+        parent: Optional[EntityReference] = None,
+    ):
+        bucket_name = bucket_response.name
+        client = self.gcs_clients.storage_client.clients[bucket_response.project_id]
+        response = client.list_blobs(
+            bucket_name,
+            prefix=metadata_entry.dataPath,
+            max_results=1000,
+        )
+        candidate_keys = [
+            entry.name
+            for entry in response
+            if entry and entry.name and not entry.name.endswith("/")
+        ]
+        for key in candidate_keys:
+            if self.is_valid_unstructured_file(metadata_entry.unstructuredFormats, key):
+                logger.debug(
+                    f"Extracting metadata from path {key.strip(KEY_SEPARATOR)} "
+                    f"and generating unstructured container"
+                )
+                list_of_parent = key.strip(KEY_SEPARATOR).split(KEY_SEPARATOR)
+                yield from self._yield_parents_of_unstructured_container(
+                    bucket_name, bucket_response.project_id, list_of_parent, parent
+                )
+                parent_fqn = fqn._build(  # pylint: disable=protected-access
+                    *(
+                        self.context.get().objectstore_service,
+                        bucket_name,
+                        *list_of_parent[:-1],
+                    )
+                )
+                parent_id, parent_relative_path = self._unstructured_container_cache[
+                    parent_fqn
+                ]
+                container_fqn = fqn._build(  # pylint: disable=protected-access
+                    *(
+                        self.context.get().objectstore_service,
+                        bucket_name,
+                        *list_of_parent,
+                    )
+                )
+                size = self.get_size(bucket_name, bucket_response.project_id, key)
+                yield GCSContainerDetails(
+                    name=list_of_parent[-1],
+                    prefix=KEY_SEPARATOR + parent_relative_path
+                    if parent_relative_path
+                    else KEY_SEPARATOR,
+                    file_formats=[],
+                    size=size,
+                    container_fqn=container_fqn,
+                    leaf_container=True,
+                    parent=EntityReference(id=parent_id, type="container"),
+                    fullPath=self._get_full_path(bucket_name, key),
+                    sourceUrl=self._get_object_source_url(
+                        bucket=bucket_response,
+                        prefix=key,  # Use the full key path for the file
+                        is_file=True,  # Leaf containers are files
+                    ),
+                )
+
+    def _generate_unstructured_containers(
+        self,
+        bucket_response: GCSBucketResponse,
+        entries: List[MetadataEntry],
+        parent: Optional[EntityReference] = None,
+    ) -> Iterable[GCSContainerDetails]:
+        bucket_name = bucket_response.name
+        for metadata_entry in entries:
+            if metadata_entry.structureFormat:
+                continue
+            if metadata_entry.unstructuredFormats:
+                yield from self._yield_nested_unstructured_containers(
+                    bucket_response=bucket_response,
+                    metadata_entry=metadata_entry,
+                    parent=parent,
+                )
+            else:
+                logger.debug(
+                    f"Extracting metadata from path {metadata_entry.dataPath.strip(KEY_SEPARATOR)} "
+                    f"and generating unstructured container"
+                )
+                prefix = (
+                    f"{KEY_SEPARATOR}{metadata_entry.dataPath.strip(KEY_SEPARATOR)}"
+                )
+                yield GCSContainerDetails(
+                    name=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                    prefix=prefix,
+                    file_formats=[],
+                    data_model=None,
+                    parent=parent,
+                    size=self.get_size(
+                        bucket_name=bucket_name,
+                        project_id=bucket_response.project_id,
+                        file_path=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                    ),
+                    fullPath=self._get_full_path(bucket_name, prefix),
+                    sourceUrl=self._get_object_source_url(
+                        bucket=bucket_response,
+                        prefix=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                        is_file=True,  # Individual unstructured files
+                    ),
+                )
+
     def _load_metadata_file(
         self, bucket: GCSBucketResponse
     ) -> Optional[StorageContainerConfig]:

metadata/ingestion/source/storage/gcs/models.py

@@ -53,21 +53,21 @@ class GCSContainerDetails(BaseModel):
     description: Optional[basic.Markdown] = Field(
         None, description="Description of the container instance."
     )
-    number_of_objects: float = Field(
-        ...,
+    number_of_objects: Optional[float] = Field(
+        None,
         description="Total nr. of objects",
     )
-    size: float = Field(
-        ...,
+    size: Optional[float] = Field(
+        None,
         description="Total size in bytes of all objects",
         title="Total size(bytes) of objects",
     )
     file_formats: Optional[List[FileFormat]] = Field(
-        ...,
+        None,
         description="File formats",
     )
     data_model: Optional[ContainerDataModel] = Field(
-        ...,
+        None,
         description="Data Model of the container",
     )
     creation_date: Optional[str] = Field(
@@ -84,3 +84,9 @@ class GCSContainerDetails(BaseModel):
     fullPath: Optional[str] = Field(
         None, description="Full path of the container/file."
     )
+    container_fqn: Optional[str] = Field(
+        None, description="Fully qualified name of the container."
+    )
+    leaf_container: Optional[bool] = Field(
+        None, description="Whether this is a leaf container."
+    )

metadata/utils/db_utils.py

@@ -12,7 +12,7 @@
 """
 Helpers module for db sources
 """
-
+import time
 import traceback
 from typing import Iterable, List, Union
 
@@ -32,6 +32,7 @@ from metadata.ingestion.lineage.sql_lineage import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.models import TableView
 from metadata.utils import fqn
+from metadata.utils.execution_time_tracker import calculate_execution_time_generator
 from metadata.utils.logger import utils_logger
 
 logger = utils_logger()
@@ -47,6 +48,8 @@ def get_host_from_host_port(uri: str) -> str:
     return uri.split(":")[0]
 
 
+#  pylint: disable=too-many-locals
+@calculate_execution_time_generator()
 def get_view_lineage(
     view: TableView,
     metadata: OpenMetadata,
@@ -85,6 +88,8 @@ def get_view_lineage(
     try:
         connection_type = str(connection_type)
         dialect = ConnectionTypeDialectMapper.dialect_of(connection_type)
+        start_time = time.time()
+        logger.debug(f"Processing view lineage for: {table_fqn}")
         lineage_parser = LineageParser(
             view_definition, dialect, timeout_seconds=timeout_seconds
         )
@@ -94,6 +99,10 @@ def get_view_lineage(
             schema_name = PUBLIC_SCHEMA
             schema_fallback = True
 
+        end_time = time.time()
+        logger.debug(
+            f"Time taken to parse view lineage for: {table_fqn} is {end_time - start_time} seconds"
+        )
         if lineage_parser.source_tables and lineage_parser.target_tables:
             yield from get_lineage_by_query(
                 metadata,
@@ -104,6 +113,7 @@ def get_view_lineage(
                 dialect=dialect,
                 timeout_seconds=timeout_seconds,
                 lineage_source=LineageSource.ViewLineage,
+                lineage_parser=lineage_parser,
                 schema_fallback=schema_fallback,
             ) or []
 
@@ -118,6 +128,7 @@ def get_view_lineage(
                 dialect=dialect,
                 timeout_seconds=timeout_seconds,
                 lineage_source=LineageSource.ViewLineage,
+                lineage_parser=lineage_parser,
                 schema_fallback=schema_fallback,
             ) or []
     except Exception as exc:

metadata/utils/helpers.py

@@ -581,3 +581,14 @@ def pprint_format_object(data: Any) -> str:
     Pretty print an object in a format that is easy to read
     """
     return pprint.pformat(data, width=150)
+
+
+def can_spawn_child_process() -> bool:
+    """
+    Check if the current process can spawn a child process
+    """
+    # pylint: disable=import-outside-toplevel
+    from multiprocessing import Process
+
+    process = Process(target=lambda: None)
+    return not process.daemon

metadata/utils/ssl_manager.py

@@ -37,6 +37,9 @@ from metadata.generated.schema.entity.services.connections.database.dorisConnect
 from metadata.generated.schema.entity.services.connections.database.greenplumConnection import (
     GreenplumConnection,
 )
+from metadata.generated.schema.entity.services.connections.database.hiveConnection import (
+    HiveConnection,
+)
 from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import (
     MongoDBConnection,
 )
@@ -247,6 +250,25 @@ class SSLManager:
         connection.connectionArguments.root["ssl_context"] = ssl_context
         return connection
 
+    @setup_ssl.register(HiveConnection)
+    def _(self, connection):
+        connection = cast(HiveConnection, connection)
+
+        if not connection.connectionArguments:
+            connection.connectionArguments = init_empty_connection_arguments()
+
+        # Add certificate paths if available (following MySQL pattern)
+        ssl_args = connection.connectionArguments.root.get("ssl", {})
+        if self.ca_file_path:
+            ssl_args["ssl_ca"] = self.ca_file_path
+        if self.cert_file_path:
+            ssl_args["ssl_cert"] = self.cert_file_path
+        if self.key_file_path:
+            ssl_args["ssl_key"] = self.key_file_path
+        connection.connectionArguments.root["ssl"] = ssl_args
+
+        return connection
+
 
 @singledispatch
 def check_ssl_and_init(
@@ -375,6 +397,25 @@ def _(connection):
     return None
 
 
+@check_ssl_and_init.register(HiveConnection)
+def _(connection):
+    service_connection = cast(HiveConnection, connection)
+    if hasattr(service_connection, "useSSL") and service_connection.useSSL:
+        # Check if SSL config is provided in sslConfig (following MySQL pattern)
+        if hasattr(service_connection, "sslConfig") and service_connection.sslConfig:
+            if (
+                service_connection.sslConfig.root.caCertificate
+                or service_connection.sslConfig.root.sslCertificate
+                or service_connection.sslConfig.root.sslKey
+            ):
+                return SSLManager(
+                    ca=service_connection.sslConfig.root.caCertificate,
+                    cert=service_connection.sslConfig.root.sslCertificate,
+                    key=service_connection.sslConfig.root.sslKey,
+                )
+    return None
+
+
 def get_ssl_connection(service_config):
     try:
         # To be cleaned up as part of https://github.com/open-metadata/OpenMetadata/issues/15913