openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/source/storage/gcs/metadata.py

@@ -12,9 +12,10 @@
 import json
 import secrets
 import traceback
+from copy import deepcopy
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import Dict, Iterable, List, Optional
+from typing import Dict, Iterable, List, Optional, Tuple
 
 from google.cloud.exceptions import NotFound
 from google.cloud.monitoring_v3.types import TimeInterval
@@ -42,6 +43,7 @@ from metadata.generated.schema.metadataIngestion.storage.containerMetadataConfig
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
@@ -63,6 +65,8 @@ from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
+WILD_CARD = "*"
+
 
 class GCSMetric(Enum):
     NUMBER_OF_OBJECTS = "storage.googleapis.com/storage/object_count"
@@ -82,6 +86,7 @@ class GcsSource(StorageServiceSource):
             for project_id, client in self.gcs_clients.storage_client.clients.items()
         }
         self._bucket_cache: Dict[str, Container] = {}
+        self._unstructured_container_cache: Dict[str, Tuple[str, str]] = {}
 
     @classmethod
     def create(
@@ -115,6 +120,10 @@ class GcsSource(StorageServiceSource):
                     entity=Container, fqn=container_fqn
                 )
                 self._bucket_cache[bucket_name] = container_entity
+                self._unstructured_container_cache[container_fqn] = (
+                    container_entity.id.root,
+                    "",  # Empty string for bucket root (no relative path)
+                )
                 parent_entity: EntityReference = EntityReference(
                     id=self._bucket_cache[bucket_name].id.root, type="container"
                 )
@@ -132,6 +141,11 @@ class GcsSource(StorageServiceSource):
                             entries=manifest_entries_for_current_bucket,
                             parent=parent_entity,
                         )
+                        yield from self._generate_unstructured_containers(
+                            bucket_response=bucket_response,
+                            entries=manifest_entries_for_current_bucket,
+                            parent=parent_entity,
+                        )
                         # nothing else do to for the current bucket, skipping to the next
                         continue
                 # If no global file, or no valid entries in the manifest, check for bucket level metadata file
@@ -142,6 +156,14 @@ class GcsSource(StorageServiceSource):
                         entries=metadata_config.entries,
                         parent=parent_entity,
                     )
+                    yield from self._generate_unstructured_containers(
+                        bucket_response=bucket_response,
+                        entries=metadata_config.entries,
+                        parent=parent_entity,
+                    )
+
+                # clean up the cache after each bucket
+                self._unstructured_container_cache.clear()
 
             except ValidationError as err:
                 self.status.failed(
@@ -164,12 +186,12 @@ class GcsSource(StorageServiceSource):
         self, container_details: GCSContainerDetails
     ) -> Iterable[Either[CreateContainerRequest]]:
         container_request = CreateContainerRequest(
-            name=container_details.name,
+            name=EntityName(container_details.name),
             prefix=container_details.prefix,
             numberOfObjects=container_details.number_of_objects,
             size=container_details.size,
             dataModel=container_details.data_model,
-            service=self.context.get().objectstore_service,
+            service=FullyQualifiedEntityName(self.context.get().objectstore_service),
             parent=container_details.parent,
             sourceUrl=container_details.sourceUrl,
             fileFormats=container_details.file_formats,
@@ -178,6 +200,33 @@ class GcsSource(StorageServiceSource):
         yield Either(right=container_request)
         self.register_record(container_request=container_request)
 
+    def get_size(
+        self, bucket_name: str, project_id: str, file_path: str
+    ) -> Optional[float]:
+        """
+        Method to get the size of the file
+        """
+        try:
+            client = self.gcs_clients.storage_client.clients[project_id]
+            bucket = client.get_bucket(bucket_name)
+            blob = bucket.blob(file_path)
+            blob.reload()
+            return blob.size
+        except Exception as exc:
+            logger.debug(f"Failed to get size of file due to {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
+    def is_valid_unstructured_file(self, accepted_extensions: List, key: str) -> bool:
+        if WILD_CARD in accepted_extensions:
+            return True
+
+        for ext in accepted_extensions:
+            if key.endswith(ext):
+                return True
+
+        return False
+
     def _generate_container_details(
         self,
         bucket_response: GCSBucketResponse,
@@ -185,6 +234,10 @@ class GcsSource(StorageServiceSource):
         parent: Optional[EntityReference] = None,
     ) -> Optional[GCSContainerDetails]:
         bucket_name = bucket_response.name
+
+        if not metadata_entry.structureFormat:
+            return None
+
         sample_key = self._get_sample_file_path(
             bucket=bucket_response, metadata_entry=metadata_entry
         )
@@ -226,33 +279,80 @@ class GcsSource(StorageServiceSource):
                     sourceUrl=self._get_object_source_url(
                         bucket=bucket_response,
                         prefix=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                        is_file=False,  # Structured containers are directories
                     ),
                 )
         return None
 
+    def _generate_structured_containers_by_depth(
+        self,
+        bucket_response: GCSBucketResponse,
+        metadata_entry: MetadataEntry,
+        parent: Optional[EntityReference] = None,
+    ) -> Iterable[GCSContainerDetails]:
+        try:
+            prefix = self._get_sample_file_prefix(metadata_entry=metadata_entry)
+            if prefix:
+                client = self.gcs_clients.storage_client.clients[
+                    bucket_response.project_id
+                ]
+                response = client.list_blobs(
+                    bucket_response.name,
+                    prefix=prefix,
+                    max_results=1000,
+                )
+                # total depth is depth of prefix + depth of the metadata entry
+                total_depth = metadata_entry.depth + len(prefix[:-1].split("/"))
+                candidate_keys = {
+                    "/".join(entry.name.split("/")[:total_depth]) + "/"
+                    for entry in response
+                    if entry and entry.name and len(entry.name.split("/")) > total_depth
+                }
+                for key in candidate_keys:
+                    metadata_entry_copy = deepcopy(metadata_entry)
+                    metadata_entry_copy.dataPath = key.strip(KEY_SEPARATOR)
+                    structured_container: Optional[
+                        GCSContainerDetails
+                    ] = self._generate_container_details(
+                        bucket_response=bucket_response,
+                        metadata_entry=metadata_entry_copy,
+                        parent=parent,
+                    )
+                    if structured_container:
+                        yield structured_container
+        except Exception as err:
+            logger.warning(
+                f"Error while generating structured containers by depth for {metadata_entry.dataPath} - {err}"
+            )
+            logger.debug(traceback.format_exc())
+
     def _generate_structured_containers(
         self,
         bucket_response: GCSBucketResponse,
         entries: List[MetadataEntry],
         parent: Optional[EntityReference] = None,
-    ) -> List[GCSContainerDetails]:
-        result: List[GCSContainerDetails] = []
+    ) -> Iterable[GCSContainerDetails]:
         for metadata_entry in entries:
             logger.info(
                 f"Extracting metadata from path {metadata_entry.dataPath.strip(KEY_SEPARATOR)} "
                 f"and generating structured container"
             )
-            structured_container: Optional[
-                GCSContainerDetails
-            ] = self._generate_container_details(
-                bucket_response=bucket_response,
-                metadata_entry=metadata_entry,
-                parent=parent,
-            )
-            if structured_container:
-                result.append(structured_container)
-
-        return result
+            if metadata_entry.depth == 0:
+                structured_container: Optional[
+                    GCSContainerDetails
+                ] = self._generate_container_details(
+                    bucket_response=bucket_response,
+                    metadata_entry=metadata_entry,
+                    parent=parent,
+                )
+                if structured_container:
+                    yield structured_container
+            else:
+                yield from self._generate_structured_containers_by_depth(
+                    bucket_response=bucket_response,
+                    metadata_entry=metadata_entry,
+                    parent=parent,
+                )
 
     def _fetch_bucket(self, bucket_name: str) -> GCSBucketResponse:
         for project_id, client in self.gcs_clients.storage_client.clients.items():
@@ -409,31 +509,190 @@ class GcsSource(StorageServiceSource):
         Method to get the source url of GCS bucket
         """
         try:
-            return (
-                f"https://console.cloud.google.com/storage/browser/{bucket.name}"
-                f";tab=objects?project={bucket.project_id}"
-            )
+            return f"https://console.cloud.google.com/storage/browser/{bucket.name}?project={bucket.project_id}"
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Unable to get source url: {exc}")
         return None
 
     def _get_object_source_url(
-        self, bucket: GCSBucketResponse, prefix: str
+        self, bucket: GCSBucketResponse, prefix: str, is_file: bool = False
     ) -> Optional[str]:
         """
-        Method to get the source url of GCS object
+        Method to get the source url of GCS object or directory
         """
         try:
-            return (
-                f"https://console.cloud.google.com/storage/browser/_details/{bucket.name}/{prefix}"
-                f";tab=live_object?project={bucket.project_id}"
-            )
+            # Remove trailing slash from prefix if present
+            clean_prefix = prefix.rstrip("/")
+
+            if is_file:
+                # For files, use the _details path with tab=live_object
+                return f"https://console.cloud.google.com/storage/browser/_details/{bucket.name}/{clean_prefix};tab=live_object"
+            else:
+                # For directories/prefixes, use the browser view
+                return f"https://console.cloud.google.com/storage/browser/{bucket.name}/{clean_prefix}?project={bucket.project_id}"
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Unable to get source url: {exc}")
         return None
 
+    def _yield_parents_of_unstructured_container(
+        self,
+        bucket_name: str,
+        project_id: str,
+        list_of_parent: List[str],
+        parent: Optional[EntityReference] = None,
+    ):
+        relative_path = ""  # Path relative to bucket for URLs
+        sub_parent = parent
+        for i in range(len(list_of_parent) - 1):
+            container_fqn = fqn._build(  # pylint: disable=protected-access
+                *(
+                    self.context.get().objectstore_service,
+                    bucket_name,
+                    *list_of_parent[: i + 1],
+                )
+            )
+            if container_fqn in self._unstructured_container_cache:
+                parent_id, relative_path = self._unstructured_container_cache[
+                    container_fqn
+                ]
+                sub_parent = EntityReference(id=parent_id, type="container")
+                continue
+
+            # Build the relative path (no gs:// prefix)
+            current_relative_path = KEY_SEPARATOR.join(list_of_parent[: i + 1])
+            yield GCSContainerDetails(
+                name=list_of_parent[i],
+                prefix=KEY_SEPARATOR + current_relative_path,
+                file_formats=[],
+                parent=sub_parent,
+                fullPath=self._get_full_path(bucket_name, current_relative_path),
+                sourceUrl=self._get_object_source_url(
+                    bucket=GCSBucketResponse(
+                        name=bucket_name, project_id=project_id, creation_date=None
+                    ),
+                    prefix=current_relative_path,
+                    is_file=False,  # Parent containers are directories
+                ),
+            )
+            container_entity = self.metadata.get_by_name(
+                entity=Container, fqn=container_fqn
+            )
+            relative_path = current_relative_path
+            self._unstructured_container_cache[container_fqn] = (
+                container_entity.id.root,
+                relative_path,
+            )
+            sub_parent = EntityReference(id=container_entity.id.root, type="container")
+
+    def _yield_nested_unstructured_containers(
+        self,
+        bucket_response: GCSBucketResponse,
+        metadata_entry: MetadataEntry,
+        parent: Optional[EntityReference] = None,
+    ):
+        bucket_name = bucket_response.name
+        client = self.gcs_clients.storage_client.clients[bucket_response.project_id]
+        response = client.list_blobs(
+            bucket_name,
+            prefix=metadata_entry.dataPath,
+            max_results=1000,
+        )
+        candidate_keys = [
+            entry.name
+            for entry in response
+            if entry and entry.name and not entry.name.endswith("/")
+        ]
+        for key in candidate_keys:
+            if self.is_valid_unstructured_file(metadata_entry.unstructuredFormats, key):
+                logger.debug(
+                    f"Extracting metadata from path {key.strip(KEY_SEPARATOR)} "
+                    f"and generating unstructured container"
+                )
+                list_of_parent = key.strip(KEY_SEPARATOR).split(KEY_SEPARATOR)
+                yield from self._yield_parents_of_unstructured_container(
+                    bucket_name, bucket_response.project_id, list_of_parent, parent
+                )
+                parent_fqn = fqn._build(  # pylint: disable=protected-access
+                    *(
+                        self.context.get().objectstore_service,
+                        bucket_name,
+                        *list_of_parent[:-1],
+                    )
+                )
+                parent_id, parent_relative_path = self._unstructured_container_cache[
+                    parent_fqn
+                ]
+                container_fqn = fqn._build(  # pylint: disable=protected-access
+                    *(
+                        self.context.get().objectstore_service,
+                        bucket_name,
+                        *list_of_parent,
+                    )
+                )
+                size = self.get_size(bucket_name, bucket_response.project_id, key)
+                yield GCSContainerDetails(
+                    name=list_of_parent[-1],
+                    prefix=KEY_SEPARATOR + parent_relative_path
+                    if parent_relative_path
+                    else KEY_SEPARATOR,
+                    file_formats=[],
+                    size=size,
+                    container_fqn=container_fqn,
+                    leaf_container=True,
+                    parent=EntityReference(id=parent_id, type="container"),
+                    fullPath=self._get_full_path(bucket_name, key),
+                    sourceUrl=self._get_object_source_url(
+                        bucket=bucket_response,
+                        prefix=key,  # Use the full key path for the file
+                        is_file=True,  # Leaf containers are files
+                    ),
+                )
+
+    def _generate_unstructured_containers(
+        self,
+        bucket_response: GCSBucketResponse,
+        entries: List[MetadataEntry],
+        parent: Optional[EntityReference] = None,
+    ) -> Iterable[GCSContainerDetails]:
+        bucket_name = bucket_response.name
+        for metadata_entry in entries:
+            if metadata_entry.structureFormat:
+                continue
+            if metadata_entry.unstructuredFormats:
+                yield from self._yield_nested_unstructured_containers(
+                    bucket_response=bucket_response,
+                    metadata_entry=metadata_entry,
+                    parent=parent,
+                )
+            else:
+                logger.debug(
+                    f"Extracting metadata from path {metadata_entry.dataPath.strip(KEY_SEPARATOR)} "
+                    f"and generating unstructured container"
+                )
+                prefix = (
+                    f"{KEY_SEPARATOR}{metadata_entry.dataPath.strip(KEY_SEPARATOR)}"
+                )
+                yield GCSContainerDetails(
+                    name=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                    prefix=prefix,
+                    file_formats=[],
+                    data_model=None,
+                    parent=parent,
+                    size=self.get_size(
+                        bucket_name=bucket_name,
+                        project_id=bucket_response.project_id,
+                        file_path=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                    ),
+                    fullPath=self._get_full_path(bucket_name, prefix),
+                    sourceUrl=self._get_object_source_url(
+                        bucket=bucket_response,
+                        prefix=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                        is_file=True,  # Individual unstructured files
+                    ),
+                )
+
     def _load_metadata_file(
         self, bucket: GCSBucketResponse
     ) -> Optional[StorageContainerConfig]:

metadata/ingestion/source/storage/gcs/models.py

@@ -53,21 +53,21 @@ class GCSContainerDetails(BaseModel):
     description: Optional[basic.Markdown] = Field(
         None, description="Description of the container instance."
     )
-    number_of_objects: float = Field(
-        ...,
+    number_of_objects: Optional[float] = Field(
+        None,
         description="Total nr. of objects",
     )
-    size: float = Field(
-        ...,
+    size: Optional[float] = Field(
+        None,
         description="Total size in bytes of all objects",
         title="Total size(bytes) of objects",
     )
     file_formats: Optional[List[FileFormat]] = Field(
-        ...,
+        None,
         description="File formats",
     )
     data_model: Optional[ContainerDataModel] = Field(
-        ...,
+        None,
         description="Data Model of the container",
     )
     creation_date: Optional[str] = Field(
@@ -84,3 +84,9 @@ class GCSContainerDetails(BaseModel):
     fullPath: Optional[str] = Field(
         None, description="Full path of the container/file."
     )
+    container_fqn: Optional[str] = Field(
+        None, description="Fully qualified name of the container."
+    )
+    leaf_container: Optional[bool] = Field(
+        None, description="Whether this is a leaf container."
+    )

metadata/ingestion/source/storage/s3/metadata.py

@@ -356,6 +356,7 @@ class S3Source(StorageServiceSource):
                     if entry
                     and entry.get("Key")
                     and len(entry.get("Key").split("/")) > total_depth
+                    and "/_delta_log/" not in entry.get("Key")
                 }
                 for key in candidate_keys:
                     metadata_entry_copy = deepcopy(metadata_entry)
@@ -469,7 +470,10 @@ class S3Source(StorageServiceSource):
         candidate_keys = [
             entry["Key"]
             for entry in response
-            if entry and entry.get("Key") and not entry.get("Key").endswith("/")
+            if entry
+            and entry.get("Key")
+            and not entry.get("Key").endswith("/")
+            and "/_delta_log/" not in entry.get("Key")
         ]
         for key in candidate_keys:
             if self.is_valid_unstructured_file(metadata_entry.unstructuredFormats, key):
@@ -678,7 +682,10 @@ class S3Source(StorageServiceSource):
                 candidate_keys = [
                     entry["Key"]
                     for entry in response[S3_CLIENT_ROOT_RESPONSE]
-                    if entry and entry.get("Key") and not entry.get("Key").endswith("/")
+                    if entry
+                    and entry.get("Key")
+                    and not entry.get("Key").endswith("/")
+                    and "/_delta_log/" not in entry.get("Key")
                 ]
                 # pick a random key out of the candidates if any were returned
                 if candidate_keys:
@@ -716,11 +723,25 @@ class S3Source(StorageServiceSource):
         Method to get the source url of s3 bucket
         """
         try:
-            region = self.get_aws_bucket_region(bucket_name=bucket_name)
-            return (
-                f"https://s3.console.aws.amazon.com/s3/buckets/{bucket_name}"
-                f"?region={region}&tab=objects"
+            # Check if custom console endpoint URL is configured (for external S3-compatible storage)
+            console_endpoint_url = getattr(
+                self.service_connection, "consoleEndpointURL", None
             )
+
+            # If no custom console endpoint, use AWS S3 console with region
+            if not console_endpoint_url:
+                region = self.get_aws_bucket_region(bucket_name=bucket_name)
+                return (
+                    f"https://s3.console.aws.amazon.com/s3/buckets/{bucket_name}"
+                    f"?region={region}&tab=objects"
+                )
+
+            # For external S3-compatible storage, user provides the full base path
+            # (e.g., http://localhost:9001/browser/ for MinIO)
+            # We just append the bucket name
+            base_url = str(console_endpoint_url).rstrip("/")
+            return f"{base_url}/{bucket_name}/"
+
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Unable to get source url: {exc}")
@@ -731,12 +752,26 @@ class S3Source(StorageServiceSource):
         Method to get the source url of s3 bucket
         """
         try:
-            region = self.get_aws_bucket_region(bucket_name=bucket_name)
-            return (
-                f"https://s3.console.aws.amazon.com/s3/buckets/{bucket_name}"
-                f"?region={region}&prefix={prefix}/"
-                f"&showversions=false"
+            # Check if custom console endpoint URL is configured (for external S3-compatible storage)
+            console_endpoint_url = getattr(
+                self.service_connection, "consoleEndpointURL", None
             )
+
+            # If no custom console endpoint, use AWS S3 console with region
+            if not console_endpoint_url:
+                region = self.get_aws_bucket_region(bucket_name=bucket_name)
+                return (
+                    f"https://s3.console.aws.amazon.com/s3/buckets/{bucket_name}"
+                    f"?region={region}&prefix={prefix}/"
+                    f"&showversions=false"
+                )
+
+            # For external S3-compatible storage, user provides the full base path
+            # (e.g., http://localhost:9001/browser/ for MinIO)
+            # We just append the bucket name and prefix
+            base_url = str(console_endpoint_url).rstrip("/")
+            return f"{base_url}/{bucket_name}/{prefix}/"
+
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Unable to get source url: {exc}")

metadata/ingestion/source/storage/storage_service.py

@@ -274,6 +274,8 @@ class StorageServiceSource(TopologyRunnerMixin, Source, ABC):
                 isPartitioned=entry.isPartitioned,
                 partitionColumns=entry.partitionColumns,
                 separator=entry.separator,
+                depth=entry.depth,
+                unstructuredFormats=entry.unstructuredFormats,
             )
             for entry in manifest.entries
             if entry.containerName == container_name

metadata/parsers/json_schema_parser.py

@@ -68,9 +68,49 @@ def parse_json_schema(
 
 def get_child_models(key, value, field_models, cls: Type[BaseModel] = FieldModel):
     """
-    Method to parse the child objects in the json schema
+    Method to parse the child objects in the json schema.
+    Handles oneOf union types (e.g., Debezium CDC nullable fields).
     """
     try:
+        # Handle oneOf union types (e.g., [{"type": "null"}, {"type": "object", "properties": {...}}])
+        # Common in Debezium CDC schemas for nullable fields like "after" and "before"
+        if "oneOf" in value and isinstance(value["oneOf"], list):
+            # Find the non-null object schema in the union
+            object_schema = None
+            for option in value["oneOf"]:
+                if (
+                    isinstance(option, dict)
+                    and option.get("type") == JsonSchemaDataTypes.RECORD.value
+                ):
+                    object_schema = option
+                    break
+
+            if object_schema:
+                # Use the object schema's properties and metadata
+                cls_obj = cls(
+                    name=key,
+                    displayName=value.get("title") or object_schema.get("title"),
+                    dataType=JsonSchemaDataTypes.RECORD.name,
+                    description=value.get("description")
+                    or object_schema.get("description"),
+                )
+                children = get_json_schema_fields(
+                    object_schema.get("properties", {}), cls=cls
+                )
+                cls_obj.children = children
+                field_models.append(cls_obj)
+                return
+            # If no object found in oneOf, treat as UNKNOWN
+            cls_obj = cls(
+                name=key,
+                displayName=value.get("title"),
+                dataType=JsonSchemaDataTypes.UNKNOWN.name,
+                description=value.get("description"),
+            )
+            field_models.append(cls_obj)
+            return
+
+        # Standard type handling
         cls_obj = cls(
             name=key,
             displayName=value.get("title"),