openmetadata-ingestion 1.2.5.1__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/ingestion/source/pipeline/airflow/metadata.py

@@ -12,6 +12,7 @@
 Airflow source to extract metadata from OM UI
 """
 import traceback
+from collections import Counter
 from datetime import datetime
 from enum import Enum
 from typing import Iterable, List, Optional, cast
@@ -34,13 +35,16 @@ from metadata.generated.schema.entity.data.pipeline import (
 from metadata.generated.schema.entity.services.connections.pipeline.airflowConnection import (
     AirflowConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
 from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.connections.session import create_and_bind_session
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
@@ -52,6 +56,7 @@ from metadata.ingestion.source.pipeline.airflow.lineage_parser import (
 from metadata.ingestion.source.pipeline.airflow.models import (
     AirflowDag,
     AirflowDagDetails,
+    AirflowTask,
 )
 from metadata.ingestion.source.pipeline.airflow.utils import get_schedule_interval
 from metadata.ingestion.source.pipeline.pipeline_service import PipelineServiceSource
@@ -68,12 +73,14 @@ class AirflowTaskStatus(Enum):
     FAILED = "failed"
     QUEUED = "queued"
     REMOVED = "removed"
+    SKIPPED = "skipped"
 
 
 STATUS_MAP = {
     AirflowTaskStatus.SUCCESS.value: StatusType.Successful.value,
     AirflowTaskStatus.FAILED.value: StatusType.Failed.value,
     AirflowTaskStatus.QUEUED.value: StatusType.Pending.value,
+    AirflowTaskStatus.SKIPPED.value: StatusType.Skipped.value,
 }
 
 
@@ -104,7 +111,7 @@ class AirflowSource(PipelineServiceSource):
         self._session = None
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(cls, config_dict, metadata: OpenMetadata) -> "AirflowSource":
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: AirflowConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, AirflowConnection):
@@ -158,7 +165,9 @@ class AirflowSource(PipelineServiceSource):
             for elem in dag_run_dict
         ]
 
-    def get_task_instances(self, dag_id: str, run_id: str) -> List[OMTaskInstance]:
+    def get_task_instances(
+        self, dag_id: str, run_id: str, serialized_tasks: List[AirflowTask]
+    ) -> List[OMTaskInstance]:
         """
         We are building our own scoped TaskInstance
         class to only focus on core properties required
@@ -168,6 +177,7 @@ class AirflowSource(PipelineServiceSource):
         sources we support.
         """
         task_instance_list = None
+        serialized_tasks_ids = {task.task_id for task in serialized_tasks}
 
         try:
             task_instance_list = (
@@ -182,7 +192,7 @@ class AirflowSource(PipelineServiceSource):
                     TaskInstance.dag_id == dag_id,
                     TaskInstance.run_id == run_id,
                     # updating old runs flag deleted tasks as `removed`
-                    TaskInstance.state != AirflowTaskStatus.REMOVED,
+                    TaskInstance.state != AirflowTaskStatus.REMOVED.value,
                 )
                 .all()
             )
@@ -204,6 +214,7 @@ class AirflowSource(PipelineServiceSource):
                 end_date=elem.get("end_date"),
             )
             for elem in task_instance_dict
+            if elem.get("task_id") in serialized_tasks_ids
         ]
 
     def yield_pipeline_status(
@@ -217,7 +228,9 @@ class AirflowSource(PipelineServiceSource):
                     dag_run.run_id and self.context.task_names
                 ):  # Airflow dags can have old task which are turned off/commented out in code
                     tasks = self.get_task_instances(
-                        dag_id=dag_run.dag_id, run_id=dag_run.run_id
+                        dag_id=dag_run.dag_id,
+                        run_id=dag_run.run_id,
+                        serialized_tasks=pipeline_details.tasks,
                     )
 
                     task_statuses = [
@@ -259,7 +272,7 @@ class AirflowSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=f"{pipeline_details.dag_id} Pipeline Status",
                     error=f"Wild error trying to extract status from DAG {pipeline_details.dag_id} - {exc}.",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -292,7 +305,7 @@ class AirflowSource(PipelineServiceSource):
                     start_date=data.get("start_date", None),
                     tasks=data.get("tasks", []),
                     schedule_interval=get_schedule_interval(data),
-                    owners=self.fetch_owners(data),
+                    owner=self.fetch_dag_owners(data),
                 )
 
                 yield dag
@@ -305,12 +318,29 @@ class AirflowSource(PipelineServiceSource):
                 logger.debug(traceback.format_exc())
                 logger.warning(f"Wild error yielding dag {serialized_dag} - {err}")
 
-    def fetch_owners(self, data) -> Optional[str]:
+    def fetch_dag_owners(self, data) -> Optional[str]:
+        """
+        In Airflow, ownership is defined as:
+        - `default_args`: Applied to all tasks and available on the DAG payload
+        - `owners`: Applied at the tasks. In Airflow's source code, DAG ownership is then a
+          list joined with the owners of all the tasks.
+
+        We will pick the owner from the tasks that appears in most tasks.
+        """
         try:
-            if self.source_config.includeOwners and data.get("default_args"):
-                return data.get("default_args", [])["__var"].get("email", [])
-        except TypeError:
-            pass
+            if self.source_config.includeOwners:
+                task_owners = [
+                    task.get("owner")
+                    for task in data.get("tasks", [])
+                    if task.get("owner") is not None
+                ]
+                if task_owners:
+                    most_common_owner, _ = Counter(task_owners).most_common(1)[0]
+                    return most_common_owner
+        except Exception as exc:
+            self.status.warning(
+                data.get("dag_id"), f"Could not extract owner information due to {exc}"
+            )
         return None
 
     def get_pipeline_name(self, pipeline_details: SerializedDAG) -> str:
@@ -319,8 +349,7 @@ class AirflowSource(PipelineServiceSource):
         """
         return pipeline_details.dag_id
 
-    @staticmethod
-    def get_tasks_from_dag(dag: AirflowDagDetails, host_port: str) -> List[Task]:
+    def get_tasks_from_dag(self, dag: AirflowDagDetails, host_port: str) -> List[Task]:
         """
         Obtain the tasks from a SerializedDAG
         :param dag: AirflowDagDetails
@@ -341,28 +370,26 @@ class AirflowSource(PipelineServiceSource):
                 startDate=task.start_date.isoformat() if task.start_date else None,
                 endDate=task.end_date.isoformat() if task.end_date else None,
                 taskType=task.task_type,
+                owner=self.get_owner(task.owner),
             )
             for task in cast(Iterable[BaseOperator], dag.tasks)
         ]
 
-    def get_user_details(self, email) -> Optional[EntityReference]:
-        user = self.metadata.get_user_by_email(email=email)
-        if user:
-            return EntityReference(id=user.id.__root__, type="user")
-        return None
-
-    def get_owner(self, owners) -> Optional[EntityReference]:
-        try:
-            if isinstance(owners, str) and owners:
-                return self.get_user_details(email=owners)
+    def get_owner(self, owner) -> Optional[EntityReference]:
+        """
+        Fetching users by name via ES to keep things as fast as possible.
 
-            if isinstance(owners, List) and owners:
-                for owner in owners or []:
-                    return self.get_user_details(email=owner)
+        We use the `owner` field since it's the onw used by Airflow to showcase
+        the info in its UI. In other connectors we might use the mail (e.g., in Looker),
+        but we use name here to be consistent with Airflow itself.
 
-            logger.debug(f"No user found with email [{owners}] in OMD")
+        If data is not indexed, we can live without this information
+        until the next run.
+        """
+        try:
+            return self.metadata.get_reference_by_name(name=owner)
         except Exception as exc:
-            logger.warning(f"Error while getting details of user {owners} - {exc}")
+            logger.warning(f"Error while getting details of user {owner} - {exc}")
         return None
 
     def yield_pipeline(
@@ -375,11 +402,13 @@ class AirflowSource(PipelineServiceSource):
         """
 
         try:
+            # Airflow uses /dags/dag_id/grid to show pipeline / dag
+            source_url = f"{clean_uri(self.service_connection.hostPort)}/dags/{pipeline_details.dag_id}/grid"
 
             pipeline_request = CreatePipelineRequest(
                 name=pipeline_details.dag_id,
                 description=pipeline_details.description,
-                sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/tree?dag_id={pipeline_details.dag_id}",
+                sourceUrl=source_url,
                 concurrency=pipeline_details.max_active_runs,
                 pipelineLocation=pipeline_details.fileloc,
                 startDate=pipeline_details.start_date.isoformat()
@@ -389,7 +418,7 @@ class AirflowSource(PipelineServiceSource):
                     pipeline_details, self.service_connection.hostPort
                 ),
                 service=self.context.pipeline_service,
-                owner=self.get_owner(pipeline_details.owners),
+                owner=self.get_owner(pipeline_details.owner),
                 scheduleInterval=pipeline_details.schedule_interval,
             )
             yield Either(right=pipeline_request)
@@ -406,7 +435,7 @@ class AirflowSource(PipelineServiceSource):
                         f"Error building DAG information from {pipeline_details}. There might be Airflow version"
                         f" incompatibilities - {err}"
                     ),
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except ValidationError as err:
@@ -415,7 +444,7 @@ class AirflowSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_details.dag_id,
                     error=f"Error building pydantic model for {pipeline_details} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -425,7 +454,7 @@ class AirflowSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_details.dag_id,
                     error=f"Wild error ingesting pipeline {pipeline_details} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 

metadata/ingestion/source/pipeline/airflow/models.py

@@ -31,7 +31,7 @@ class AirflowBaseModel(BaseModel):
     dag_id: str
 
 
-class Task(BaseModel):
+class AirflowTask(BaseModel):
     pool: Optional[str]
     doc_md: Optional[str]
     inlets: Optional[List[Any]] = Field(alias="_inlets")
@@ -41,6 +41,7 @@ class Task(BaseModel):
     downstream_task_ids: Optional[List[str]]
     start_date: Optional[datetime]
     end_date: Optional[datetime]
+    owner: Optional[str]
 
     # Allow picking up data from key `inlets` and `_inlets`
     class Config:
@@ -48,7 +49,7 @@ class Task(BaseModel):
 
 
 class TaskList(BaseModel):
-    __root__: List[Task]
+    __root__: List[AirflowTask]
 
 
 class Dag(BaseModel):
@@ -68,6 +69,6 @@ class AirflowDagDetails(AirflowBaseModel):
     max_active_runs: Optional[int]
     description: Optional[str]
     start_date: Optional[datetime]
-    tasks: List[Task]
-    owners: Optional[Any]
+    tasks: List[AirflowTask]
+    owner: Optional[str]
     schedule_interval: Optional[str]

metadata/ingestion/source/pipeline/dagster/metadata.py

@@ -26,10 +26,13 @@ from metadata.generated.schema.entity.data.pipeline import (
 from metadata.generated.schema.entity.services.connections.pipeline.dagsterConnection import (
     DagsterConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.step import WorkflowFatalError
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -141,7 +144,7 @@ class DagsterSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_details.name,
                     error=f"Error to yield pipeline for {pipeline_details}: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -197,7 +200,7 @@ class DagsterSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=run.runId,
                     error=f"Error to yield run status for {run}: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -227,7 +230,7 @@ class DagsterSource(PipelineServiceSource):
                     left=StackTraceError(
                         name=f"{pipeline_details.name} Pipeline Status",
                         error=f"Error to yield pipeline status for {pipeline_details}: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 

metadata/ingestion/source/pipeline/databrickspipeline/metadata.py

@@ -30,10 +30,13 @@ from metadata.generated.schema.entity.data.pipeline import (
 from metadata.generated.schema.entity.services.connections.pipeline.databricksPipelineConnection import (
     DatabricksPipelineConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -108,7 +111,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
                         f"Error building Databricks Pipeline information from {pipeline_details}."
                         f" There might be Databricks Jobs API version incompatibilities - {err}"
                     ),
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except ValidationError as err:
@@ -116,7 +119,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
                 left=StackTraceError(
                     name="Pipeline",
                     error=f"Error building pydantic model for {pipeline_details} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except Exception as err:
@@ -124,18 +127,18 @@ class DatabrickspipelineSource(PipelineServiceSource):
                 left=StackTraceError(
                     name="Pipeline",
                     error=f"Wild error ingesting pipeline {pipeline_details} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
     def get_tasks(self, pipeline_details: dict) -> List[Task]:
         task_list = []
-        self._append_context(key="job_id_list", value=pipeline_details["job_id"])
+        self.context.append(key="job_id_list", value=pipeline_details["job_id"])
 
         downstream_tasks = self.get_downstream_tasks(
             pipeline_details["settings"].get("tasks")
         )
-        for task in pipeline_details["settings"].get("tasks"):
+        for task in pipeline_details["settings"].get("tasks", []):
             task_list.append(
                 Task(
                     name=task["task_key"],
@@ -156,7 +159,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
         return task_key
 
     def get_downstream_tasks(self, workflow):
-        task_key_list = [task["task_key"] for task in workflow]
+        task_key_list = [task["task_key"] for task in workflow or []]
 
         dependent_tasks = self.get_dependent_tasks(workflow)
 
@@ -179,7 +182,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
     def get_dependent_tasks(self, workflow):
         dependent_tasks = {}
 
-        for task in workflow:
+        for task in workflow or []:
             depends_on = task.get("depends_on")
             if depends_on:
                 dependent_tasks[task["task_key"]] = [v["task_key"] for v in depends_on]
@@ -238,7 +241,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
                     left=StackTraceError(
                         name=pipeline_fqn,
                         error=f"Failed to yield pipeline status: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 

metadata/ingestion/source/pipeline/domopipeline/metadata.py

@@ -27,10 +27,13 @@ from metadata.generated.schema.entity.data.pipeline import (
 from metadata.generated.schema.entity.services.connections.pipeline.domoPipelineConnection import (
     DomoPipelineConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -103,7 +106,7 @@ class DomopipelineSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_details.get("name", "unknown"),
                     error=f"Error extracting data from {pipeline_details.get('name', 'unknown')} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except Exception as err:
@@ -111,7 +114,7 @@ class DomopipelineSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_details.get("name", "unknown"),
                     error=f"Wild error ingesting pipeline {pipeline_details.get('name', 'unknown')} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -175,7 +178,7 @@ class DomopipelineSource(PipelineServiceSource):
                 left=StackTraceError(
                     name=pipeline_fqn,
                     error=f"Error extracting status for {pipeline_id} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 

metadata/ingestion/source/pipeline/gluepipeline/metadata.py

@@ -28,10 +28,13 @@ from metadata.generated.schema.entity.data.pipeline import (
 from metadata.generated.schema.entity.services.connections.pipeline.gluePipelineConnection import (
     GluePipelineConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -183,7 +186,7 @@ class GluepipelineSource(PipelineServiceSource):
                     left=StackTraceError(
                         name=pipeline_fqn,
                         error=f"Failed to yield pipeline status: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 

metadata/ingestion/source/pipeline/pipeline_service.py

@@ -37,8 +37,8 @@ from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.models.topology import (
     NodeStage,
     ServiceTopology,
+    TopologyContext,
     TopologyNode,
-    create_source_context,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
@@ -116,7 +116,7 @@ class PipelineServiceSource(TopologyRunnerMixin, Source, ABC):
     service_connection: PipelineConnection.__fields__["config"].type_
 
     topology = PipelineServiceTopology()
-    context = create_source_context(topology)
+    context = TopologyContext.create(topology)
     pipeline_source_state: Set = set()
 
     def __init__(
@@ -138,6 +138,10 @@ class PipelineServiceSource(TopologyRunnerMixin, Source, ABC):
         self.client = self.connection
         self.test_connection()
 
+    @property
+    def name(self) -> str:
+        return self.service_connection.type.name
+
     @abstractmethod
     def yield_pipeline(
         self, pipeline_details: Any

metadata/ingestion/source/pipeline/spline/metadata.py

@@ -131,7 +131,6 @@ class SplineSource(PipelineServiceSource):
         return None
 
     def _get_table_from_datasource_name(self, datasource: str) -> Optional[Table]:
-
         if (
             not datasource
             and not datasource.startswith("dbfs")

metadata/ingestion/source/search/elasticsearch/connection.py

@@ -191,9 +191,12 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
+    def test_get_search_indexes():
+        client.indices.get_alias(expand_wildcards="open")
+
     test_fn = {
         "CheckAccess": client.info,
-        "GetSearchIndexes": client.indices.get_alias,
+        "GetSearchIndexes": test_get_search_indexes,
     }
 
     test_connection_steps(

metadata/ingestion/source/search/elasticsearch/metadata.py

@@ -69,7 +69,7 @@ class ElasticsearchSource(SearchServiceSource):
         """
         Get List of all search index
         """
-        index_list = self.client.indices.get_alias() or {}
+        index_list = self.client.indices.get_alias(expand_wildcards="open") or {}
         for index in index_list.keys():
             yield self.client.indices.get(index=str(index))
 
@@ -111,7 +111,6 @@ class ElasticsearchSource(SearchServiceSource):
         Method to Get Sample Data of Search Index Entity
         """
         if self.source_config.includeSampleData and self.context.search_index:
-
             sample_data = self.client.search(
                 index=self.context.search_index,
                 q=WILDCARD_SEARCH,

metadata/ingestion/source/search/search_service.py

@@ -43,8 +43,8 @@ from metadata.ingestion.models.search_index_data import OMetaIndexSampleData
 from metadata.ingestion.models.topology import (
     NodeStage,
     ServiceTopology,
+    TopologyContext,
     TopologyNode,
-    create_source_context,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
@@ -110,7 +110,7 @@ class SearchServiceSource(TopologyRunnerMixin, Source, ABC):
     service_connection: SearchConnection.__fields__["config"].type_
 
     topology = SearchServiceTopology()
-    context = create_source_context(topology)
+    context = TopologyContext.create(topology)
     index_source_state: Set = set()
 
     def __init__(
@@ -131,6 +131,10 @@ class SearchServiceSource(TopologyRunnerMixin, Source, ABC):
         self.connection_obj = self.connection
         self.test_connection()
 
+    @property
+    def name(self) -> str:
+        return self.service_connection.type.name
+
     @abstractmethod
     def yield_search_index(
         self, search_index_details: Any

metadata/ingestion/source/storage/s3/metadata.py

@@ -30,6 +30,9 @@ from metadata.generated.schema.entity.services.connections.database.datalake.s3C
 from metadata.generated.schema.entity.services.connections.storage.s3Connection import (
     S3Connection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.storage.containerMetadataConfig import (
     MetadataEntry,
     StorageContainerConfig,
@@ -38,7 +41,7 @@ from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.storage.s3.models import (
@@ -159,7 +162,7 @@ class S3Source(StorageServiceSource):
                     StackTraceError(
                         name=bucket_response.name,
                         error=f"Validation error while creating Container from bucket details - {err}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
             except Exception as err:
@@ -167,26 +170,26 @@ class S3Source(StorageServiceSource):
                     StackTraceError(
                         name=bucket_response.name,
                         error=f"Wild error while creating Container from bucket details - {err}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
     def yield_create_container_requests(
         self, container_details: S3ContainerDetails
     ) -> Iterable[Either[CreateContainerRequest]]:
-        yield Either(
-            right=CreateContainerRequest(
-                name=container_details.name,
-                prefix=container_details.prefix,
-                numberOfObjects=container_details.number_of_objects,
-                size=container_details.size,
-                dataModel=container_details.data_model,
-                service=self.context.objectstore_service,
-                parent=container_details.parent,
-                sourceUrl=container_details.sourceUrl,
-                fileFormats=container_details.file_formats,
-            )
+        container_request = CreateContainerRequest(
+            name=container_details.name,
+            prefix=container_details.prefix,
+            numberOfObjects=container_details.number_of_objects,
+            size=container_details.size,
+            dataModel=container_details.data_model,
+            service=self.context.objectstore_service,
+            parent=container_details.parent,
+            sourceUrl=container_details.sourceUrl,
+            fileFormats=container_details.file_formats,
         )
+        yield Either(right=container_request)
+        self.register_record(container_request=container_request)
 
     def _generate_container_details(
         self,
@@ -424,7 +427,7 @@ class S3Source(StorageServiceSource):
         """
         try:
             logger.info(
-                f"Found metadata template file at - s3://{bucket_name}/{OPENMETADATA_TEMPLATE_FILE_NAME}"
+                f"Looking for metadata template file at - s3://{bucket_name}/{OPENMETADATA_TEMPLATE_FILE_NAME}"
             )
             response_object = self.s3_reader.read(
                 path=OPENMETADATA_TEMPLATE_FILE_NAME,
@@ -435,7 +438,9 @@ class S3Source(StorageServiceSource):
             metadata_config = StorageContainerConfig.parse_obj(content)
             return metadata_config
         except ReadException:
-            pass
+            logger.warning(
+                f"No metadata file found at s3://{bucket_name}/{OPENMETADATA_TEMPLATE_FILE_NAME}"
+            )
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(