openmetadata-ingestion 1.4.6.0rc1__py3-none-any.whl → 1.5.0.0rc1__py3-none-any.whl

metadata/utils/secrets/gcp_secrets_manager.py

@@ -0,0 +1,128 @@
+"""
+Secrets manager implementation using GCP Secret Manager
+"""
+
+import os
+from abc import ABC
+from typing import Optional
+
+import google_crc32c
+from google import auth
+from google.cloud import secretmanager
+
+from metadata.generated.schema.security.secrets.secretsManagerClientLoader import (
+    SecretsManagerClientLoader,
+)
+from metadata.generated.schema.security.secrets.secretsManagerProvider import (
+    SecretsManagerProvider,
+)
+from metadata.utils.dispatch import enum_register
+from metadata.utils.secrets.external_secrets_manager import (
+    SECRET_MANAGER_AIRFLOW_CONF,
+    ExternalSecretsManager,
+    SecretsManagerConfigException,
+)
+
+FIXED_VERSION_ID = "latest"
+
+secrets_manager_client_loader = enum_register()
+
+
+# pylint: disable=import-outside-toplevel
+@secrets_manager_client_loader.add(SecretsManagerClientLoader.noop.value)
+def _() -> Optional["GCPCredentials"]:
+    from metadata.generated.schema.security.credentials.gcpCredentials import (
+        GCPCredentials,
+        gcpValues,
+    )
+
+    _, project_id = auth.default()
+    if project_id:
+        config = gcpValues.GcpCredentialsValues(projectId=project_id)
+        credentials = GCPCredentials(gcpConfig=config)
+        return credentials
+
+    return None
+
+
+@secrets_manager_client_loader.add(SecretsManagerClientLoader.airflow.value)
+def _() -> Optional["GCPCredentials"]:
+    from airflow.configuration import conf
+
+    from metadata.generated.schema.security.credentials.gcpCredentials import (
+        GCPCredentials,
+        gcpValues,
+    )
+
+    project_id = conf.get(SECRET_MANAGER_AIRFLOW_CONF, "project_id", fallback=None)
+    if project_id:
+        config = gcpValues.GcpCredentialsValues(projectId=project_id)
+        credentials = GCPCredentials(gcpConfig=config)
+        return credentials
+
+    return None
+
+
+@secrets_manager_client_loader.add(SecretsManagerClientLoader.env.value)
+def _() -> Optional["GCPCredentials"]:
+    from metadata.generated.schema.security.credentials.gcpCredentials import (
+        GCPCredentials,
+        gcpValues,
+    )
+
+    # https://google-auth.readthedocs.io/en/master/reference/google.auth.html#google.auth.default
+    project_id = os.getenv("GOOGLE_CLOUD_PROJECT")
+    if project_id:
+        config = gcpValues.GcpCredentialsValues(projectId=project_id)
+        credentials = GCPCredentials(gcpConfig=config)
+        return credentials
+
+    return None
+
+
+class GCPSecretsManager(ExternalSecretsManager, ABC):
+    """
+    Secrets Manager Implementation Class
+    """
+
+    def __init__(self, loader: SecretsManagerClientLoader):
+        super().__init__(SecretsManagerProvider.gcp, loader=loader)
+
+    def get_string_value(self, secret_id: str) -> str:
+        """
+        Access the payload for the given secret version if one exists. The version
+        can be a version number as a string (e.g. "5") or an alias (e.g. "latest").
+        """
+
+        # Create the Secret Manager client.
+        client = secretmanager.SecretManagerServiceClient()
+
+        # Build the resource name of the secret version.
+
+        project_id = self.credentials.gcpConfig.projectId.root
+        secret_id = (
+            f"projects/{project_id}/secrets/{secret_id}/versions/{FIXED_VERSION_ID}"
+        )
+
+        # Access the secret version.
+        response = client.access_secret_version(request={"name": secret_id})
+
+        # Verify payload checksum.
+        crc32c = google_crc32c.Checksum()
+        crc32c.update(response.payload.data)
+        if response.payload.data_crc32c != int(crc32c.hexdigest(), 16):
+            raise ValueError("Data corruption detected.")
+
+        # Print the secret payload.
+        #
+        # WARNING: Do not print the secret in a production environment - this
+        # snippet is showing how to access the secret material.
+        return response.payload.data.decode("UTF-8")
+
+    def load_credentials(self) -> Optional["GCPCredentials"]:
+        """Load the provider credentials based on the loader type"""
+        try:
+            loader_fn = secrets_manager_client_loader.registry.get(self.loader.value)
+            return loader_fn()
+        except Exception as err:
+            raise SecretsManagerConfigException(f"Error loading credentials - [{err}]")

metadata/utils/secrets/secrets_manager_factory.py

@@ -24,6 +24,7 @@ from metadata.utils.secrets.aws_secrets_manager import AWSSecretsManager
 from metadata.utils.secrets.aws_ssm_secrets_manager import AWSSSMSecretsManager
 from metadata.utils.secrets.azure_kv_secrets_manager import AzureKVSecretsManager
 from metadata.utils.secrets.db_secrets_manager import DBSecretsManager
+from metadata.utils.secrets.gcp_secrets_manager import GCPSecretsManager
 from metadata.utils.secrets.secrets_manager import SecretsManager
 from metadata.utils.singleton import Singleton
 
@@ -93,6 +94,8 @@ class SecretsManagerFactory(metaclass=Singleton):
             SecretsManagerProvider.managed_azure_kv,
         ):
             return AzureKVSecretsManager(secrets_manager_loader)
+        if secrets_manager_provider in (SecretsManagerProvider.gcp,):
+            return GCPSecretsManager(secrets_manager_loader)
         raise NotImplementedError(f"[{secrets_manager_provider}] is not implemented.")
 
     def get_secrets_manager(self):

metadata/utils/source_hash.py

@@ -42,7 +42,7 @@ def generate_source_hash(
             else SOURCE_HASH_EXCLUDE_FIELDS
         )
 
-        create_request_json = create_request.json(exclude=exclude_fields)
+        create_request_json = create_request.model_dump_json(exclude=exclude_fields)
 
         json_bytes = create_request_json.encode("utf-8")
         return hashlib.md5(json_bytes).hexdigest()

metadata/utils/ssl_manager.py

@@ -96,14 +96,14 @@ class SSLManager:
         connection.connectionArguments = (
             connection.connectionArguments or init_empty_connection_arguments()
         )
-        ssl_args = connection.connectionArguments.__root__.get("ssl", {})
-        if connection.sslConfig.__root__.caCertificate:
+        ssl_args = connection.connectionArguments.root.get("ssl", {})
+        if connection.sslConfig.root.caCertificate:
             ssl_args["ssl_ca"] = self.ca_file_path
-        if connection.sslConfig.__root__.sslCertificate:
+        if connection.sslConfig.root.sslCertificate:
             ssl_args["ssl_cert"] = self.cert_file_path
-        if connection.sslConfig.__root__.sslKey:
+        if connection.sslConfig.root.sslKey:
             ssl_args["ssl_key"] = self.key_file_path
-        connection.connectionArguments.__root__["ssl"] = ssl_args
+        connection.connectionArguments.root["ssl"] = ssl_args
         return connection
 
     @setup_ssl.register(PostgresConnection)
@@ -117,15 +117,13 @@ class SSLManager:
 
         if not connection.connectionArguments:
             connection.connectionArguments = init_empty_connection_arguments()
-        connection.connectionArguments.__root__["sslmode"] = connection.sslMode.value
+        connection.connectionArguments.root["sslmode"] = connection.sslMode.value
         if connection.sslMode in (
             verifySSLConfig.SslMode.verify_ca,
             verifySSLConfig.SslMode.verify_full,
         ):
             if self.ca_file_path:
-                connection.connectionArguments.__root__[
-                    "sslrootcert"
-                ] = self.ca_file_path
+                connection.connectionArguments.root["sslrootcert"] = self.ca_file_path
             else:
                 raise ValueError(
                     "CA certificate is required for SSL mode verify-ca or verify-full"
@@ -145,10 +143,10 @@ class SSLManager:
             session.verify = self.ca_file_path
         if self.cert_file_path and self.key_file_path:
             session.cert = (self.cert_file_path, self.key_file_path)
-        connection.connectionArguments.__root__ = (
-            connection.connectionArguments.__root__ or {}
+        connection.connectionArguments.root = (
+            connection.connectionArguments.root or {}
         )  # to satisfy mypy
-        connection.connectionArguments.__root__["session"] = session
+        connection.connectionArguments.root["session"] = session
         return connection
 
     @setup_ssl.register(QlikSenseConnection)
@@ -180,13 +178,13 @@ def check_ssl_and_init(_) -> None:
 def _(connection) -> Union[SSLManager, None]:
     service_connection = cast(SalesforceConnection, connection)
     ssl: Optional[verifySSLConfig.SslConfig] = service_connection.sslConfig
-    if ssl and ssl.__root__.caCertificate:
+    if ssl and ssl.root.caCertificate:
         ssl_dict: dict[str, Union[CustomSecretStr, None]] = {
-            "ca": ssl.__root__.caCertificate
+            "ca": ssl.root.caCertificate
         }
-        if (ssl.__root__.sslCertificate) and (ssl.__root__.sslKey):
-            ssl_dict["cert"] = ssl.__root__.sslCertificate
-            ssl_dict["key"] = ssl.__root__.sslKey
+        if (ssl.root.sslCertificate) and (ssl.root.sslKey):
+            ssl_dict["cert"] = ssl.root.sslCertificate
+            ssl_dict["key"] = ssl.root.sslKey
         return SSLManager(**ssl_dict)
     return None
 
@@ -196,13 +194,11 @@ def _(connection) -> Union[SSLManager, None]:
 def _(connection):
     service_connection = cast(Union[MysqlConnection, DorisConnection], connection)
     ssl: Optional[verifySSLConfig.SslConfig] = service_connection.sslConfig
-    if ssl and (
-        ssl.__root__.caCertificate or ssl.__root__.sslCertificate or ssl.__root__.sslKey
-    ):
+    if ssl and (ssl.root.caCertificate or ssl.root.sslCertificate or ssl.root.sslKey):
         return SSLManager(
-            ca=ssl.__root__.caCertificate,
-            cert=ssl.__root__.sslCertificate,
-            key=ssl.__root__.sslKey,
+            ca=ssl.root.caCertificate,
+            cert=ssl.root.sslCertificate,
+            key=ssl.root.sslKey,
         )
     return None
 
@@ -217,9 +213,7 @@ def _(connection):
     )
     if connection.sslMode:
         return SSLManager(
-            ca=connection.sslConfig.__root__.caCertificate
-            if connection.sslConfig
-            else None
+            ca=connection.sslConfig.root.caCertificate if connection.sslConfig else None
         )
     return None
 

metadata/utils/ssl_registry.py

@@ -39,7 +39,11 @@ def ignore_ssl_init(_: Optional[SslConfig]) -> bool:
 
 @ssl_verification_registry.add(VerifySSL.validate.value)
 def validate_ssl_init(ssl_config: Optional[SslConfig]) -> str:
-    return ssl_config.__root__.caCertificate.get_secret_value()
+    if ssl_config is None:
+        raise InvalidSSLVerificationException(
+            "You have Verify SSL but the SSL Config is missing. Make sure to inform the CA Certificate path."
+        )
+    return ssl_config.root.caCertificate.get_secret_value()
 
 
 def get_verify_ssl_fn(verify_ssl: VerifySSL) -> Callable:

metadata/utils/storage_metadata_config.py

@@ -79,7 +79,8 @@ def _(config: StorageMetadataLocalConfig) -> ManifestMetadataConfig:
             logger.debug(f"Reading [manifestFilePath] from: {config.manifestFilePath}")
             with open(config.manifestFilePath, "r", encoding="utf-8") as manifest:
                 metadata_manifest = manifest.read()
-        return ManifestMetadataConfig.parse_obj(json.loads(metadata_manifest))
+            return ManifestMetadataConfig.model_validate(json.loads(metadata_manifest))
+        raise StorageMetadataConfigException("Manifest file path not provided")
     except Exception as exc:
         logger.debug(traceback.format_exc())
         raise StorageMetadataConfigException(
@@ -98,7 +99,7 @@ def _(config: StorageMetadataHttpConfig) -> ManifestMetadataConfig:
             raise StorageMetadataConfigException(
                 "Manifest file not found in file server"
             )
-        return ManifestMetadataConfig.parse_obj(http_manifest.json())
+        return ManifestMetadataConfig.model_validate(http_manifest.json())
     except Exception as exc:
         logger.debug(traceback.format_exc())
         raise StorageMetadataConfigException(
@@ -131,7 +132,7 @@ def _(config: StorageMetadataS3Config) -> ManifestMetadataConfig:
         )
 
         manifest = reader.read(path=path, bucket_name=bucket_name)
-        return ManifestMetadataConfig.parse_obj(json.loads(manifest))
+        return ManifestMetadataConfig.model_validate(json.loads(manifest))
     except Exception as exc:
         logger.debug(traceback.format_exc())
         raise StorageMetadataConfigException(
@@ -162,7 +163,7 @@ def _(config: StorageMetadataAdlsConfig) -> ManifestMetadataConfig:
         )
 
         manifest = reader.read(path=path, bucket_name=bucket_name)
-        return ManifestMetadataConfig.parse_obj(json.loads(manifest))
+        return ManifestMetadataConfig.model_validate(json.loads(manifest))
     except Exception as exc:
         logger.debug(traceback.format_exc())
         raise StorageMetadataConfigException(
@@ -196,7 +197,7 @@ def _(config: StorageMetadataGcsConfig) -> ManifestMetadataConfig:
         )
 
         manifest = reader.read(path=path, bucket_name=bucket_name)
-        return ManifestMetadataConfig.parse_obj(json.loads(manifest))
+        return ManifestMetadataConfig.model_validate(json.loads(manifest))
     except Exception as exc:
         logger.debug(traceback.format_exc())
         raise StorageMetadataConfigException(

metadata/utils/tag_utils.py

@@ -14,7 +14,7 @@ Tag utils Module
 
 import functools
 import traceback
-from typing import Iterable, List, Optional, Union
+from typing import Iterable, List, Optional, Type, Union
 
 from metadata.generated.schema.api.classification.createClassification import (
     CreateClassificationRequest,
@@ -25,10 +25,15 @@ from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
-from metadata.generated.schema.type.basic import FullyQualifiedEntityName
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+)
 from metadata.generated.schema.type.tagLabel import (
     LabelType,
     State,
+    TagFQN,
     TagLabel,
     TagSource,
 )
@@ -44,8 +49,8 @@ logger = ingestion_logger()
 def get_ometa_tag_and_classification(
     tags: List[str],
     classification_name: str,
-    tag_description: Optional[str] = None,
-    classification_description: Optional[str] = None,
+    tag_description: str,
+    classification_description: str,
     include_tags: bool = True,
     tag_fqn: Optional[FullyQualifiedEntityName] = None,
 ) -> Iterable[Either[OMetaTagAndClassification]]:
@@ -58,13 +63,15 @@ def get_ometa_tag_and_classification(
                 classification = OMetaTagAndClassification(
                     fqn=tag_fqn,
                     classification_request=CreateClassificationRequest(
-                        name=classification_name,
-                        description=classification_description,
+                        name=EntityName(classification_name),
+                        description=Markdown(classification_description),
                     ),
                     tag_request=CreateTagRequest(
-                        classification=classification_name,
-                        name=tag,
-                        description=tag_description,
+                        classification=FullyQualifiedEntityName(classification_name),
+                        name=EntityName(tag),
+                        description=Markdown(tag_description)
+                        if tag_description
+                        else None,
                     ),
                 )
                 yield Either(right=classification)
@@ -86,7 +93,7 @@ def get_tag_label(
     metadata: OpenMetadata,
     tag_name: str,
     classification_name: Optional[str],
-    tag_type: Union[Tag, GlossaryTerm] = Tag,
+    tag_type: Union[Type[Tag], Type[GlossaryTerm]] = Tag,
 ) -> Optional[TagLabel]:
     """
     Returns the tag label if the tag is created
@@ -102,7 +109,8 @@ def get_tag_label(
             )
             source = TagSource.Classification.value
 
-        if tag_type == GlossaryTerm:
+        # We either have a Tag or a Glossary
+        else:
             tag_fqn = tag_name
             source = TagSource.Glossary.value
 
@@ -110,7 +118,7 @@ def get_tag_label(
         tag = metadata.get_by_name(entity=tag_type, fqn=tag_fqn)
         if tag:
             return TagLabel(
-                tagFQN=tag_fqn,
+                tagFQN=TagFQN(tag_fqn),
                 labelType=LabelType.Automated.value,
                 state=State.Suggested.value,
                 source=source,
@@ -129,7 +137,7 @@ def get_tag_labels(
     tags: List[str],
     classification_name: Optional[str] = None,
     include_tags: bool = True,
-    tag_type: Union[Tag, GlossaryTerm] = Tag,
+    tag_type: Union[Type[Tag], Type[GlossaryTerm]] = Tag,
 ) -> Optional[List[TagLabel]]:
     """
     Method to create tag labels from the collected tags

metadata/utils/time_utils.py

@@ -14,6 +14,7 @@ Time utility functions
 """
 
 from datetime import datetime, time, timedelta, timezone
+from math import floor
 from typing import Union
 
 from metadata.utils.helpers import datetime_to_ts
@@ -55,7 +56,7 @@ def get_beginning_of_day_timestamp_mill(
     Returns:
         int: timestamp milliseconds
     """
-    now_utc = datetime.utcnow()
+    now_utc = datetime.now(timezone.utc)
     delta = timedelta(
         weeks=weeks,
         days=days,
@@ -87,7 +88,7 @@ def get_end_of_day_timestamp_mill(
     Returns:
         int: timestamp milliseconds
     """
-    now_utc = datetime.utcnow()
+    now_utc = datetime.now(timezone.utc)
     delta = timedelta(
         weeks=weeks,
         days=days,
@@ -114,13 +115,52 @@ def convert_timestamp(timestamp: str) -> Union[int, float]:
     return float(timestamp) / 1000
 
 
-def convert_timestamp_to_milliseconds(timestamp: int) -> int:
+def convert_timestamp_to_milliseconds(timestamp: Union[int, float]) -> int:
     """convert timestamp to milliseconds
     Args:
         timestamp (int):
-    Retunrs:
+    Returns:
         int
     """
     if len(str(round(timestamp))) == 13:
         return timestamp
-    return timestamp * 1000
+    return round(timestamp * 1000)
+
+
+def timedelta_to_string(td: timedelta):
+    """Convert timedelta to human readable string
+
+    Example:
+        >>> timedelta_to_string(timedelta(days=1, hours=2, minutes=3, seconds=4))
+        '1 days 2 hours 3 minutes 4 seconds (total seconds: 93784.0)'
+
+    Args:
+        td (timedelta): timedelta object
+
+    Returns:
+        str: human readable string
+    """
+    res = []
+    current = td
+    if current.days:
+        res.append(f"{floor(td.days)} day")
+        if current.days > 1:
+            res[-1] += "s"
+        current -= timedelta(days=floor(td.days))
+    hours = current.seconds // 3600
+    if hours:
+        res.append(f"{hours} hour")
+        if hours > 1:
+            res[-1] += "s"
+        current -= timedelta(hours=hours)
+    minutes = current.seconds // 60
+    if minutes:
+        res.append(f"{minutes} minute")
+        if minutes > 1:
+            res[-1] += "s"
+        current -= timedelta(minutes=minutes)
+    res.append(f"{current.seconds} second")
+    if current.seconds != 1:
+        res[-1] += "s"
+    total_seconds = "total seconds: " + str(td.total_seconds())
+    return " ".join(res) + f" ({total_seconds})"

metadata/workflow/application.py

@@ -51,9 +51,9 @@ class AppRunner(Step, ABC):
         config: OpenMetadataApplicationConfig,
         metadata: OpenMetadata,
     ):
-        self.app_config = config.appConfig.__root__ if config.appConfig else None
+        self.app_config = config.appConfig.root if config.appConfig else None
         self.private_config = (
-            config.appPrivateConfig.__root__ if config.appPrivateConfig else None
+            config.appPrivateConfig.root if config.appPrivateConfig else None
         )
         self.metadata = metadata
 
@@ -74,7 +74,7 @@ class AppRunner(Step, ABC):
         metadata: OpenMetadata,
         pipeline_name: Optional[str] = None,
     ) -> "Step":
-        config = OpenMetadataApplicationConfig.parse_obj(config_dict)
+        config = OpenMetadataApplicationConfig.model_validate(config_dict)
         return cls(config=config, metadata=metadata)
 
 
@@ -87,7 +87,7 @@ class ApplicationWorkflow(BaseWorkflow, ABC):
     def __init__(self, config_dict: dict):
         self.runner = None  # Will be passed in post-init
         # TODO: Create a parse_gracefully method
-        self.config = OpenMetadataApplicationConfig.parse_obj(config_dict)
+        self.config = OpenMetadataApplicationConfig.model_validate(config_dict)
 
         # Applications are associated to the OpenMetadata Service
         self.service_type: ServiceType = ServiceType.Metadata

metadata/workflow/application_output_handler.py

@@ -8,27 +8,14 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-
 """
-Module handles the output messages for applications
+Generic Workflow entrypoint to execute Applications
 """
-import time
-
-from metadata.utils.helpers import pretty_print_time_duration
-from metadata.utils.logger import ANSI, log_ansi_encoded_string
-from metadata.workflow.output_handler import print_workflow_summary
 
+from metadata.utils.deprecation import deprecated
+from metadata.workflow.base import BaseWorkflow
 
-def print_status(workflow: "ApplicationWorkflow") -> None:
-    """
-    Print the workflow results
-    """
-    print_workflow_summary(workflow)
 
-    if workflow.runner.get_status().source_start_time:
-        log_ansi_encoded_string(
-            color=ANSI.BRIGHT_CYAN,
-            bold=True,
-            message="Workflow finished in time: "
-            f"{pretty_print_time_duration(time.time()-workflow.runner.get_status().source_start_time)}",
-        )
+@deprecated(message="Use 'workflow.print_status()' instead.", release="1.6")
+def print_status(workflow: BaseWorkflow):
+    workflow.print_status()

metadata/workflow/base.py

@@ -12,6 +12,7 @@
 Base workflow definition.
 """
 
+import traceback
 import uuid
 from abc import ABC, abstractmethod
 from datetime import datetime
@@ -47,7 +48,7 @@ from metadata.utils.class_helper import (
 from metadata.utils.execution_time_tracker import ExecutionTimeTracker
 from metadata.utils.helpers import datetime_to_ts
 from metadata.utils.logger import ingestion_logger, set_loggers_level
-from metadata.workflow.output_handler import report_ingestion_status
+from metadata.workflow.workflow_output_handler import WorkflowOutputHandler
 from metadata.workflow.workflow_status_mixin import (
     SUCCESS_THRESHOLD_VALUE,
     WorkflowStatusMixin,
@@ -84,10 +85,12 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
         log_level: LogLevels,
         metadata_config: OpenMetadataConnection,
         service_type: ServiceType,
+        output_handler: WorkflowOutputHandler = WorkflowOutputHandler(),
     ):
         """
         Disabling pylint to wait for workflow reimplementation as a topology
         """
+        self.output_handler = output_handler
         self.config = config
         self.service_type = service_type
         self._timer: Optional[RepeatedTimer] = None
@@ -107,7 +110,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
         self.post_init()
 
     @property
-    def ingestion_pipeline(self):
+    def ingestion_pipeline(self) -> Optional[IngestionPipeline]:
         """Get or create the Ingestion Pipeline from the configuration"""
         if not self._ingestion_pipeline and self.config.ingestionPipelineFQN:
             self._ingestion_pipeline = self.get_or_create_ingestion_pipeline()
@@ -136,7 +139,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
         """
         if not self._timer:
             self._timer = RepeatedTimer(
-                REPORTS_INTERVAL_SECONDS, report_ingestion_status, logger, self
+                REPORTS_INTERVAL_SECONDS, self._report_ingestion_status
             )
 
         return self._timer
@@ -205,7 +208,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
         """
         if not self._run_id:
             if self.config.pipelineRunId:
-                self._run_id = str(self.config.pipelineRunId.__root__)
+                self._run_id = str(self.config.pipelineRunId.root)
             else:
                 self._run_id = str(uuid.uuid4())
 
@@ -249,7 +252,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
                             ),
                         ),
                         pipelineType=get_pipeline_type_from_source_config(
-                            self.config.source.sourceConfig.config
+                            self.config.source.sourceConfig
                         ),
                         sourceConfig=self.config.source.sourceConfig,
                         airflowConfig=AirflowConfig(),
@@ -277,3 +280,37 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
             entity=get_service_class_from_service_type(self.service_type),
             fqn=self.config.source.serviceName,
         )
+
+    def _report_ingestion_status(self):
+        """
+        Given a logger, use it to INFO the workflow status
+        """
+        try:
+            for step in self.workflow_steps():
+                logger.info(
+                    f"{step.name}: Processed {len(step.status.records)} records,"
+                    f" filtered {len(step.status.filtered)} records,"
+                    f" found {len(step.status.failures)} errors"
+                )
+
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.error(f"Wild exception reporting status - {exc}")
+
+    def _is_debug_enabled(self) -> bool:
+        return (
+            hasattr(self, "config")
+            and hasattr(self.config, "workflowConfig")
+            and hasattr(self.config.workflowConfig, "loggerLevel")
+            and self.config.workflowConfig.loggerLevel is LogLevels.DEBUG
+        )
+
+    def print_status(self):
+        start_time = self.workflow_steps()[0].get_status().source_start_time
+
+        self.output_handler.print_status(
+            self.result_status(),
+            self.workflow_steps(),
+            start_time,
+            self._is_debug_enabled(),
+        )