openmetadata-ingestion 1.5.13.0__py3-none-any.whl → 1.6.0.0rc2__py3-none-any.whl

metadata/utils/service_spec/service_spec.py

@@ -0,0 +1,146 @@
+"""
+Manifests are used to store class information
+"""
+
+from typing import Optional, Type, cast
+
+from pydantic import model_validator
+
+from metadata.data_quality.interface.test_suite_interface import TestSuiteInterface
+from metadata.generated.schema.entity.services.serviceType import ServiceType
+from metadata.ingestion.api.steps import Source
+from metadata.ingestion.models.custom_pydantic import BaseModel
+from metadata.profiler.interface.profiler_interface import ProfilerInterface
+from metadata.sampler.sampler_interface import SamplerInterface
+from metadata.utils.importer import (
+    TYPE_SEPARATOR,
+    DynamicImportException,
+    get_class_path,
+    get_module_dir,
+    import_from_module,
+)
+from metadata.utils.logger import utils_logger
+
+logger = utils_logger()
+
+
+class BaseSpec(BaseModel):
+    """
+    # The OpenMetadata Ingestion Service Specification (Spec)
+
+    This is the API for defining a service in OpenMetadata it needs to be in the classpath of the connector in
+    the form:
+
+    metadata.ingestion.source.{service_type}.{service_name}.service_spec.ServiceSpec
+
+    Example for postres:
+
+    metadata.ingestion.source.database.postgres.service_spec.ServiceSpec
+
+    You can supply either strings with the full classpath or concrete classes that will be converted to strings.
+
+    The use of strings for the values gives us a few advantages:
+    1. manifests can be defined using json/yaml and deserialized into this class.
+    2. We can dynamically import the class when needed and avoid dependency issues.
+    3. We avoid circular imports.
+    4. We can hot-swap the class implementation without changing the manifest (example: for testing).
+    """
+
+    profiler_class: Optional[str] = None
+    test_suite_class: Optional[str] = None
+    metadata_source_class: str
+    lineage_source_class: Optional[str] = None
+    usage_source_class: Optional[str] = None
+    sampler_class: Optional[str] = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def transform_fields(cls, values):
+        """This allows us to pass in the class directly instead of the string representation of the class. The
+        validator will convert the class to a string representation of the class."""
+        for field in list(cls.model_fields.keys()):
+            if isinstance(values.get(field), type):
+                values[field] = get_class_path(values[field])
+        return values
+
+    @classmethod
+    def get_for_source(
+        cls, service_type: ServiceType, source_type: str, from_: str = "ingestion"
+    ) -> "BaseSpec":
+        """Retrieves the manifest for a given source type. If it does not exist will attempt to retrieve
+        a default manifest for the service type.
+
+        Args:
+            service_type (ServiceType): The service type.
+            source_type (str): The source type.
+            from_ (str, optional): The module to import from. Defaults to "ingestion".
+
+        Returns:
+            BaseSpec: The manifest for the source type.
+        """
+        return cls.model_validate(
+            import_from_module(
+                "metadata.{}.source.{}.{}.{}.ServiceSpec".format(  # pylint: disable=C0209
+                    from_,
+                    service_type.name.lower(),
+                    get_module_dir(source_type),
+                    "service_spec",
+                )
+            )
+        )
+
+
+def import_source_class(
+    service_type: ServiceType, source_type: str, from_: str = "ingestion"
+) -> Type[Source]:
+    source_class_type = source_type.split(TYPE_SEPARATOR)[-1]
+    if source_class_type in ["usage", "lineage"]:
+        field = f"{source_class_type}_source_class"
+    else:
+        field = "metadata_source_class"
+    spec = BaseSpec.get_for_source(service_type, source_type, from_)
+    return cast(
+        Type[Source],
+        import_from_module(spec.model_dump()[field]),
+    )
+
+
+def import_profiler_class(
+    service_type: ServiceType, source_type: str
+) -> Type[ProfilerInterface]:
+    class_path = BaseSpec.get_for_source(service_type, source_type).profiler_class
+    return cast(Type[ProfilerInterface], import_from_module(class_path))
+
+
+def import_test_suite_class(
+    service_type: ServiceType,
+    source_type: str,
+    source_config_type: Optional[str] = None,
+) -> Type[TestSuiteInterface]:
+    try:
+        class_path = BaseSpec.get_for_source(service_type, source_type).test_suite_class
+    except DynamicImportException:
+        if source_config_type:
+            class_path = BaseSpec.get_for_source(
+                service_type, source_config_type.lower()
+            ).test_suite_class
+        else:
+            raise
+    return cast(Type[TestSuiteInterface], import_from_module(class_path))
+
+
+def import_sampler_class(
+    service_type: ServiceType,
+    source_type: str,
+    source_config_type: Optional[str] = None,
+) -> Type[SamplerInterface]:
+    try:
+        class_path = BaseSpec.get_for_source(service_type, source_type).sampler_class
+    except DynamicImportException:
+        if source_config_type:
+            class_path = BaseSpec.get_for_source(
+                service_type, source_config_type.lower()
+            ).sampler_class
+        else:
+            raise
+    return cast(Type[SamplerInterface], import_from_module(class_path))

metadata/utils/ssl_manager.py

@@ -51,6 +51,9 @@ from metadata.generated.schema.entity.services.connections.database.salesforceCo
 from metadata.generated.schema.entity.services.connections.messaging.kafkaConnection import (
     KafkaConnection,
 )
+from metadata.generated.schema.entity.services.connections.pipeline.matillionConnection import (
+    MatillionConnection,
+)
 from metadata.generated.schema.security.ssl import verifySSLConfig
 from metadata.ingestion.connections.builders import init_empty_connection_arguments
 from metadata.ingestion.models.custom_pydantic import CustomSecretStr
@@ -112,6 +115,21 @@ class SSLManager:
         connection.connectionArguments.root["ssl"] = ssl_args
         return connection
 
+    @setup_ssl.register(MatillionConnection)
+    def _(self, connection):
+        matillion_connection = cast(MatillionConnection, connection)
+        if (
+            matillion_connection.connection
+            and matillion_connection.connection.sslConfig
+        ):
+            if matillion_connection.connection.sslConfig.root.caCertificate:
+                setattr(
+                    matillion_connection.connection.sslConfig.root,
+                    "caCertificate",
+                    self.ca_file_path,
+                )
+        return connection
+
     @setup_ssl.register(PostgresConnection)
     @setup_ssl.register(RedshiftConnection)
     @setup_ssl.register(GreenplumConnection)
@@ -194,6 +212,21 @@ def check_ssl_and_init(_) -> Optional[SSLManager]:
     return None
 
 
+@check_ssl_and_init.register(MatillionConnection)
+def _(connection) -> Union[SSLManager, None]:
+    service_connection = cast(MatillionConnection, connection)
+    if service_connection.connection:
+        ssl: Optional[
+            verifySSLConfig.SslConfig
+        ] = service_connection.connection.sslConfig
+        if ssl and ssl.root.caCertificate:
+            ssl_dict: dict[str, Union[CustomSecretStr, None]] = {
+                "ca": ssl.root.caCertificate
+            }
+            return SSLManager(**ssl_dict)
+    return None
+
+
 @check_ssl_and_init.register(cls=SalesforceConnection)
 def _(connection) -> Union[SSLManager, None]:
     service_connection = cast(SalesforceConnection, connection)

metadata/utils/time_utils.py

@@ -17,28 +17,55 @@ from datetime import datetime, time, timedelta, timezone
 from math import floor
 from typing import Union
 
+from metadata.generated.schema.type.basic import Timestamp
+from metadata.utils.deprecation import deprecated
 from metadata.utils.helpers import datetime_to_ts
+from metadata.utils.logger import utils_logger
 
+logger = utils_logger()
 
-def datetime_to_timestamp(datetime_value, milliseconds=False) -> int:
-    """Convert a datetime object to timestamp integer
+
+def datetime_to_timestamp(datetime_value: datetime, milliseconds=False) -> int:
+    """Convert a datetime object to timestamp integer. If datetime is timezone aware, it will be converted to UTC.
+    If it is naive it will be assumed to be in UTC.
+
+    # TODO: not sure the milliseconds flag is useful. Maybe this should return a 'Timestamp' object instead.
 
     Args:
         datetime_value (_type_): datetime object
         milliseconds (bool, optional): make it a milliseconds timestamp. Defaults to False.
 
     Returns:
-        int:
+        int : timestamp in seconds or milliseconds
     """
     if not getattr(datetime_value, "timestamp", None):
-        raise TypeError(f"Object of type {datetime_value} has not method `timestamp()`")
-
+        raise TypeError(
+            f"Object of type {type(datetime_value).__name__} has not method `timestamp()`"
+        )
+
+    datetime_value = (
+        datetime_value.replace(tzinfo=timezone.utc)
+        if datetime_value.tzinfo is None
+        else datetime_value.astimezone(timezone.utc)
+    )
     tmsap = datetime_value.timestamp()
     if milliseconds:
         return int(tmsap * 1000)
     return int(tmsap)
 
 
+def timestamp_to_datetime(ts: Timestamp) -> datetime:
+    """Convert a timestamp to datetime object in UTC.
+
+    Args:
+        ts (Timestamp): timestamp
+
+    Returns:
+        datetime: datetime object
+    """
+    return datetime.fromtimestamp(ts.root / 1000, tz=timezone.utc)
+
+
 def get_beginning_of_day_timestamp_mill(
     days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0
 ) -> int:
@@ -115,6 +142,7 @@ def convert_timestamp(timestamp: str) -> Union[int, float]:
     return float(timestamp) / 1000
 
 
+@deprecated("Use `datetime_to_timestamp` instead", "1.7.0")
 def convert_timestamp_to_milliseconds(timestamp: Union[int, float]) -> int:
     """convert timestamp to milliseconds
     Args:

metadata/workflow/application.py

@@ -14,10 +14,6 @@ Generic Workflow entrypoint to execute Applications
 from abc import ABC, abstractmethod
 from typing import List, Optional
 
-from metadata.config.common import WorkflowExecutionError
-from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
-    OpenMetadataConnection,
-)
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
@@ -25,13 +21,11 @@ from metadata.generated.schema.entity.services.serviceType import ServiceType
 from metadata.generated.schema.metadataIngestion.application import (
     OpenMetadataApplicationConfig,
 )
-from metadata.generated.schema.metadataIngestion.workflow import LogLevels
-from metadata.ingestion.api.step import Step, Summary
+from metadata.ingestion.api.step import Step
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils.importer import import_from_module
 from metadata.utils.logger import ingestion_logger
 from metadata.workflow.base import BaseWorkflow
-from metadata.workflow.workflow_status_mixin import SUCCESS_THRESHOLD_VALUE
 
 logger = ingestion_logger()
 
@@ -84,29 +78,24 @@ class ApplicationWorkflow(BaseWorkflow, ABC):
     config: OpenMetadataApplicationConfig
     runner: Optional[AppRunner]
 
-    def __init__(self, config_dict: dict):
+    def __init__(self, config: OpenMetadataApplicationConfig):
         self.runner = None  # Will be passed in post-init
-        # TODO: Create a parse_gracefully method
-        self.config = OpenMetadataApplicationConfig.model_validate(config_dict)
+        self.config = config
 
         # Applications are associated to the OpenMetadata Service
         self.service_type: ServiceType = ServiceType.Metadata
 
-        metadata_config: OpenMetadataConnection = (
-            self.config.workflowConfig.openMetadataServerConfig
-        )
-        log_level: LogLevels = self.config.workflowConfig.loggerLevel
-
         super().__init__(
             config=self.config,
-            log_level=log_level,
-            metadata_config=metadata_config,
+            workflow_config=config.workflowConfig,
             service_type=self.service_type,
         )
 
     @classmethod
     def create(cls, config_dict: dict):
-        return cls(config_dict)
+        # TODO: Create a parse_gracefully method
+        config = OpenMetadataApplicationConfig.model_validate(config_dict)
+        return cls(config)
 
     def post_init(self) -> None:
         """
@@ -134,26 +123,8 @@ class ApplicationWorkflow(BaseWorkflow, ABC):
         """Workflow-specific logic to execute safely"""
         self.runner.run()
 
-    def calculate_success(self) -> float:
-        return self.runner.get_status().calculate_success()
-
     def get_failures(self) -> List[StackTraceError]:
         return self.workflow_steps()[0].get_status().failures
 
     def workflow_steps(self) -> List[Step]:
         return [self.runner]
-
-    def raise_from_status_internal(self, raise_warnings=False):
-        """Check failed status in the runner"""
-        if (
-            self.runner.get_status().failures
-            and self.calculate_success() < SUCCESS_THRESHOLD_VALUE
-        ):
-            raise WorkflowExecutionError(
-                f"{self.runner.name} reported errors: {Summary.from_step(self.runner)}"
-            )
-
-        if raise_warnings and self.runner.get_status().warnings:
-            raise WorkflowExecutionError(
-                f"{self.runner.name} reported warning: {Summary.from_step(self.runner)}"
-            )

metadata/workflow/base.py

@@ -16,8 +16,10 @@ import traceback
 import uuid
 from abc import ABC, abstractmethod
 from datetime import datetime
+from statistics import mean
 from typing import Any, Dict, List, Optional, TypeVar, Union
 
+from metadata.config.common import WorkflowExecutionError
 from metadata.generated.schema.api.services.ingestionPipelines.createIngestionPipeline import (
     CreateIngestionPipelineRequest,
 )
@@ -32,10 +34,13 @@ from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipel
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
-from metadata.generated.schema.metadataIngestion.workflow import LogLevels
+from metadata.generated.schema.metadataIngestion.workflow import (
+    LogLevels,
+    WorkflowConfig,
+)
 from metadata.generated.schema.tests.testSuite import ServiceType
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.step import Step
+from metadata.ingestion.api.step import Step, Summary
 from metadata.ingestion.ometa.client_utils import create_ometa_client
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.timer.repeated_timer import RepeatedTimer
@@ -49,10 +54,7 @@ from metadata.utils.execution_time_tracker import ExecutionTimeTracker
 from metadata.utils.helpers import datetime_to_ts
 from metadata.utils.logger import ingestion_logger, set_loggers_level
 from metadata.workflow.workflow_output_handler import WorkflowOutputHandler
-from metadata.workflow.workflow_status_mixin import (
-    SUCCESS_THRESHOLD_VALUE,
-    WorkflowStatusMixin,
-)
+from metadata.workflow.workflow_status_mixin import WorkflowStatusMixin
 
 logger = ingestion_logger()
 
@@ -82,8 +84,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
     def __init__(
         self,
         config: Union[Any, Dict],
-        log_level: LogLevels,
-        metadata_config: OpenMetadataConnection,
+        workflow_config: WorkflowConfig,
         service_type: ServiceType,
         output_handler: WorkflowOutputHandler = WorkflowOutputHandler(),
     ):
@@ -92,19 +93,22 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
         """
         self.output_handler = output_handler
         self.config = config
+        self.workflow_config = workflow_config
         self.service_type = service_type
         self._timer: Optional[RepeatedTimer] = None
         self._ingestion_pipeline: Optional[IngestionPipeline] = None
         self._start_ts = datetime_to_ts(datetime.now())
+
         self._execution_time_tracker = ExecutionTimeTracker(
-            log_level == LogLevels.DEBUG
+            self.workflow_config.loggerLevel == LogLevels.DEBUG
         )
 
-        set_loggers_level(log_level.value)
+        set_loggers_level(self.workflow_config.loggerLevel.value)
 
         # We create the ometa client at the workflow level and pass it to the steps
-        self.metadata_config = metadata_config
-        self.metadata = create_ometa_client(metadata_config)
+        self.metadata = create_ometa_client(
+            self.workflow_config.openMetadataServerConfig
+        )
         self.set_ingestion_pipeline_status(state=PipelineState.running)
 
         self.post_init()
@@ -157,9 +161,22 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
     def execute_internal(self) -> None:
         """Workflow-specific logic to execute safely"""
 
-    @abstractmethod
-    def calculate_success(self) -> float:
-        """Get the success % of the internal execution"""
+    def calculate_success(self) -> Optional[float]:
+        """
+        Get the success % of the internal execution.
+        Since we'll use this to get a single success % from multiple steps, we'll take
+        the minimum success % from all the steps. This way, we can have a proper
+        workflow status.
+        E.g., if we have no errors on the source but a bunch of them on the sink,
+        we still want the flow to be marked as a failure or partial success.
+        """
+        if not self.workflow_steps():
+            logger.warning("No steps to calculate success")
+            return None
+
+        return mean(
+            [step.get_status().calculate_success() for step in self.workflow_steps()]
+        )
 
     @abstractmethod
     def get_failures(self) -> List[StackTraceError]:
@@ -169,9 +186,22 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
     def workflow_steps(self) -> List[Step]:
         """Steps to report status from"""
 
-    @abstractmethod
     def raise_from_status_internal(self, raise_warnings=False) -> None:
         """Based on the internal workflow status, raise a WorkflowExecutionError"""
+        for step in self.workflow_steps():
+            if (
+                step.get_status().failures
+                and step.get_status().calculate_success()
+                < self.workflow_config.successThreshold
+            ):
+                raise WorkflowExecutionError(
+                    f"{step.name} reported errors: {Summary.from_step(step)}"
+                )
+
+            if raise_warnings and step.status.warnings:
+                raise WorkflowExecutionError(
+                    f"{step.name} reported warning: {Summary.from_step(step)}"
+                )
 
     def execute(self) -> None:
         """
@@ -186,7 +216,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
         try:
             self.execute_internal()
 
-            if SUCCESS_THRESHOLD_VALUE <= self.calculate_success() < 100:
+            if self.workflow_config.successThreshold <= self.calculate_success() < 100:
                 pipeline_state = PipelineState.partialSuccess
 
         # Any unhandled exception breaking the workflow should update the status

metadata/workflow/classification.py

@@ -0,0 +1,53 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Workflow definition for the profiler
+"""
+from typing import cast
+
+from metadata.generated.schema.metadataIngestion.databaseServiceAutoClassificationPipeline import (
+    DatabaseServiceAutoClassificationPipeline,
+)
+from metadata.ingestion.api.steps import Processor
+from metadata.pii.processor import PIIProcessor
+from metadata.sampler.processor import SamplerProcessor
+from metadata.utils.logger import profiler_logger
+from metadata.workflow.profiler import ProfilerWorkflow
+
+logger = profiler_logger()
+
+
+class AutoClassificationWorkflow(ProfilerWorkflow):
+    """Auto Classification workflow implementation. Based on the Profiler logic with different steps"""
+
+    def set_steps(self):
+        source_class = self._get_source_class()
+        self.source = source_class.create(self.config.model_dump(), self.metadata)
+
+        sink = self._get_sink()
+        sampler_processor = self._get_sampler_processor()
+
+        # Only instantiate the PII Processor on demand
+        source_config: DatabaseServiceAutoClassificationPipeline = cast(
+            DatabaseServiceAutoClassificationPipeline,
+            self.config.source.sourceConfig.config,
+        )
+        if source_config.enableAutoClassification:
+            pii_processor = self._get_pii_processor()
+            self.steps = (sampler_processor, pii_processor, sink)
+        else:
+            self.steps = (sampler_processor, sink)
+
+    def _get_pii_processor(self) -> Processor:
+        return PIIProcessor.create(self.config.model_dump(), self.metadata)
+
+    def _get_sampler_processor(self) -> Processor:
+        return SamplerProcessor.create(self.config.model_dump(), self.metadata)

metadata/workflow/ingestion.py

@@ -24,9 +24,6 @@ from abc import ABC, abstractmethod
 from typing import List, Tuple, Type, cast
 
 from metadata.config.common import WorkflowExecutionError
-from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
-    OpenMetadataConnection,
-)
 from metadata.generated.schema.entity.services.connections.serviceConnection import (
     ServiceConnection,
 )
@@ -38,7 +35,7 @@ from metadata.generated.schema.metadataIngestion.workflow import (
     OpenMetadataWorkflowConfig,
 )
 from metadata.ingestion.api.parser import parse_workflow_config_gracefully
-from metadata.ingestion.api.step import Step, Summary
+from metadata.ingestion.api.step import Step
 from metadata.ingestion.api.steps import BulkSink, Processor, Sink, Source, Stage
 from metadata.ingestion.models.custom_types import ServiceWithConnectionType
 from metadata.profiler.api.models import ProfilerProcessorConfig
@@ -51,18 +48,19 @@ from metadata.utils.importer import (
     DynamicImportException,
     MissingPluginException,
     import_from_module,
-    import_source_class,
 )
 from metadata.utils.logger import ingestion_logger
+from metadata.utils.service_spec.service_spec import import_source_class
 from metadata.workflow.base import BaseWorkflow, InvalidWorkflowJSONException
-from metadata.workflow.workflow_status_mixin import SUCCESS_THRESHOLD_VALUE
 
 logger = ingestion_logger()
 
 
 class IngestionWorkflow(BaseWorkflow, ABC):
     """
-    Base Ingestion Workflow implementation
+    Base Ingestion Workflow implementation. This is used for all
+    workflows minus the application one, which directly inherits the
+    BaseWorkflow.
     """
 
     config: OpenMetadataWorkflowConfig
@@ -79,14 +77,9 @@ class IngestionWorkflow(BaseWorkflow, ABC):
             self.config.source.type
         )
 
-        metadata_config: OpenMetadataConnection = (
-            self.config.workflowConfig.openMetadataServerConfig
-        )
-
         super().__init__(
             config=config,
-            log_level=config.workflowConfig.loggerLevel,
-            metadata_config=metadata_config,
+            workflow_config=config.workflowConfig,
             service_type=self.service_type,
         )
 
@@ -137,37 +130,12 @@ class IngestionWorkflow(BaseWorkflow, ABC):
         if bulk_sink:
             bulk_sink.run()
 
-    def calculate_success(self) -> float:
-        return self.source.get_status().calculate_success()
-
     def get_failures(self) -> List[StackTraceError]:
         return self.source.get_status().failures
 
     def workflow_steps(self) -> List[Step]:
         return [self.source] + list(self.steps)
 
-    def raise_from_status_internal(self, raise_warnings=False):
-        """
-        Check the status of all steps
-        """
-        if (
-            self.source.get_status().failures
-            and self.calculate_success() < SUCCESS_THRESHOLD_VALUE
-        ):
-            raise WorkflowExecutionError(
-                f"{self.source.name} reported errors: {Summary.from_step(self.source)}"
-            )
-
-        for step in self.steps:
-            if step.status.failures:
-                raise WorkflowExecutionError(
-                    f"{step.name} reported errors: {Summary.from_step(step)}"
-                )
-            if raise_warnings and step.status.warnings:
-                raise WorkflowExecutionError(
-                    f"{step.name} reported warnings: {Summary.from_step(step)}"
-                )
-
     def _retrieve_service_connection_if_needed(self, service_type: ServiceType) -> None:
         """
         We override the current `serviceConnection` source config object if source workflow service already exists

metadata/workflow/profiler.py

@@ -11,11 +11,7 @@
 """
 Workflow definition for the profiler
 """
-from typing import cast
 
-from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline import (
-    DatabaseServiceProfilerPipeline,
-)
 from metadata.generated.schema.metadataIngestion.workflow import (
     OpenMetadataWorkflowConfig,
 )
@@ -24,7 +20,6 @@ from metadata.ingestion.connections.test_connections import (
     raise_test_connection_exception,
 )
 from metadata.ingestion.source.connections import get_test_connection_fn
-from metadata.pii.processor import PIIProcessor
 from metadata.profiler.processor.processor import ProfilerProcessor
 from metadata.profiler.source.metadata import OpenMetadataSource
 from metadata.profiler.source.metadata_ext import OpenMetadataSourceExt
@@ -68,15 +63,7 @@ class ProfilerWorkflow(IngestionWorkflow):
         profiler_processor = self._get_profiler_processor()
         sink = self._get_sink()
 
-        # Only instantiate the PII Processor on demand
-        source_config: DatabaseServiceProfilerPipeline = cast(
-            DatabaseServiceProfilerPipeline, self.config.source.sourceConfig.config
-        )
-        if source_config.processPiiSensitive:
-            pii_processor = self._get_pii_processor()
-            self.steps = (profiler_processor, pii_processor, sink)
-        else:
-            self.steps = (profiler_processor, sink)
+        self.steps = (profiler_processor, sink)
 
     def test_connection(self) -> None:
         service_config = self.config.source.serviceConnection.root.config
@@ -97,6 +84,3 @@ class ProfilerWorkflow(IngestionWorkflow):
 
     def _get_profiler_processor(self) -> Processor:
         return ProfilerProcessor.create(self.config.model_dump(), self.metadata)
-
-    def _get_pii_processor(self) -> Processor:
-        return PIIProcessor.create(self.config.model_dump(), self.metadata)