openmetadata-ingestion 1.5.4.1__py3-none-any.whl → 1.5.6.0__py3-none-any.whl

metadata/ingestion/source/metadata/alationsink/metadata.py

@@ -14,11 +14,17 @@ AlationSink source to extract metadata
 """
 
 import traceback
-from typing import Iterable, Optional
+from typing import Iterable, List, Optional
 
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
-from metadata.generated.schema.entity.data.table import Column, Constraint, Table
+from metadata.generated.schema.entity.data.table import (
+    Column,
+    Constraint,
+    ConstraintType,
+    Table,
+    TableConstraint,
+)
 from metadata.generated.schema.entity.services.connections.metadata.alationSinkConnection import (
     AlationSinkConnection,
 )
@@ -108,7 +114,9 @@ class AlationsinkSource(Source):
                     ),
                 ),
                 db_username="Test",
-                title=model_str(om_database.name),
+                title=om_database.displayName
+                if om_database.displayName
+                else model_str(om_database.name),
                 description=model_str(om_database.description),
             )
         except Exception as exc:
@@ -129,7 +137,9 @@ class AlationsinkSource(Source):
                 key=fqn._build(  # pylint: disable=protected-access
                     str(alation_datasource_id), model_str(om_schema.name)
                 ),
-                title=model_str(om_schema.name),
+                title=om_schema.displayName
+                if om_schema.displayName
+                else model_str(om_schema.name),
                 description=model_str(om_schema.description),
             )
         except Exception as exc:
@@ -150,7 +160,9 @@ class AlationsinkSource(Source):
                 key=fqn._build(  # pylint: disable=protected-access
                     str(alation_datasource_id), schema_name, model_str(om_table.name)
                 ),
-                title=model_str(om_table.name),
+                title=om_table.displayName
+                if om_table.displayName
+                else model_str(om_table.name),
                 description=model_str(om_table.description),
                 table_type=TABLE_TYPE_MAPPER.get(om_table.tableType, "TABLE"),
                 sql=om_table.schemaDefinition,
@@ -162,14 +174,60 @@ class AlationsinkSource(Source):
             )
         return None
 
-    def _get_column_index(self, om_column: Column) -> Optional[ColumnIndex]:
+    def _update_foreign_key(
+        self,
+        alation_datasource_id: int,
+        om_column: Column,
+        table_constraints: Optional[List[TableConstraint]],
+        column_index: ColumnIndex,
+    ):
+        """
+        Method to update the foreign key metadata in columns index
+        """
+        try:
+            for table_constraint in table_constraints or []:
+                if table_constraint.constraintType == ConstraintType.FOREIGN_KEY:
+                    for i, constraint_column in enumerate(
+                        table_constraint.columns or []
+                    ):
+                        if constraint_column == model_str(om_column.name):
+                            column_index.isForeignKey = True
+                            # update the service name of OM with the alation datasource id in the column FQN
+                            splitted_col_fqn = fqn.split(
+                                model_str(table_constraint.referredColumns[i])
+                            )
+                            splitted_col_fqn[0] = str(alation_datasource_id)
+                            column_index.referencedColumnId = (
+                                fqn._build(  # pylint: disable=protected-access
+                                    *splitted_col_fqn
+                                )
+                            )
+                            break
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Failed to update foreign key for {model_str(om_column.name)}: {exc}"
+            )
+
+    def _get_column_index(
+        self,
+        alation_datasource_id: int,
+        om_column: Column,
+        table_constraints: Optional[List[TableConstraint]],
+    ) -> Optional[ColumnIndex]:
         """
         Method to get the alation column index
         """
         column_index = ColumnIndex()
         try:
+            # Attach the primary key
             if om_column.constraint == Constraint.PRIMARY_KEY:
-                return ColumnIndex(isPrimaryKey=True)
+                column_index.isPrimaryKey = True
+
+            # Attach the foreign key
+            self._update_foreign_key(
+                alation_datasource_id, om_column, table_constraints, column_index
+            )
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(
@@ -199,6 +257,7 @@ class AlationsinkSource(Source):
         schema_name: str,
         table_name: str,
         om_column: Column,
+        table_constraints: Optional[List[TableConstraint]],
     ) -> Optional[CreateColumnRequest]:
         """
         Method to form the CreateColumnRequest object
@@ -211,13 +270,19 @@ class AlationsinkSource(Source):
                     table_name,
                     model_str(om_column.name),
                 ),
-                column_type=om_column.dataType.value.lower(),
-                title=model_str(om_column.name),
+                column_type=om_column.dataTypeDisplay.lower()
+                if om_column.dataTypeDisplay
+                else om_column.dataType.value.lower(),
+                title=om_column.displayName
+                if om_column.displayName
+                else model_str(om_column.name),
                 description=model_str(om_column.description),
                 position=str(om_column.ordinalPosition)
                 if om_column.ordinalPosition
                 else None,
-                index=self._get_column_index(om_column),
+                index=self._get_column_index(
+                    alation_datasource_id, om_column, table_constraints
+                ),
                 nullable=self._check_nullable_column(om_column),
             )
         except Exception as exc:
@@ -241,6 +306,7 @@ class AlationsinkSource(Source):
                     schema_name=schema_name,
                     table_name=model_str(om_table.name),
                     om_column=om_column,
+                    table_constraints=om_table.tableConstraints,
                 )
                 if create_column_request:
                     create_requests.root.append(create_column_request)
@@ -266,6 +332,7 @@ class AlationsinkSource(Source):
                     entity=Table,
                     skip_on_failure=True,
                     params={"database": model_str(om_schema.fullyQualifiedName)},
+                    fields=["tableConstraints, columns"],
                 )
             )
             create_requests = CreateTableRequestList(root=[])

metadata/ingestion/source/pipeline/dbtcloud/metadata.py

@@ -314,7 +314,7 @@ class DbtcloudSource(PipelineServiceSource):
                 timestamp=Timestamp(
                     datetime_to_ts(
                         datetime.strptime(
-                            pipeline_details.created_at, "%Y-%m-%dT%H:%M:%S.%f%z"
+                            pipeline_details.created_at, "%Y-%m-%d %H:%M:%S.%f%z"
                         )
                         if pipeline_details.created_at
                         else None

metadata/pii/ner.py

@@ -51,7 +51,8 @@ class NEREntity(Enum):
     UK_NHS = TagType.SENSITIVE.value
 
     # Spain
-    NIF = TagType.SENSITIVE.value
+    ES_NIF = TagType.SENSITIVE.value
+    ES_NIE = TagType.SENSITIVE.value
 
     # Italy
     IT_FISCAL_CODE = TagType.SENSITIVE.value
@@ -60,8 +61,25 @@ class NEREntity(Enum):
     IT_PASSPORT = TagType.SENSITIVE.value
     IT_IDENTITY_CARD = TagType.SENSITIVE.value
 
+    # Poland
+    PL_PESEL = TagType.SENSITIVE.value
+
+    # Singapore
+    SG_NRIC_FIN = TagType.SENSITIVE.value
+    SG_UEN = TagType.SENSITIVE.value
+
     # Australia
     AU_ABN = TagType.SENSITIVE.value
     AU_ACN = TagType.SENSITIVE.value
     AU_TFN = TagType.SENSITIVE.value
     AU_MEDICARE = TagType.SENSITIVE.value
+
+    # India
+    IN_PAN = TagType.SENSITIVE.value
+    IN_AADHAAR = TagType.SENSITIVE.value
+    IN_VEHICLE_REGISTRATION = TagType.SENSITIVE.value
+    IN_VOTER = TagType.SENSITIVE.value
+    IN_PASSPORT = TagType.SENSITIVE.value
+
+    # Finland
+    FI_PERSONAL_IDENTITY_CODE = TagType.SENSITIVE.value

metadata/pii/processor.py

@@ -171,7 +171,7 @@ class PIIProcessor(Processor):
                 col_tags = self.process_column(
                     idx=idx,
                     column=column,
-                    table_data=record.sample_data,
+                    table_data=record.sample_data.data,
                     confidence_threshold=self.confidence_threshold,
                 )
                 if col_tags:

metadata/pii/scanners/ner_scanner.py

@@ -14,6 +14,7 @@ NER Scanner based on Presidio.
 Supported Entities https://microsoft.github.io/presidio/supported_entities/
 """
 import json
+import logging
 import traceback
 from collections import defaultdict
 from typing import Any, Dict, List, Optional, Tuple, Union
@@ -26,9 +27,11 @@ from metadata.pii.models import TagAndConfidence
 from metadata.pii.ner import NEREntity
 from metadata.pii.scanners.base import BaseScanner
 from metadata.utils import fqn
-from metadata.utils.logger import pii_logger
+from metadata.utils.logger import METADATA_LOGGER, pii_logger
 
 logger = pii_logger()
+SUPPORTED_LANG = "en"
+PRESIDIO_LOGGER = "presidio-analyzer"
 
 
 class StringAnalysis(BaseModel):
@@ -40,11 +43,16 @@ class StringAnalysis(BaseModel):
     appearances: int
 
 
+class NLPEngineModel(BaseModel):
+    """Required to pass the nlp_engine as {"lang_code": "en", "model_name": "en_core_web_lg"}"""
+
+    lang_code: str
+    model_name: str
+
+
 # pylint: disable=import-outside-toplevel
 class NERScanner(BaseScanner):
-    """
-    Based on https://microsoft.github.io/presidio/
-    """
+    """Based on https://microsoft.github.io/presidio/"""
 
     def __init__(self):
         import spacy
@@ -60,8 +68,19 @@ class NERScanner(BaseScanner):
             download(SPACY_EN_MODEL)
             spacy.load(SPACY_EN_MODEL)
 
+        nlp_engine_model = NLPEngineModel(
+            lang_code=SUPPORTED_LANG, model_name=SPACY_EN_MODEL
+        )
+
+        # Set the presidio logger to talk less about internal entities unless we are debugging
+        logging.getLogger(PRESIDIO_LOGGER).setLevel(
+            logging.INFO
+            if logging.getLogger(METADATA_LOGGER).level == logging.DEBUG
+            else logging.ERROR
+        )
+
         self.analyzer = AnalyzerEngine(
-            nlp_engine=SpacyNlpEngine(models={"en": SPACY_EN_MODEL})
+            nlp_engine=SpacyNlpEngine(models=[nlp_engine_model.model_dump()])
         )
 
     @staticmethod

metadata/profiler/api/models.py

@@ -17,9 +17,10 @@ multiple profilers per table and columns.
 """
 from typing import List, Optional, Type, Union
 
-from pydantic import ConfigDict
+from pydantic import ConfigDict, Field
 from sqlalchemy import Column
 from sqlalchemy.orm import DeclarativeMeta
+from typing_extensions import Annotated
 
 from metadata.config.common import ConfigModel
 from metadata.generated.schema.api.data.createTableProfile import (
@@ -37,6 +38,7 @@ from metadata.generated.schema.entity.services.connections.connectionBasicType i
 )
 from metadata.generated.schema.tests.customMetric import CustomMetric
 from metadata.generated.schema.type.basic import FullyQualifiedEntityName
+from metadata.ingestion.models.custom_pydantic import BaseModel
 from metadata.ingestion.models.table_metadata import ColumnTag
 from metadata.profiler.metrics.core import Metric, MetricTypes
 from metadata.profiler.processor.models import ProfilerDef
@@ -104,6 +106,15 @@ class ProfilerProcessorConfig(ConfigModel):
     databaseConfig: Optional[List[DatabaseAndSchemaConfig]] = []
 
 
+class SampleData(BaseModel):
+    """TableData wrapper to handle ephemeral SampleData"""
+
+    data: Annotated[TableData, Field(None, description="Table Sample Data")]
+    store: Annotated[
+        bool, Field(False, description="Is the sample data should be stored or not")
+    ]
+
+
 class ProfilerResponse(ConfigModel):
     """
     ORM Profiler processor response.
@@ -114,7 +125,7 @@ class ProfilerResponse(ConfigModel):
 
     table: Table
     profile: CreateTableProfileRequest
-    sample_data: Optional[TableData] = None
+    sample_data: Optional[SampleData] = None
     column_tags: Optional[List[ColumnTag]] = None
 
     def __str__(self):

metadata/profiler/metrics/composed/null_ratio.py

@@ -50,8 +50,9 @@ class NullRatio(ComposedMetric):
         results of other Metrics
         """
 
-        count = res.get(Count.name())
-        null_count = res.get(NullCount.name())
-        if count + null_count == 0:
+        count = res.get(Count.name(), 0)
+        null_count = res.get(NullCount.name(), 0)
+        total = count + null_count
+        if total == 0:
             return None
-        return null_count / (null_count + count)
+        return null_count / total

metadata/profiler/orm/functions/table_metric_computer.py

@@ -176,6 +176,8 @@ class SnowflakeTableMetricComputer(BaseTableMetricComputer):
         )
 
         rest = self._runner._session.execute(query).first()
+        if not rest:
+            return None
         if rest.rowCount is None:
             # if we don't have any row count, fallback to the base logic
             return super().compute()

metadata/profiler/orm/types/custom_hex_byte_string.py

@@ -22,6 +22,7 @@ from sqlalchemy.sql.sqltypes import String, TypeDecorator
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
+NULL_BYTE = "\x00"
 
 
 class HexByteString(TypeDecorator):
@@ -63,10 +64,22 @@ class HexByteString(TypeDecorator):
         detected_encoding = chardet.detect(bytes_value).get("encoding")
         if detected_encoding:
             try:
-                value = bytes_value.decode(encoding=detected_encoding)
-                return value
+                # Decode the bytes value with the detected encoding and replace errors with "?"
+                # if bytes cannot be decoded e.g. b"\x66\x67\x67\x9c", if detected_encoding="utf-8"
+                # will result in 'foo�' (instead of failing)
+                str_value = bytes_value.decode(
+                    encoding=detected_encoding, errors="replace"
+                )
+                # Replace NULL_BYTE with empty string to avoid errors with
+                # the database client (should be O(n))
+                str_value = (
+                    str_value.replace(NULL_BYTE, "")
+                    if NULL_BYTE in str_value
+                    else str_value
+                )
+                return str_value
             except Exception as exc:
-                logger.debug("Failed to parse bytes valud as string: %s", exc)
+                logger.debug("Failed to parse bytes value as string: %s", exc)
                 logger.debug(traceback.format_exc())
 
         return value.hex()

metadata/profiler/processor/core.py

@@ -33,7 +33,6 @@ from metadata.generated.schema.entity.data.table import (
     ColumnProfile,
     ColumnProfilerConfig,
     SystemProfile,
-    TableData,
     TableProfile,
 )
 from metadata.generated.schema.settings.settings import Settings
@@ -41,7 +40,7 @@ from metadata.generated.schema.tests.customMetric import (
     CustomMetric as CustomMetricEntity,
 )
 from metadata.generated.schema.type.basic import Timestamp
-from metadata.profiler.api.models import ProfilerResponse, ThreadPoolMetrics
+from metadata.profiler.api.models import ProfilerResponse, SampleData, ThreadPoolMetrics
 from metadata.profiler.interface.profiler_interface import ProfilerInterface
 from metadata.profiler.metrics.core import (
     ComposedMetric,
@@ -492,7 +491,12 @@ class Profiler(Generic[TMetric]):
             )
             self.compute_metrics()
 
-        if self.source_config.generateSampleData:
+        # We need the sample data for Sample Data or PII Sensitive processing.
+        # We'll nullify the Sample Data after the PII processing so that it's not stored.
+        if (
+            self.source_config.generateSampleData
+            or self.source_config.processPiiSensitive
+        ):
             sample_data = self.generate_sample_data()
         else:
             sample_data = None
@@ -510,7 +514,7 @@ class Profiler(Generic[TMetric]):
         return table_profile
 
     @calculate_execution_time(store=False)
-    def generate_sample_data(self) -> Optional[TableData]:
+    def generate_sample_data(self) -> Optional[SampleData]:
         """Fetch and ingest sample data
 
         Returns:
@@ -532,7 +536,10 @@ class Profiler(Generic[TMetric]):
                     SAMPLE_DATA_DEFAULT_COUNT, self.profiler_interface.sample_data_count
                 )
             ]
-            return table_data
+            return SampleData(
+                data=table_data, store=self.source_config.generateSampleData
+            )
+
         except Exception as err:
             logger.debug(traceback.format_exc())
             logger.warning(f"Error fetching sample data: {err}")

metadata/profiler/processor/models.py

@@ -15,11 +15,28 @@ JSON workflows to the profiler
 """
 from typing import List, Optional
 
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, BeforeValidator
+from typing_extensions import Annotated
 
 from metadata.profiler.metrics.registry import Metrics
 
 
+def valid_metric(value: str):
+    """
+    Validate that the input metrics are correctly named
+    and can be found in the Registry
+    """
+    if not Metrics.get(value.upper()):
+        raise ValueError(
+            f"Metric name {value} is not a proper metric name from the Registry"
+        )
+
+    return value.upper()
+
+
+ValidMetric = Annotated[str, BeforeValidator(valid_metric)]
+
+
 class ProfilerDef(BaseModel):
     """
     Incoming profiler definition from the
@@ -30,26 +47,4 @@ class ProfilerDef(BaseModel):
     timeout_seconds: Optional[
         int
     ] = None  # Stop running a query after X seconds and continue
-    metrics: Optional[
-        List[str]
-    ] = None  # names of currently supported Static and Composed metrics
-    # TBD:
-    # time_metrics: List[TimeMetricDef] = None
-    # custom_metrics: List[CustomMetricDef] = None
-    # rule_metrics: ...
-
-    # pylint: disable=no-self-argument
-    @validator("metrics", each_item=True)
-    def valid_metric(cls, value):
-        """
-        We are using cls as per pydantic docs
-
-        Validate that the input metrics are correctly named
-        and can be found in the Registry
-        """
-        if not Metrics.get(value.upper()):
-            raise ValueError(
-                f"Metric name {value} is not a proper metric name from the Registry"
-            )
-
-        return value.upper()
+    metrics: Optional[List[ValidMetric]] = None

metadata/workflow/base.py

@@ -289,6 +289,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
             for step in self.workflow_steps():
                 logger.info(
                     f"{step.name}: Processed {len(step.status.records)} records,"
+                    f" updated {len(step.status.updated_records)} records,"
                     f" filtered {len(step.status.filtered)} records,"
                     f" found {len(step.status.failures)} errors"
                 )

metadata/workflow/profiler.py

@@ -11,6 +11,11 @@
 """
 Workflow definition for the profiler
 """
+from typing import cast
+
+from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline import (
+    DatabaseServiceProfilerPipeline,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     OpenMetadataWorkflowConfig,
 )
@@ -58,9 +63,17 @@ class ProfilerWorkflow(IngestionWorkflow):
         self.source = source_class.create(self.config.model_dump(), self.metadata)
 
         profiler_processor = self._get_profiler_processor()
-        pii_processor = self._get_pii_processor()
         sink = self._get_sink()
-        self.steps = (profiler_processor, pii_processor, sink)
+
+        # Only instantiate the PII Processor on demand
+        source_config: DatabaseServiceProfilerPipeline = cast(
+            DatabaseServiceProfilerPipeline, self.config.source.sourceConfig.config
+        )
+        if source_config.processPiiSensitive:
+            pii_processor = self._get_pii_processor()
+            self.steps = (profiler_processor, pii_processor, sink)
+        else:
+            self.steps = (profiler_processor, sink)
 
     def test_connection(self):
         service_config = self.config.source.serviceConnection.root.config