oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/opctl/operator/lowcode/pii/constant.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2023, 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+from ads.common.extended_enum import ExtendedEnum
+
+DEFAULT_SHOW_ROWS = 25
+DEFAULT_TIME_OUT = 5
+DEFAULT_COLOR = "#D6D3D1"
+DEFAULT_REPORT_FILENAME = "report.html"
+DEFAULT_TARGET_COLUMN = "target"
+
+
+class SupportedAction(ExtendedEnum):
+    """Supported action to process detected entities."""
+
+    MASK = "mask"
+    REMOVE = "remove"
+    ANONYMIZE = "anonymize"
+
+
+class SupportedDetector(ExtendedEnum):
+    """Supported pii detectors."""
+
+    DEFAULT = "default"
+    SPACY = "spacy"
+
+
+class DataFrameColumn(ExtendedEnum):
+    REDACTED_TEXT: str = "redacted_text"
+    ENTITIES: str = "entities_cols"
+
+
+class YamlKey(ExtendedEnum):
+    """Yaml key used in pii.yaml."""
+
+    pass
+
+
+YAML_KEYS = [
+    "detectors",
+    "custom_detectors",
+    "spacy_detectors",
+    "anonymization",
+    "name",
+    "label",
+    "patterns",
+    "model",
+    "named_entities",
+    "entities",
+]
+
+################
+# Report Const #
+################
+PII_REPORT_DESCRIPTION = (
+    "This report will offer a comprehensive overview of the redaction of personal identifiable information (PII) from the provided data."
+    "The `Summary` section will provide an executive summary of this process, including key statistics, configuration, and model usage."
+    "The `Details` section will offer a more granular analysis of each row of data, including relevant statistics."
+)
+DETAILS_REPORT_DESCRIPTION = "The following report will show the details on each row. You can view the highlighted named entities and their labels in the text under `TEXT` tab."
+
+FLAT_UI_COLORS = [
+    "#1ABC9C",
+    "#2ECC71",
+    "#3498DB",
+    "#9B59B6",
+    "#34495E",
+    "#16A085",
+    "#27AE60",
+    "#2980B9",
+    "#8E44AD",
+    "#2C3E50",
+    "#F1C40F",
+    "#E67E22",
+    "#E74C3C",
+    "#ECF0F1",
+    "#95A5A6",
+    "#F39C12",
+    "#D35400",
+    "#C0392B",
+    "#BDC3C7",
+    "#7F8C8D",
+]

ads/opctl/operator/lowcode/pii/environment.yaml

@@ -0,0 +1,17 @@
+name: pii
+channels:
+  - conda-forge
+dependencies:
+  - python=3.9
+  - pip
+  - pip:
+      - aiohttp
+      - report-creator
+      - gender_guesser
+      - nameparser
+      - oracle_ads[opctl]
+      - plotly
+      - scrubadub
+      - scrubadub_spacy
+      - spacy-transformers==1.2.5
+      - spacy==3.6.1

ads/opctl/operator/lowcode/pii/errors.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+class PIISchemaYamlError(Exception):
+    """Exception raised when there is an issue with the schema."""
+
+    def __init__(self, error: str):
+        super().__init__(
+            "Invalid PII operator specification. Check the YAML structure and ensure it "
+            "complies with the required schema for PII operator. \n"
+            f"{error}"
+        )
+
+
+class PIIInputDataError(Exception):
+    """Exception raised when there is an issue with input data."""
+
+    def __init__(self, error: str):
+        super().__init__(
+            "Invalid input data. Check the input data and ensure it "
+            "complies with the validation criteria. \n"
+            f"{error}"
+        )

ads/opctl/operator/lowcode/pii/model/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/operator/lowcode/pii/model/factory.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import uuid
+
+from ads.common.decorator.runtime_dependency import (
+    OptionalDependency,
+    runtime_dependency,
+)
+from ads.opctl.operator.lowcode.pii.constant import SupportedDetector
+from ads.opctl.operator.lowcode.pii.utils import construct_filth_cls_name
+
+
+class UnSupportedDetectorError(Exception):
+    def __init__(self, dtype: str):
+        super().__init__(
+            f"Detector: `{dtype}` "
+            f"is not supported. Supported models: {SupportedDetector.values()}"
+        )
+
+
+class PiiBaseDetector:
+    @classmethod
+    def construct(cls, **kwargs):
+        raise NotImplementedError
+
+
+class BuiltInDetector(PiiBaseDetector):
+    @classmethod
+    def construct(cls, entity, **kwargs):
+        return entity
+
+
+class SpacyDetector(PiiBaseDetector):
+    DEFAULT_SPACY_NAMED_ENTITIES = ["DATE", "FAC", "GPE", "LOC", "ORG", "PER", "PERSON"]
+    DEFAULT_SPACY_MODEL = "en_core_web_trf"
+
+    @classmethod
+    @runtime_dependency(module="scrubadub", install_from=OptionalDependency.PII)
+    @runtime_dependency(module="scrubadub_spacy", install_from=OptionalDependency.PII)
+    def construct(cls, entity, model, **kwargs):
+        from scrubadub.filth import Filth
+        from scrubadub_spacy.detectors.spacy import SpacyEntityDetector
+
+        spacy_entity_detector = SpacyEntityDetector(
+            named_entities=[entity],
+            name=f"spacy_{uuid.uuid4()}",
+            model=model,
+        )
+        if entity.upper() not in cls.DEFAULT_SPACY_NAMED_ENTITIES:
+            filth_cls = type(
+                construct_filth_cls_name(entity),
+                (Filth,),
+                {"type": entity.upper()},
+            )
+            spacy_entity_detector.filth_cls_map[entity.upper()] = filth_cls
+        return spacy_entity_detector
+
+
+class PiiDetectorFactory:
+    """
+    The factory class helps to instantiate proper detector object based on the detector config.
+    """
+
+    _MAP = {
+        SupportedDetector.DEFAULT: BuiltInDetector,
+        SupportedDetector.SPACY: SpacyDetector,
+    }
+
+    @classmethod
+    def get_detector(
+        cls,
+        detector_type,
+        entity,
+        model=None,
+    ):
+        if detector_type not in cls._MAP:
+            raise UnSupportedDetectorError(detector_type)
+
+        return cls._MAP[detector_type].construct(entity=entity, model=model)

ads/opctl/operator/lowcode/pii/model/guardrails.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import os
+import time
+from datetime import datetime
+
+from ads.common.object_storage_details import ObjectStorageDetails
+from ads.opctl import logger
+from ads.opctl.operator.lowcode.pii.constant import DataFrameColumn
+from ads.opctl.operator.lowcode.pii.model.pii import PiiScrubber, detect, scrub
+from ads.opctl.operator.lowcode.pii.model.report import (
+    PIIOperatorReport,
+    PiiReportPageSpec,
+    PiiReportSpec,
+)
+from ads.opctl.operator.lowcode.pii.operator_config import PiiOperatorConfig
+from ads.opctl.operator.lowcode.pii.utils import (
+    default_signer,
+    get_output_name,
+)
+from ads.opctl.operator.lowcode.common.utils import load_data, write_data
+from ads.opctl.operator.lowcode.common.errors import InvalidParameterError
+
+
+class PIIGuardrail:
+    def __init__(self, config: PiiOperatorConfig):
+        self.config = config
+        self.spec = config.spec
+        self.pii_scrubber = PiiScrubber(config=config)
+        self.scrubber = self.pii_scrubber.config_scrubber()
+
+        output_filename = get_output_name(
+            target_name=self.spec.output_directory.name,
+            given_name=self.spec.input_data.url,
+        )
+        self.dst_uri = os.path.join(self.spec.output_directory.url, output_filename)
+        self.config.spec.output_directory.name = output_filename
+
+        self.report_uri = os.path.join(
+            self.spec.output_directory.url,
+            self.spec.report.report_filename,
+        )
+
+        self.report_context: PiiReportSpec = PiiReportSpec.from_dict(
+            {
+                "run_summary": {
+                    "config": self.config,
+                    "selected_detectors": self.pii_scrubber.detectors,
+                    "selected_entities": self.pii_scrubber.entities,
+                    "selected_spacy_model": self.pii_scrubber.spacy_model_detectors,
+                    "show_rows": self.spec.report.show_rows,
+                    "show_sensitive_info": self.spec.report.show_sensitive_content,
+                    "src_uri": self.spec.input_data.url,
+                    "total_tokens": 0,
+                },
+                "run_details": {"rows": []},
+            }
+        )
+
+        self.storage_options = (
+            default_signer()
+            if ObjectStorageDetails.is_oci_path(self.spec.output_directory.url)
+            else {}
+        )
+        self.datasets = None
+
+    def _load_data(self, uri=None, storage_options=None):
+        """Loads input data."""
+        input_data_uri = uri or self.spec.input_data.url
+        logger.info(f"Loading input data from `{input_data_uri}` ...")
+
+        try:
+            self.datasets = load_data(
+                data_spec=self.spec.input_data,
+                storage_options=storage_options or self.storage_options,
+            )
+        except InvalidParameterError as e:
+            e.args = e.args + ("Invalid Parameter: input_data",)
+            raise e
+
+        return self
+
+    def process(self, **kwargs):
+        """Process input data."""
+        self.report_context.run_summary.timestamp = datetime.now().strftime(
+            "%d/%m/%Y %H:%M:%S"
+        )
+        start_time = time.time()
+
+        data = kwargs.pop("input_data", None) or self.datasets
+        report_uri = kwargs.pop("report_uri", None) or self.report_uri
+        dst_uri = kwargs.pop("dst_uri", None) or self.dst_uri
+
+        if not data:
+            try:
+                self._load_data()
+                data = self.datasets
+            except InvalidParameterError as e:
+                e.args = e.args + ("Invalid Parameter: input_data",)
+                raise e
+
+        # process user data
+        data[DataFrameColumn.REDACTED_TEXT] = data[self.spec.target_column].apply(
+            lambda x: scrub(x, scrubber=self.scrubber)
+        )
+        self.report_context.run_summary.elapsed_time = time.time() - start_time
+        self.report_context.run_summary.total_rows = len(data.index)
+
+        # save output data
+        if dst_uri:
+            logger.info(f"Saving data into `{dst_uri}` ...")
+            write_data(
+                data=data.loc[:, data.columns != self.spec.target_column],
+                filename=dst_uri,
+                format=None,
+                storage_options=kwargs.pop("storage_options", None)
+                or self.storage_options,
+            )
+
+        # prepare pii report
+        if report_uri:
+            logger.info(f"Generating report to `{report_uri}` ...")
+
+            data[DataFrameColumn.ENTITIES] = data[self.spec.target_column].apply(
+                lambda x: detect(text=x, scrubber=self.scrubber)
+            )
+
+            for i in data.index:
+                text = data[self.spec.target_column][i]
+                ent_col = data[DataFrameColumn.ENTITIES][i]
+                page = PiiReportPageSpec.from_dict(
+                    {
+                        "id": i,
+                        "total_tokens": len(ent_col),
+                        "entities": ent_col,
+                        "raw_text": text,
+                    }
+                )
+                self.report_context.run_details.rows.append(page)
+                self.report_context.run_summary.total_tokens += len(ent_col)
+
+            self._process_context()
+            PIIOperatorReport(
+                report_spec=self.report_context, report_uri=report_uri
+            ).make_view().save_report(
+                storage_options=kwargs.pop("storage_options", None)
+                or self.storage_options
+            )
+
+    def _process_context(self):
+        """Count different type of filth."""
+        statics = {}  # statics : count Filth type in total
+        rows = self.report_context.run_details.rows
+        for row in rows:
+            entities = row.entities
+            row_statics = {}  # count row
+            for ent in entities:
+                row_statics[ent.type] = row_statics.get(ent.type, 0) + 1
+                statics[ent.type] = statics.get(ent.type, 0) + 1
+
+            row.statics = row_statics.copy()
+
+        self.report_context.run_summary.statics = statics

ads/opctl/operator/lowcode/pii/model/pii.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.common.decorator.runtime_dependency import (
+    OptionalDependency,
+    runtime_dependency,
+)
+from ads.opctl import logger
+from ads.opctl.operator.common.utils import _load_yaml_from_uri
+from ads.opctl.operator.lowcode.pii.model.factory import PiiDetectorFactory
+from ads.opctl.operator.lowcode.pii.constant import (
+    SupportedAction,
+    SupportedDetector,
+)
+from ads.opctl.operator.lowcode.pii.model.processor import (
+    POSTPROCESSOR_MAP,
+    SUPPORTED_REPLACER,
+    Remover,
+)
+
+
+class PiiScrubber:
+    """Class used for config scrubber and count the detectors in use."""
+
+    @runtime_dependency(module="scrubadub", install_from=OptionalDependency.PII)
+    def __init__(self, config):
+        logger.info(f"Loading config from {config}")
+        if isinstance(config, str):
+            config = _load_yaml_from_uri(config)
+
+        self.config = config
+        self.spec = (
+            self.config["spec"] if isinstance(self.config, dict) else self.config.spec
+        )
+        self.detector_spec = (
+            self.spec["detectors"]
+            if isinstance(self.spec, dict)
+            else self.spec.detectors
+        )
+
+        self.scrubber = scrubadub.Scrubber()
+
+        self.detectors = []
+        self.entities = []
+        self.spacy_model_detectors = []
+        self.post_processors = {}
+
+        self._reset_scrubber()
+
+    def _reset_scrubber(self):
+        # Clean up default detectors
+        defautls_enable = self.scrubber._detectors.copy()
+        for d in defautls_enable:
+            self.scrubber.remove_detector(d)
+
+    def _register(self, name, dtype, model, action, mask_with: str = None):
+        if action not in SupportedAction.values():
+            raise ValueError(
+                f"Not supported `action`: {action}. Please select from {SupportedAction.values()}."
+            )
+
+        detector = PiiDetectorFactory.get_detector(
+            detector_type=dtype, entity=name, model=model
+        )
+        self.scrubber.add_detector(detector)
+        self.entities.append(name)
+
+        if action == SupportedAction.ANONYMIZE:
+            entity = (
+                detector
+                if isinstance(detector, str)
+                else detector.filth_cls_map[name.upper()].type
+            )
+            if entity in SUPPORTED_REPLACER.keys():
+                replacer_name = SUPPORTED_REPLACER.get(entity).name
+                replacer = self.post_processors.get(
+                    replacer_name, POSTPROCESSOR_MAP.get(replacer_name)()
+                )
+                if hasattr(replacer, "_ENTITIES"):
+                    replacer._ENTITIES.append(name)
+                self.post_processors[replacer_name] = replacer
+            else:
+                raise ValueError(
+                    f"Not supported `action` {action} for this entity `{name}`. Please try with other action."
+                )
+
+        if action == SupportedAction.REMOVE:
+            remover = self.post_processors.get("remover", Remover())
+            remover._ENTITIES.append(name)
+            self.post_processors["remover"] = remover
+
+    def config_scrubber(self):
+        """Returns an instance of srubadub.Scrubber."""
+
+        self.scrubber.redact_spec_file = self.spec
+
+        for detector in self.detector_spec:
+            # example format for detector["name"]: default.phone or spacy.en_core_web_trf.person
+            d = detector["name"].split(".")
+            dtype = d[0]
+            dname = d[1] if len(d) == 2 else d[2]
+            model = None if len(d) == 2 else d[1]
+
+            action = detector.get("action", SupportedAction.MASK)
+            self._register(
+                name=dname,
+                dtype=dtype,
+                model=model,
+                action=action,
+            )
+            if dtype == SupportedDetector.SPACY:
+                exist = False
+                for spacy_detectors in self.spacy_model_detectors:
+                    if spacy_detectors["model"] == model:
+                        spacy_detectors["spacy_entites"].append(dname)
+                        exist = True
+                        break
+                if not exist:
+                    self.spacy_model_detectors.append(
+                        {"model": model, "spacy_entites": [dname]}
+                    )
+
+        self._register_post_processor()
+
+        self.detectors = list(self.scrubber._detectors.values())
+        return self.scrubber
+
+    def _register_post_processor(self):
+        for _, v in self.post_processors.items():
+            self.scrubber.add_post_processor(v)
+
+
+def scrub(text, config=None, scrubber=None):
+    if not scrubber:
+        scrubber = PiiScrubber(config=config).config_scrubber()
+    return scrubber.clean(text)
+
+
+def detect(text, config=None, scrubber=None):
+    if not scrubber:
+        scrubber = PiiScrubber(config=config).config_scrubber()
+    return list(scrubber.iter_filth(text, document_name=None))

ads/opctl/operator/lowcode/pii/model/processor/__init__.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from .email_replacer import EmailReplacer
+from .mbi_replacer import MBIReplacer
+from .name_replacer import NameReplacer
+from .number_replacer import NumberReplacer
+from .remover import Remover
+
+POSTPROCESSOR_MAP = {
+    item.name.lower(): item
+    for item in [
+        NameReplacer,
+        NumberReplacer,
+        EmailReplacer,
+        MBIReplacer,
+        Remover,
+    ]
+}
+
+# Currently only support anonymization for the following entity.
+SUPPORTED_REPLACER = {
+    "name": NameReplacer,
+    "number": NumberReplacer,
+    "phone": NumberReplacer,
+    "social_security_number": NumberReplacer,
+    "fin": NumberReplacer,
+    "mrn": NumberReplacer,
+    "email": EmailReplacer,
+    "mbi": MBIReplacer,
+}

ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.common.decorator.runtime_dependency import (
+    OptionalDependency,
+    runtime_dependency,
+)
+
+try:
+    import scrubadub
+except ImportError:
+    raise ModuleNotFoundError(
+        f"`scrubadub` module was not found. Please run "
+        f"`pip install {OptionalDependency.PII}`."
+    )
+
+
+class EmailReplacer(scrubadub.post_processors.PostProcessor):
+    name = "email_replacer"
+
+    @runtime_dependency(module="faker", install_from=OptionalDependency.PII)
+    def process_filth(self, filth_list):
+        from faker import Faker
+
+        for filth in filth_list:
+            if filth.replacement_string:
+                continue
+            if filth.type.lower() != "email":
+                continue
+            filth.replacement_string = Faker().email()
+        return filth_list

ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import random
+import string
+
+from ads.common.decorator.runtime_dependency import OptionalDependency
+
+try:
+    import scrubadub
+except ImportError:
+    raise ModuleNotFoundError(
+        f"`scrubadub` module was not found. Please run "
+        f"`pip install {OptionalDependency.PII}`."
+    )
+
+
+class MBIReplacer(scrubadub.post_processors.PostProcessor):
+    name = "mbi_replacer"
+    CHAR_POOL = "ACDEFGHJKMNPQRTUVWXY"
+
+    def generate_mbi(self):
+        return "".join(random.choices(self.CHAR_POOL + string.digits, k=11))
+
+    def process_filth(self, filth_list):
+        for filth in filth_list:
+            if filth.replacement_string:
+                continue
+            if filth.type.lower() != "mbi":
+                continue
+            filth.replacement_string = self.generate_mbi()
+        return filth_list