oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/opctl/operator/lowcode/pii/model/guardrails.py

@@ -1,167 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import os
-import time
-from datetime import datetime
-
-from ads.common.object_storage_details import ObjectStorageDetails
-from ads.opctl import logger
-from ads.opctl.operator.lowcode.pii.constant import DataFrameColumn
-from ads.opctl.operator.lowcode.pii.model.pii import PiiScrubber, detect, scrub
-from ads.opctl.operator.lowcode.pii.model.report import (
-    PIIOperatorReport,
-    PiiReportPageSpec,
-    PiiReportSpec,
-)
-from ads.opctl.operator.lowcode.pii.operator_config import PiiOperatorConfig
-from ads.opctl.operator.lowcode.pii.utils import (
-    default_signer,
-    get_output_name,
-)
-from ads.opctl.operator.lowcode.common.utils import load_data, write_data
-from ads.opctl.operator.lowcode.common.errors import InvalidParameterError
-
-
-class PIIGuardrail:
-    def __init__(self, config: PiiOperatorConfig):
-        self.config = config
-        self.spec = config.spec
-        self.pii_scrubber = PiiScrubber(config=config)
-        self.scrubber = self.pii_scrubber.config_scrubber()
-
-        output_filename = get_output_name(
-            target_name=self.spec.output_directory.name,
-            given_name=self.spec.input_data.url,
-        )
-        self.dst_uri = os.path.join(self.spec.output_directory.url, output_filename)
-        self.config.spec.output_directory.name = output_filename
-
-        self.report_uri = os.path.join(
-            self.spec.output_directory.url,
-            self.spec.report.report_filename,
-        )
-
-        self.report_context: PiiReportSpec = PiiReportSpec.from_dict(
-            {
-                "run_summary": {
-                    "config": self.config,
-                    "selected_detectors": self.pii_scrubber.detectors,
-                    "selected_entities": self.pii_scrubber.entities,
-                    "selected_spacy_model": self.pii_scrubber.spacy_model_detectors,
-                    "show_rows": self.spec.report.show_rows,
-                    "show_sensitive_info": self.spec.report.show_sensitive_content,
-                    "src_uri": self.spec.input_data.url,
-                    "total_tokens": 0,
-                },
-                "run_details": {"rows": []},
-            }
-        )
-
-        self.storage_options = (
-            default_signer()
-            if ObjectStorageDetails.is_oci_path(self.spec.output_directory.url)
-            else {}
-        )
-        self.datasets = None
-
-    def _load_data(self, uri=None, storage_options=None):
-        """Loads input data."""
-        input_data_uri = uri or self.spec.input_data.url
-        logger.info(f"Loading input data from `{input_data_uri}` ...")
-
-        try:
-            self.datasets = load_data(
-                data_spec=self.spec.input_data,
-                storage_options=storage_options or self.storage_options,
-            )
-        except InvalidParameterError as e:
-            e.args = e.args + ("Invalid Parameter: input_data",)
-            raise e
-
-        return self
-
-    def process(self, **kwargs):
-        """Process input data."""
-        self.report_context.run_summary.timestamp = datetime.now().strftime(
-            "%d/%m/%Y %H:%M:%S"
-        )
-        start_time = time.time()
-
-        data = kwargs.pop("input_data", None) or self.datasets
-        report_uri = kwargs.pop("report_uri", None) or self.report_uri
-        dst_uri = kwargs.pop("dst_uri", None) or self.dst_uri
-
-        if not data:
-            try:
-                self._load_data()
-                data = self.datasets
-            except InvalidParameterError as e:
-                e.args = e.args + ("Invalid Parameter: input_data",)
-                raise e
-
-        # process user data
-        data[DataFrameColumn.REDACTED_TEXT] = data[self.spec.target_column].apply(
-            lambda x: scrub(x, scrubber=self.scrubber)
-        )
-        self.report_context.run_summary.elapsed_time = time.time() - start_time
-        self.report_context.run_summary.total_rows = len(data.index)
-
-        # save output data
-        if dst_uri:
-            logger.info(f"Saving data into `{dst_uri}` ...")
-            write_data(
-                data=data.loc[:, data.columns != self.spec.target_column],
-                filename=dst_uri,
-                format=None,
-                storage_options=kwargs.pop("storage_options", None)
-                or self.storage_options,
-            )
-
-        # prepare pii report
-        if report_uri:
-            logger.info(f"Generating report to `{report_uri}` ...")
-
-            data[DataFrameColumn.ENTITIES] = data[self.spec.target_column].apply(
-                lambda x: detect(text=x, scrubber=self.scrubber)
-            )
-
-            for i in data.index:
-                text = data[self.spec.target_column][i]
-                ent_col = data[DataFrameColumn.ENTITIES][i]
-                page = PiiReportPageSpec.from_dict(
-                    {
-                        "id": i,
-                        "total_tokens": len(ent_col),
-                        "entities": ent_col,
-                        "raw_text": text,
-                    }
-                )
-                self.report_context.run_details.rows.append(page)
-                self.report_context.run_summary.total_tokens += len(ent_col)
-
-            self._process_context()
-            PIIOperatorReport(
-                report_spec=self.report_context, report_uri=report_uri
-            ).make_view().save_report(
-                storage_options=kwargs.pop("storage_options", None)
-                or self.storage_options
-            )
-
-    def _process_context(self):
-        """Count different type of filth."""
-        statics = {}  # statics : count Filth type in total
-        rows = self.report_context.run_details.rows
-        for row in rows:
-            entities = row.entities
-            row_statics = {}  # count row
-            for ent in entities:
-                row_statics[ent.type] = row_statics.get(ent.type, 0) + 1
-                statics[ent.type] = statics.get(ent.type, 0) + 1
-
-            row.statics = row_statics.copy()
-
-        self.report_context.run_summary.statics = statics

ads/opctl/operator/lowcode/pii/model/pii.py

@@ -1,145 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from ads.common.decorator.runtime_dependency import (
-    OptionalDependency,
-    runtime_dependency,
-)
-from ads.opctl import logger
-from ads.opctl.operator.common.utils import _load_yaml_from_uri
-from ads.opctl.operator.lowcode.pii.model.factory import PiiDetectorFactory
-from ads.opctl.operator.lowcode.pii.constant import (
-    SupportedAction,
-    SupportedDetector,
-)
-from ads.opctl.operator.lowcode.pii.model.processor import (
-    POSTPROCESSOR_MAP,
-    SUPPORTED_REPLACER,
-    Remover,
-)
-
-
-class PiiScrubber:
-    """Class used for config scrubber and count the detectors in use."""
-
-    @runtime_dependency(module="scrubadub", install_from=OptionalDependency.PII)
-    def __init__(self, config):
-        logger.info(f"Loading config from {config}")
-        if isinstance(config, str):
-            config = _load_yaml_from_uri(config)
-
-        self.config = config
-        self.spec = (
-            self.config["spec"] if isinstance(self.config, dict) else self.config.spec
-        )
-        self.detector_spec = (
-            self.spec["detectors"]
-            if isinstance(self.spec, dict)
-            else self.spec.detectors
-        )
-
-        self.scrubber = scrubadub.Scrubber()
-
-        self.detectors = []
-        self.entities = []
-        self.spacy_model_detectors = []
-        self.post_processors = {}
-
-        self._reset_scrubber()
-
-    def _reset_scrubber(self):
-        # Clean up default detectors
-        defautls_enable = self.scrubber._detectors.copy()
-        for d in defautls_enable:
-            self.scrubber.remove_detector(d)
-
-    def _register(self, name, dtype, model, action, mask_with: str = None):
-        if action not in SupportedAction.values():
-            raise ValueError(
-                f"Not supported `action`: {action}. Please select from {SupportedAction.values()}."
-            )
-
-        detector = PiiDetectorFactory.get_detector(
-            detector_type=dtype, entity=name, model=model
-        )
-        self.scrubber.add_detector(detector)
-        self.entities.append(name)
-
-        if action == SupportedAction.ANONYMIZE:
-            entity = (
-                detector
-                if isinstance(detector, str)
-                else detector.filth_cls_map[name.upper()].type
-            )
-            if entity in SUPPORTED_REPLACER.keys():
-                replacer_name = SUPPORTED_REPLACER.get(entity).name
-                replacer = self.post_processors.get(
-                    replacer_name, POSTPROCESSOR_MAP.get(replacer_name)()
-                )
-                if hasattr(replacer, "_ENTITIES"):
-                    replacer._ENTITIES.append(name)
-                self.post_processors[replacer_name] = replacer
-            else:
-                raise ValueError(
-                    f"Not supported `action` {action} for this entity `{name}`. Please try with other action."
-                )
-
-        if action == SupportedAction.REMOVE:
-            remover = self.post_processors.get("remover", Remover())
-            remover._ENTITIES.append(name)
-            self.post_processors["remover"] = remover
-
-    def config_scrubber(self):
-        """Returns an instance of srubadub.Scrubber."""
-
-        self.scrubber.redact_spec_file = self.spec
-
-        for detector in self.detector_spec:
-            # example format for detector["name"]: default.phone or spacy.en_core_web_trf.person
-            d = detector["name"].split(".")
-            dtype = d[0]
-            dname = d[1] if len(d) == 2 else d[2]
-            model = None if len(d) == 2 else d[1]
-
-            action = detector.get("action", SupportedAction.MASK)
-            self._register(
-                name=dname,
-                dtype=dtype,
-                model=model,
-                action=action,
-            )
-            if dtype == SupportedDetector.SPACY:
-                exist = False
-                for spacy_detectors in self.spacy_model_detectors:
-                    if spacy_detectors["model"] == model:
-                        spacy_detectors["spacy_entites"].append(dname)
-                        exist = True
-                        break
-                if not exist:
-                    self.spacy_model_detectors.append(
-                        {"model": model, "spacy_entites": [dname]}
-                    )
-
-        self._register_post_processor()
-
-        self.detectors = list(self.scrubber._detectors.values())
-        return self.scrubber
-
-    def _register_post_processor(self):
-        for _, v in self.post_processors.items():
-            self.scrubber.add_post_processor(v)
-
-
-def scrub(text, config=None, scrubber=None):
-    if not scrubber:
-        scrubber = PiiScrubber(config=config).config_scrubber()
-    return scrubber.clean(text)
-
-
-def detect(text, config=None, scrubber=None):
-    if not scrubber:
-        scrubber = PiiScrubber(config=config).config_scrubber()
-    return list(scrubber.iter_filth(text, document_name=None))

ads/opctl/operator/lowcode/pii/model/processor/__init__.py

@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from .email_replacer import EmailReplacer
-from .mbi_replacer import MBIReplacer
-from .name_replacer import NameReplacer
-from .number_replacer import NumberReplacer
-from .remover import Remover
-
-POSTPROCESSOR_MAP = {
-    item.name.lower(): item
-    for item in [
-        NameReplacer,
-        NumberReplacer,
-        EmailReplacer,
-        MBIReplacer,
-        Remover,
-    ]
-}
-
-# Currently only support anonymization for the following entity.
-SUPPORTED_REPLACER = {
-    "name": NameReplacer,
-    "number": NumberReplacer,
-    "phone": NumberReplacer,
-    "social_security_number": NumberReplacer,
-    "fin": NumberReplacer,
-    "mrn": NumberReplacer,
-    "email": EmailReplacer,
-    "mbi": MBIReplacer,
-}

ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py

@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from ads.common.decorator.runtime_dependency import (
-    OptionalDependency,
-    runtime_dependency,
-)
-
-try:
-    import scrubadub
-except ImportError:
-    raise ModuleNotFoundError(
-        f"`scrubadub` module was not found. Please run "
-        f"`pip install {OptionalDependency.PII}`."
-    )
-
-
-class EmailReplacer(scrubadub.post_processors.PostProcessor):
-    name = "email_replacer"
-
-    @runtime_dependency(module="faker", install_from=OptionalDependency.PII)
-    def process_filth(self, filth_list):
-        from faker import Faker
-
-        for filth in filth_list:
-            if filth.replacement_string:
-                continue
-            if filth.type.lower() != "email":
-                continue
-            filth.replacement_string = Faker().email()
-        return filth_list

ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py

@@ -1,35 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import random
-import string
-
-from ads.common.decorator.runtime_dependency import OptionalDependency
-
-try:
-    import scrubadub
-except ImportError:
-    raise ModuleNotFoundError(
-        f"`scrubadub` module was not found. Please run "
-        f"`pip install {OptionalDependency.PII}`."
-    )
-
-
-class MBIReplacer(scrubadub.post_processors.PostProcessor):
-    name = "mbi_replacer"
-    CHAR_POOL = "ACDEFGHJKMNPQRTUVWXY"
-
-    def generate_mbi(self):
-        return "".join(random.choices(self.CHAR_POOL + string.digits, k=11))
-
-    def process_filth(self, filth_list):
-        for filth in filth_list:
-            if filth.replacement_string:
-                continue
-            if filth.type.lower() != "mbi":
-                continue
-            filth.replacement_string = self.generate_mbi()
-        return filth_list

ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py

@@ -1,225 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-
-from ads.common.decorator.runtime_dependency import (
-    OptionalDependency,
-    runtime_dependency,
-)
-
-try:
-    import scrubadub
-except ImportError:
-    raise ModuleNotFoundError(
-        f"`scrubadub` module was not found. Please run "
-        f"`pip install {OptionalDependency.PII}`."
-    )
-
-
-class NameReplacer(scrubadub.post_processors.PostProcessor):
-    name = "name_replacer"
-
-    @runtime_dependency(module="faker", install_from=OptionalDependency.PII)
-    @runtime_dependency(module="gender_guesser", install_from=OptionalDependency.PII)
-    def __init__(self, name: str = None, mapping: dict = None):
-        import gender_guesser.detector as gender_detector
-        from faker import Faker
-
-        if mapping:
-            self.mapping = mapping
-        else:
-            self.mapping = {}
-
-        self.gender_detector = gender_detector.Detector()
-        self.fake = Faker()
-        self.groups = {
-            "first": self.first_name_generator,
-            "middle": self.first_name_generator,
-            "last": self.last_name_generator,
-            "suffix": lambda x: "",
-        }
-        super().__init__(name)
-
-    def first_name_generator(self, name):
-        detected_gender = self.gender_detector.get_gender(name)
-        if "female" in detected_gender:
-            return self.fake.first_name_female()
-        elif "male" in detected_gender:
-            return self.fake.first_name_male()
-        return self.fake.first_name_nonbinary()
-
-    def last_name_generator(self, *args):
-        return self.fake.last_name()
-
-    def unwrap_filth(self, filth_list):
-        """Un-merge the filths if they have different types."""
-        processed = []
-        for filth in filth_list:
-            # MergedFilths has the property "filths"
-            # Do nothing if filth has a type already
-            if filth.type in ["unknown", "", None] and hasattr(filth, "filths"):
-                filth_types = set([f.type.lower() for f in filth.filths])
-                # Do nothing if the filth does not contain a name
-                if "name" not in filth_types:
-                    processed.append(filth)
-                    continue
-                if len(filth_types) > 1:
-                    processed.extend(filth.filths)
-                    continue
-                filth.type = filth.filths[0].type
-                filth.detector_name = filth.filths[0].detector_name
-            processed.append(filth)
-        return processed
-
-    @staticmethod
-    def has_initial(name: "nameparser.HumanName") -> bool:
-        for attr in ["first", "middle", "last"]:
-            if len(str(getattr(name, attr)).strip(".")) == 1:
-                return True
-        return False
-
-    @staticmethod
-    def has_non_initial(name: "nameparser.HumanName") -> bool:
-        for attr in ["first", "middle", "last"]:
-            if len(str(getattr(name, attr)).strip(".")) > 1:
-                return True
-        return False
-
-    @staticmethod
-    def generate_component(name_component: str, generator):
-        fake_component = generator(name_component)
-        if len(name_component.rstrip(".")) == 1:
-            fake_component = fake_component[0]
-            if name_component.endswith("."):
-                fake_component += "."
-        return fake_component
-
-    def save_name_mapping(
-        self, name: "nameparser.HumanName", fake_name: "nameparser.HumanName"
-    ):
-        """Saves the names with initials to the mapping so that a new name will not be generated.
-        For example, if name is "John Richard Doe", this method will save the following keys to the mapping:
-        - J Doe
-        - John D
-        - J R Doe
-        - John R D
-        - John R Doe
-        """
-        # Both first name and last name must be presented
-        if not name.first or not name.last:
-            return
-        # Remove any dot at the end of the name component.
-        for attr in ["first", "middle", "last"]:
-            setattr(name, attr, getattr(name, attr).rstrip("."))
-
-        self.mapping[
-            f"{name.first[0]} {name.last}"
-        ] = f"{fake_name.first[0]} {fake_name.last}"
-
-        self.mapping[
-            f"{name.first} {name.last[0]}"
-        ] = f"{fake_name.first} {fake_name.last[0]}"
-
-        if name.middle:
-            self.mapping[
-                f"{name.first[0]} {name.middle[0]} {name.last}"
-            ] = f"{fake_name.first[0]} {fake_name.middle[0]} {fake_name.last}"
-
-            self.mapping[
-                f"{name.first} {name.middle[0]} {name.last[0]}"
-            ] = f"{fake_name.first} {fake_name.middle[0]} {fake_name.last[0]}"
-
-            self.mapping[
-                f"{name.first} {name.middle[0]} {name.last}"
-            ] = f"{fake_name.first} {fake_name.middle[0]} {fake_name.last}"
-
-    @runtime_dependency(module="nameparser", install_from=OptionalDependency.PII)
-    def replace(self, text):
-        """Replaces a name with fake name.
-
-        Parameters
-        ----------
-        text : str or HumanName
-            The name to be replaced.
-            If text is a HumanName object, the object will be modified to have the new fake names.
-
-        Returns
-        -------
-        str
-            The replaced name as text.
-        """
-        from nameparser import HumanName
-
-        if isinstance(text, HumanName):
-            name = text
-        else:
-            name = HumanName(text)
-        skip = []
-        # Check if the name is given with initial for one of the first name/last name
-        key = None
-        if self.has_initial(name) and self.has_non_initial(name):
-            if name.middle:
-                key = f'{name.first.rstrip(".")} {name.middle.rstrip(".")} {name.last.rstrip(".")}'
-            else:
-                key = f'{name.first.rstrip(".")} {name.last.rstrip(".")}'
-            fake_name = self.mapping.get(key)
-            # If a fake name is found matching the first initial + last name or first name + last initial
-            # Replace the the initial with the corresponding initial
-            # and skip processing the first and last name in the replacement.
-            if fake_name:
-                fake_name = HumanName(fake_name)
-                name.first = fake_name.first
-                name.last = fake_name.last
-                skip = ["first", "last"]
-                if name.middle:
-                    name.middle = fake_name.middle
-                    skip.append("middle")
-        # Replace each component in the name
-        for attr, generator in self.groups.items():
-            if attr in skip:
-                continue
-            name_component = getattr(name, attr, None)
-            if not name_component:
-                continue
-            # Check if a fake name has been generated for this name
-            fake_component = self.mapping.get(name_component)
-            if not fake_component:
-                fake_component = self.generate_component(name_component, generator)
-                # Generate a unique fake name that is not already in the mapping
-                while fake_component and (
-                    fake_component in self.mapping.keys()
-                    or fake_component in self.mapping.values()
-                ):
-                    fake_component = self.generate_component(name_component, generator)
-                self.mapping[name_component] = fake_component
-            setattr(name, attr, fake_component)
-
-        # Save name with initials to mapping
-        original_name = text if isinstance(text, HumanName) else HumanName(text)
-        self.save_name_mapping(original_name, name)
-        return str(name)
-
-    @runtime_dependency(module="nameparser", install_from=OptionalDependency.PII)
-    def process_filth(self, filth_list):
-        from nameparser import HumanName
-
-        filth_list = self.unwrap_filth(filth_list)
-
-        name_filths = []
-        # Filter to keep only the names
-        for filth in filth_list:
-            if filth.replacement_string:
-                continue
-            if filth.type.lower() != "name":
-                continue
-            name_filths.append(filth)
-
-        # Sort reverse by last name so that names having a last name will be processed first.
-        # When a name is referred by last name (e.g. Mr. White), HumanName will parse it as first name.
-        name_filths.sort(key=lambda x: HumanName(x.text).last, reverse=True)
-        for filth in name_filths:
-            filth.replacement_string = self.replace(filth.text)
-        return filth_list